fix: tuner heartbeat also recorded in DB — file-only heartbeat was unverifiable
The 07-22 'tuner NOT running' alert could not be diagnosed remotely: the heartbeat is a file on the scheduler container's volume, invisible to everything else, and indistinguishable from a stale scheduler image or a DATA_DIR mismatch. _write_heartbeat now also inserts a telemetry_events row (best-effort); the boot staleness check falls back to the DB row when the file is absent, so 'tuner ran, no profitable config' stops looking like 'tuner dead'. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
148e300af4
commit
625abee66f
2 files changed
+27
No files matched your search
@@ -40,6 +40,19 @@ def _write_heartbeat(outcome: str) -> None:
|
||||
"outcome": outcome}, f)
|
||||
except Exception: # noqa: BLE001 — a heartbeat must never break tuning
|
||||
pass
|
||||
# ALSO record in the DB: the file lives on one container's volume, so a
|
||||
# DATA_DIR mismatch / stale scheduler image makes it silently unverifiable
|
||||
# from anywhere else (the 2026-07-22 "tuner NOT running" alert autopsy).
|
||||
# The DB row is queryable remotely and survives container churn.
|
||||
try:
|
||||
con = open_research_db("market")
|
||||
con.execute(
|
||||
"INSERT INTO telemetry_events (event_type, source, reason, metadata_json) "
|
||||
"VALUES (?, ?, ?, ?)",
|
||||
["tuner_heartbeat", "nightly_tune", outcome,
|
||||
json.dumps({"ran_at": datetime.now(timezone.utc).isoformat()})])
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
@@ -163,6 +163,20 @@ def _load_learned(now: datetime | None = None) -> dict:
|
||||
hb_age = _learned_params_age_days(json.load(fh).get("ran_at"), now)
|
||||
except Exception: # noqa: BLE001
|
||||
hb_age = None
|
||||
if hb_age is None:
|
||||
# file may live on a different container's volume — the DB heartbeat
|
||||
# is the durable record (see nightly_tune._write_heartbeat)
|
||||
try:
|
||||
from research.multi_schema import open_research_db
|
||||
r = open_research_db("market").execute(
|
||||
"SELECT metadata_json FROM telemetry_events "
|
||||
"WHERE event_type='tuner_heartbeat' ORDER BY created_at DESC LIMIT 1"
|
||||
).fetchone()
|
||||
if r and r[0]:
|
||||
hb_age = _learned_params_age_days(
|
||||
json.loads(r[0]).get("ran_at"), now)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
if hb_age is not None and hb_age <= max_age:
|
||||
# tuner IS running — params are old only because nothing profitable
|
||||
# was found. Informational, not an alarm.
|
||||
|
||||
Reference in new issue
Block a user