fix: tuner heartbeat also recorded in DB — file-only heartbeat was unverifiable

The 07-22 'tuner NOT running' alert could not be diagnosed remotely: the
heartbeat is a file on the scheduler container's volume, invisible to
everything else, and indistinguishable from a stale scheduler image or a
DATA_DIR mismatch. _write_heartbeat now also inserts a telemetry_events
row (best-effort); the boot staleness check falls back to the DB row when
the file is absent, so 'tuner ran, no profitable config' stops looking
like 'tuner dead'.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
momentum devandClaude Fable 5 committed 2026-07-22 19:35:50 -05:00
1 parent 148e300af4
commit 625abee66f
2 files changed
+27

No files matched your search

+13
View File
@@ -40,6 +40,19 @@ def _write_heartbeat(outcome: str) -> None:
"outcome": outcome}, f)
except Exception: # noqa: BLE001 — a heartbeat must never break tuning
pass
# ALSO record in the DB: the file lives on one container's volume, so a
# DATA_DIR mismatch / stale scheduler image makes it silently unverifiable
# from anywhere else (the 2026-07-22 "tuner NOT running" alert autopsy).
# The DB row is queryable remotely and survives container churn.
try:
con = open_research_db("market")
con.execute(
"INSERT INTO telemetry_events (event_type, source, reason, metadata_json) "
"VALUES (?, ?, ?, ?)",
["tuner_heartbeat", "nightly_tune", outcome,
json.dumps({"ran_at": datetime.now(timezone.utc).isoformat()})])
except Exception: # noqa: BLE001
pass
def main():
+14
View File
@@ -163,6 +163,20 @@ def _load_learned(now: datetime | None = None) -> dict:
hb_age = _learned_params_age_days(json.load(fh).get("ran_at"), now)
except Exception: # noqa: BLE001
hb_age = None
if hb_age is None:
# file may live on a different container's volume — the DB heartbeat
# is the durable record (see nightly_tune._write_heartbeat)
try:
from research.multi_schema import open_research_db
r = open_research_db("market").execute(
"SELECT metadata_json FROM telemetry_events "
"WHERE event_type='tuner_heartbeat' ORDER BY created_at DESC LIMIT 1"
).fetchone()
if r and r[0]:
hb_age = _learned_params_age_days(
json.loads(r[0]).get("ran_at"), now)
except Exception: # noqa: BLE001
pass
if hb_age is not None and hb_age <= max_age:
# tuner IS running — params are old only because nothing profitable
# was found. Informational, not an alarm.