From 625abee66f34784a42ae029e916107293f7a7ee7 Mon Sep 17 00:00:00 2001
From: momentum dev <dev@momentum.local>
Date: Wed, 22 Jul 2026 19:35:50 -0500
Subject: [PATCH] =?UTF-8?q?fix:=20tuner=20heartbeat=20also=20recorded=20in?=
 =?UTF-8?q?=20DB=20=E2=80=94=20file-only=20heartbeat=20was=20unverifiable?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The 07-22 'tuner NOT running' alert could not be diagnosed remotely: the
heartbeat is a file on the scheduler container's volume, invisible to
everything else, and indistinguishable from a stale scheduler image or a
DATA_DIR mismatch. _write_heartbeat now also inserts a telemetry_events
row (best-effort); the boot staleness check falls back to the DB row when
the file is absent, so 'tuner ran, no profitable config' stops looking
like 'tuner dead'.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 nightly_tune.py   | 13 +++++++++++++
 run_live_paper.py | 14 ++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/nightly_tune.py b/nightly_tune.py
index b6286f0..c19dd28 100644
--- a/nightly_tune.py
+++ b/nightly_tune.py
@@ -40,6 +40,19 @@ def _write_heartbeat(outcome: str) -> None:
                        "outcome": outcome}, f)
     except Exception:  # noqa: BLE001 — a heartbeat must never break tuning
         pass
+    # ALSO record in the DB: the file lives on one container's volume, so a
+    # DATA_DIR mismatch / stale scheduler image makes it silently unverifiable
+    # from anywhere else (the 2026-07-22 "tuner NOT running" alert autopsy).
+    # The DB row is queryable remotely and survives container churn.
+    try:
+        con = open_research_db("market")
+        con.execute(
+            "INSERT INTO telemetry_events (event_type, source, reason, metadata_json) "
+            "VALUES (?, ?, ?, ?)",
+            ["tuner_heartbeat", "nightly_tune", outcome,
+             json.dumps({"ran_at": datetime.now(timezone.utc).isoformat()})])
+    except Exception:  # noqa: BLE001
+        pass
 
 
 def main():
diff --git a/run_live_paper.py b/run_live_paper.py
index 08e2100..e36e511 100644
--- a/run_live_paper.py
+++ b/run_live_paper.py
@@ -163,6 +163,20 @@ def _load_learned(now: datetime | None = None) -> dict:
                     hb_age = _learned_params_age_days(json.load(fh).get("ran_at"), now)
         except Exception:  # noqa: BLE001
             hb_age = None
+        if hb_age is None:
+            # file may live on a different container's volume — the DB heartbeat
+            # is the durable record (see nightly_tune._write_heartbeat)
+            try:
+                from research.multi_schema import open_research_db
+                r = open_research_db("market").execute(
+                    "SELECT metadata_json FROM telemetry_events "
+                    "WHERE event_type='tuner_heartbeat' ORDER BY created_at DESC LIMIT 1"
+                ).fetchone()
+                if r and r[0]:
+                    hb_age = _learned_params_age_days(
+                        json.loads(r[0]).get("ran_at"), now)
+            except Exception:  # noqa: BLE001
+                pass
         if hb_age is not None and hb_age <= max_age:
             # tuner IS running — params are old only because nothing profitable
             # was found. Informational, not an alarm.