From 5b95a9712cf6084808d811bb16aee47d90803d26 Mon Sep 17 00:00:00 2001 From: momentum dev Date: Mon, 13 Jul 2026 16:37:01 -0400 Subject: [PATCH] qabot: dead-man's switch on the loop's event heartbeat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The loop's Telegram notifier runs inside the loop process — loop-down means alerts-down. The always-up qabot now reads max(events.timestamp) every ~120s (ET-naive vs ET wall clock per RUNBOOK): stale > DEADMAN_STALE_MIN during market hours (holiday-aware via runtime.market_calendar) alerts at most every 30 min; DEADMAN_DAILY_PING sends one OK-ping after 20:30 ET. Injected clock/read/send; 6 unit tests, no network or DB. Co-Authored-By: Claude Fable 5 --- .env.example | 4 ++ deploy/docker/truenas-compose.yml | 2 + telegram_bot.py | 98 ++++++++++++++++++++++++++++++- tests/unit/test_qabot_deadman.py | 69 ++++++++++++++++++++++ 4 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_qabot_deadman.py diff --git a/.env.example b/.env.example index e5e9129..4e1596d 100644 --- a/.env.example +++ b/.env.example @@ -46,6 +46,10 @@ DISCORD_DEVELOPER_WEBHOOK_URL= # Telegram notifier (bot @traitor_momentum_bot) — gated; no-op if unset TELEGRAM_BOT_TOKEN= TELEGRAM_CHAT_ID='2082795871' +# Dead-man's switch (qabot container): the loop's own notifier dies WITH the +# loop, so the always-up qabot watches max(events.timestamp) from outside. +DEADMAN_STALE_MIN=5 # alert when no events for this many minutes during market hours +DEADMAN_DAILY_PING=1 # nightly OK-ping after 20:30 ET — proves the watchdog itself is alive #Market Data these are free plans so work within days/months capacity. Alpha_Vantage_Api_Key= diff --git a/deploy/docker/truenas-compose.yml b/deploy/docker/truenas-compose.yml index b89c76b..e9e0341 100644 --- a/deploy/docker/truenas-compose.yml +++ b/deploy/docker/truenas-compose.yml @@ -26,6 +26,8 @@ x-app: &app ALPACA_PAPER_SYNC_ENABLED: '1' ALPACA_SECRET_KEY: '' DATABASE_URL: 'postgresql://momentum:@postgres:5432/momentum' + DEADMAN_DAILY_PING: '1' + DEADMAN_STALE_MIN: '5' DISCOVER_INTERVAL_SECONDS: '120' FMP_API_KEY: '' INGEST_STALE_BLOCK: '0' diff --git a/telegram_bot.py b/telegram_bot.py index 345b6e4..b8cb433 100644 --- a/telegram_bot.py +++ b/telegram_bot.py @@ -9,8 +9,14 @@ per-symbol dive when the question names a ticker) and asks the local Ollama mode to answer. Falls back to the raw snapshot if Ollama is down. Never accepts trade commands. Sends a help menu so the user knows what it can answer (direction). +Also the DEAD-MAN'S SWITCH: the loop's own notifier lives inside the loop +process and cannot report the loop's death — this always-up bot watches the +event heartbeat (max(events.timestamp)) from outside and alerts when it goes +stale during market hours. + Env: TELEGRAM_BOT_TOKEN, TELEGRAM_CHAT_ID (required), DATABASE_URL, -OLLAMA_HOST / OLLAMA_MODEL / OLLAMA_TIMEOUT_SECONDS / OLLAMA_ENABLED. +OLLAMA_HOST / OLLAMA_MODEL / OLLAMA_TIMEOUT_SECONDS / OLLAMA_ENABLED, +DEADMAN_STALE_MIN / DEADMAN_DAILY_PING. """ from __future__ import annotations @@ -22,9 +28,15 @@ import re import time import urllib.request from collections import Counter +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +from runtime.market_calendar import is_regular_hours logger = logging.getLogger("telegram_bot") +_ET = ZoneInfo("America/New_York") + _API = "https://api.telegram.org" _ACTION_WORDS = ("buy ", "sell ", "flatten", "close all", "cancel ", "short ", "liquidate", "exit all", "place order", "submit order", "go long") @@ -215,6 +227,84 @@ def answer(store, question: str, *, host: str, model: str, timeout: int, return f"(LLM unavailable — here's the raw snapshot)\n{context}" +# ------------------------------------------------------------------- dead-man +def _now_et() -> datetime: + """ET wall clock, tz-NAIVE — events.timestamp is ET-naive (RUNBOOK), so the + staleness math must be naive-vs-naive or it reads 4-5h stale from UTC.""" + return datetime.now(_ET).replace(tzinfo=None) + + +def _max_event_ts(store) -> datetime | None: + """Newest events.timestamp as a naive datetime (None on an empty store). + Raw query on purpose: query_events() orders ASC and stringifies timestamps.""" + cur = store.con.cursor() + cur.execute("SELECT max(timestamp) FROM events") + row = cur.fetchone() + return row[0] if row else None + + +class DeadmanCheck: + """Loop-death watchdog. During market hours (holiday-aware, from + runtime.market_calendar) a heartbeat older than DEADMAN_STALE_MIN minutes + raises ONE alert, re-raised at most every REALERT_MIN. After 20:30 ET a + once-daily ping (DEADMAN_DAILY_PING=1, default on) proves the watchdog + itself is alive. Clock/read/send are injectable for tests.""" + + CHECK_EVERY_S = 120 + REALERT_MIN = 30 + + def __init__(self, read_max_ts, send, *, stale_min=None, daily_ping=None, + now_et=_now_et, is_market_hours=is_regular_hours): + self._read_max_ts = read_max_ts + self._send = send + self._stale_min = float(stale_min if stale_min is not None + else os.environ.get("DEADMAN_STALE_MIN", "5")) + dp = daily_ping if daily_ping is not None \ + else os.environ.get("DEADMAN_DAILY_PING", "1") + self._daily_ping = str(dp) not in ("0", "false", "False") + self._now_et = now_et + self._is_market_hours = is_market_hours + self._next_check = 0.0 + self._last_alerted: datetime | None = None + self._last_ping_date = None + + def maybe_check(self) -> str | None: + """Rate-limited entry point for the poll loop (wakes at least ~every 60s).""" + if time.monotonic() < self._next_check: + return None + self._next_check = time.monotonic() + self.CHECK_EVERY_S + return self.check() + + def check(self) -> str | None: + """Run one check now; returns the message sent, if any.""" + now = self._now_et() + try: + max_ts = self._read_max_ts() + except Exception: # noqa: BLE001 — a flaky DB read must not kill the bot + logger.warning("deadman: heartbeat read failed", exc_info=True) + return None + stale_min = None if max_ts is None \ + else (now - max_ts).total_seconds() / 60.0 + msg = None + if self._is_market_hours(now) and (stale_min is None or stale_min > self._stale_min): + if (self._last_alerted is None + or now - self._last_alerted >= timedelta(minutes=self.REALERT_MIN)): + desc = "no events ever" if stale_min is None \ + else f"no events for {stale_min:.0f} min" + msg = f"⚠️ DEAD-MAN: {desc} during market hours — loop may be down." + self._last_alerted = now + elif (self._daily_ping and (now.hour, now.minute) >= (20, 30) + and self._last_ping_date != now.date()): + self._last_ping_date = now.date() + if max_ts is not None and max_ts.date() == now.date(): + msg = f"✅ heartbeat: loop wrote events today; last event {max_ts}." + else: + msg = f"⚠️ heartbeat: NO events today; last event {max_ts}." + if msg: + self._send(msg) + return msg + + # ------------------------------------------------------------------- transport def is_authorized(chat_id, allowed) -> bool: return allowed is not None and str(chat_id) == str(allowed) @@ -251,9 +341,15 @@ def main() -> None: store = EventStore("momentum") logger.info("Q&A bot online (chat %s, ollama=%s %s)", allowed, ollama_enabled, model) _send(token, allowed, HELP_TEXT) + deadman = DeadmanCheck(lambda: _max_event_ts(store), + lambda text: _send(token, allowed, text)) offset = None while True: + try: + deadman.maybe_check() + except Exception: # noqa: BLE001 — the watchdog must never kill the bot + logger.warning("deadman check failed", exc_info=True) try: url = f"{_API}/bot{token}/getUpdates?timeout=50" if offset is not None: diff --git a/tests/unit/test_qabot_deadman.py b/tests/unit/test_qabot_deadman.py new file mode 100644 index 0000000..2f5d770 --- /dev/null +++ b/tests/unit/test_qabot_deadman.py @@ -0,0 +1,69 @@ +"""Dead-man's switch in the qabot — alert on a stale event heartbeat during +market hours, at most every 30 min, silent when fresh or when the market is +closed. Clock, heartbeat read, and send are all injected: no Telegram, no +Postgres (uses the real holiday-aware market calendar with fixed dates).""" + +from datetime import datetime + +from telegram_bot import DeadmanCheck + +TUE_10AM = datetime(2026, 7, 7, 10, 0) # regular trading day +SAT_10AM = datetime(2026, 7, 11, 10, 0) # weekend +TUE_EVENING = datetime(2026, 7, 7, 20, 35) # after the 20:30 ping gate + + +def make(now, max_ts, **kw): + """DeadmanCheck with a settable clock/heartbeat and a capturing send.""" + state = {"now": now, "max_ts": max_ts, "sent": []} + dm = DeadmanCheck(lambda: state["max_ts"], state["sent"].append, + stale_min=kw.pop("stale_min", 5), + daily_ping=kw.pop("daily_ping", "1"), + now_et=lambda: state["now"], **kw) + return dm, state + + +def test_stale_during_market_hours_alerts_once(): + dm, st = make(TUE_10AM, TUE_10AM.replace(hour=9, minute=40)) # 20 min stale + msg = dm.check() + assert msg is not None and "DEAD-MAN" in msg and "20 min" in msg + assert st["sent"] == [msg] + + +def test_stale_realert_suppressed_within_30_min(): + dm, st = make(TUE_10AM, TUE_10AM.replace(hour=9, minute=40)) + assert dm.check() is not None + st["now"] = TUE_10AM.replace(minute=10) # +10 min, still stale + assert dm.check() is None + assert len(st["sent"]) == 1 + st["now"] = TUE_10AM.replace(minute=31) # 31 min after first alert + assert dm.check() is not None # re-alert allowed + assert len(st["sent"]) == 2 + + +def test_fresh_heartbeat_no_alert(): + dm, st = make(TUE_10AM, datetime(2026, 7, 7, 9, 59)) # 1 min old + assert dm.check() is None + assert st["sent"] == [] + + +def test_stale_outside_market_hours_no_alert(): + dm, st = make(SAT_10AM, datetime(2026, 7, 6, 15, 59)) # very stale, Saturday + assert dm.check() is None + assert st["sent"] == [] + + +def test_nightly_ping_once_after_2030(): + dm, st = make(datetime(2026, 7, 7, 20, 0), datetime(2026, 7, 7, 15, 59)) + assert dm.check() is None # before 20:30: no ping + st["now"] = TUE_EVENING + msg = dm.check() + assert msg is not None and "heartbeat" in msg and "✅" in msg + st["now"] = TUE_EVENING.replace(minute=50) + assert dm.check() is None # once per day only + assert len(st["sent"]) == 1 + + +def test_nightly_ping_flags_a_day_with_no_events(): + dm, st = make(TUE_EVENING, datetime(2026, 7, 6, 15, 59)) # last event yesterday + msg = dm.check() + assert msg is not None and "NO events today" in msg