From bbbcc4e18b4df90dad3a60554469fa8f4692f6b5 Mon Sep 17 00:00:00 2001 From: momentum dev Date: Sat, 4 Jul 2026 13:57:52 -0400 Subject: [PATCH] Nightly events retention: purge observability-spam events >14 days The events table hit 3.5 GB / 1.8M rows, ~80% of it module_tick / criteria_evaluated / signal_blocked / account_* snapshots that are only useful for a few days of debugging. Trading-decision events are kept forever. One-time purge (353k rows) + missing hot-path indexes (raw_news_items payload_hash + fetched_at: 911k seq scans -> index-only) and per-table autovacuum tuning applied directly to the DB. Co-Authored-By: Claude Opus 4.8 --- research/premarket_eval.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/research/premarket_eval.py b/research/premarket_eval.py index 7c2231f..08850bb 100644 --- a/research/premarket_eval.py +++ b/research/premarket_eval.py @@ -89,6 +89,20 @@ def main() -> None: f.write(line + "\n") except Exception as exc: # noqa: BLE001 — never let the refetch break the eval print(f"SIP refetch skipped: {exc}") + # nightly DB housekeeping: observability-spam events (module ticks, per-symbol criteria + # evaluations, account snapshots) are only useful for a few days of debugging but were + # ~80% of the events table (3.5 GB before the 2026-07-04 purge). Trading-decision events + # are kept forever. Autovacuum (tuned to 2%) reclaims the space. + try: + spam = ("module_tick", "criteria_evaluated", "signal_blocked", + "account_summary_updated", "account_positions_updated", "account_orders_updated") + ph = ",".join(["?"] * len(spam)) + cur = con.execute( + f"DELETE FROM events WHERE event_type IN ({ph}) " + "AND timestamp < now() - interval '14 days'", list(spam)) + print(f"events retention: purged {getattr(cur, 'rowcount', '?')} spam rows >14d") + except Exception as exc: # noqa: BLE001 — housekeeping must never break the eval + print(f"events retention skipped: {exc}") labeler.build_premarket(con, rebuild=True) # rebuild from the grown DB m = _metrics(con) ts = datetime.now(timezone.utc).isoformat(timespec="seconds")