momentum devandClaude Fable 5
12f7fefec0
- runtime/retention.py: prune 8 high-volume debug event types older than RETENTION_DAYS (default 14) once/day at RETENTION_HOUR (default 03:00). Audit events (orders, fills, closes, risk rules) are never pruned. The events table hit 6 GB in 3 weeks at ~340k rows/day; this caps it. - runtime/telemetry_push.py: POST a JSON snapshot (equity, day realized P&L, trades/wins, open positions, last-event age) to TELEMETRY_PUSH_URL every TELEMETRY_PUSH_INTERVAL_SECONDS (default 300). Unset URL = step disabled. Push failures are reported in the step result, never raised — an external outage can't touch the loop. Works as healthchecks.io dead-man ping + payload log in one. - wired both as scheduler steps in run_live_paper (retention gated by RETENTION_ENABLED, telemetry by TELEMETRY_PUSH_URL presence) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
61 lines
2.2 KiB
Python
61 lines
2.2 KiB
Python
"""Event-store retention: prune high-volume debug event types.
|
|
|
|
The events table grows ~340k rows/day and ~95% of it is per-tick diagnostic
|
|
spam (criteria scores, heartbeats, account snapshots). Left alone it reached
|
|
6 GB in three weeks and unindexed scans took minutes. Orders, fills, closes,
|
|
and risk events are the audit trail — never pruned.
|
|
"""
|
|
|
|
from datetime import datetime, timedelta
|
|
import os
|
|
|
|
# High-volume diagnostic types safe to expire. Anything NOT listed is kept
|
|
# forever (order_*, position_*, risk_rule_triggered, session_summary, ...).
|
|
PRUNE_EVENT_TYPES = (
|
|
"criteria_evaluated",
|
|
"signal_blocked",
|
|
"module_tick",
|
|
"entry_gate_evaluated",
|
|
"account_summary_updated",
|
|
"account_positions_updated",
|
|
"account_orders_updated",
|
|
"symbol_evaluation_detail",
|
|
)
|
|
|
|
|
|
def prune_events(store, days: int = 14, event_types=PRUNE_EVENT_TYPES,
|
|
now: datetime | None = None) -> dict:
|
|
"""Delete prunable events older than ``days``. Idempotent."""
|
|
cutoff = (now or datetime.now()) - timedelta(days=days)
|
|
placeholders = ",".join("?" for _ in event_types)
|
|
with store._lock:
|
|
res = store.con.execute(
|
|
f"DELETE FROM events WHERE event_type IN ({placeholders})"
|
|
f" AND timestamp < ?",
|
|
[*event_types, cutoff],
|
|
)
|
|
store.con.commit()
|
|
deleted = getattr(getattr(res, "_cur", None), "rowcount", -1)
|
|
return {"deleted": deleted, "cutoff": cutoff.isoformat(timespec="seconds")}
|
|
|
|
|
|
_last_run: str | None = None # date of the last prune (process-local; restart re-runs, idempotent)
|
|
|
|
|
|
def maybe_prune(store, now: datetime | None = None) -> str:
|
|
"""Scheduler step: run prune_events once per day in the configured hour
|
|
(default 03:00 container-local — market quiet, premarket discovery not yet
|
|
running). Safe to call every scheduler pass."""
|
|
global _last_run
|
|
now = now or datetime.now()
|
|
hour = int(os.environ.get("RETENTION_HOUR", "3"))
|
|
days = int(os.environ.get("RETENTION_DAYS", "14"))
|
|
today = now.date().isoformat()
|
|
if now.hour != hour:
|
|
return "idle"
|
|
if _last_run == today:
|
|
return "done-today"
|
|
result = prune_events(store, days=days, now=now)
|
|
_last_run = today
|
|
return f"pruned {result['deleted']} rows older than {days}d"
|