diff --git a/deploy/backup_live_db.py b/deploy/backup_live_db.py new file mode 100644 index 0000000..ee1bf52 --- /dev/null +++ b/deploy/backup_live_db.py @@ -0,0 +1,71 @@ +"""Nightly off-box backup of the LIVE (Tradier) Postgres, from the owner's workstation. + +The live stack on TrueNAS has no db-backup service; the paper stack backs up only its own database. This keeps the +live test's full record (events, trades, settings) somewhere other than the TrueNAS pool. + + - pg_dump -Fc (compressed, restorable with pg_restore) of the live database (DATABASE_URL host, port 5434) + - verified: pg_restore --list must read it and it must contain the events and config_overrides tables + - keeps the newest KEEP dumps (default 7) in LIVE_BACKUP_DIR (default C:\\Users\\Admin\\momentum-backups) + - writes /status.json for research.daily_report to alert on a missed or failed backup + +usage: python deploy/backup_live_db.py (Task Scheduler "Momentum\\LiveDbBackup", nightly) +restore: pg_restore -d --no-owner +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse +from zoneinfo import ZoneInfo + +ROOT = Path(__file__).resolve().parent.parent +PG_BIN = Path(os.environ.get("PG_BIN", r"C:\Users\Admin\pgsql18\bin")) +OUT = Path(os.environ.get("LIVE_BACKUP_DIR", r"C:\Users\Admin\momentum-backups")) +KEEP = int(os.environ.get("LIVE_BACKUP_KEEP", "7")) +PORT = int(os.environ.get("LIVE_DB_PORT", "5434")) +ET = ZoneInfo("America/New_York") +MUST_HAVE = ("TABLE DATA public events", "TABLE DATA public config_overrides") + + +def main() -> int: + from dotenv import dotenv_values + u = urlparse(dotenv_values(ROOT / ".env")["DATABASE_URL"]) + OUT.mkdir(parents=True, exist_ok=True) + stamp = datetime.now(ET).strftime("%Y-%m-%d_%H%M") + dump = OUT / f"live_{stamp}.dump" + env = dict(os.environ, PGPASSWORD=u.password or "") + status = dict(at=datetime.now(ET).isoformat(timespec="seconds"), file=str(dump), ok=False) + t0 = time.time() + try: + subprocess.run([str(PG_BIN / "pg_dump.exe"), "-h", u.hostname, "-p", str(PORT), "-U", u.username, + "-d", u.path[1:], "-Fc", "-Z", "6", "-f", str(dump)], + env=env, check=True, capture_output=True, text=True, timeout=3600) + listing = subprocess.run([str(PG_BIN / "pg_restore.exe"), "--list", str(dump)], + check=True, capture_output=True, text=True, timeout=600).stdout + missing = [m for m in MUST_HAVE if m not in listing] + if missing: + raise RuntimeError(f"dump is missing {missing}") + status.update(ok=True, size_mb=round(dump.stat().st_size / 1e6, 1), seconds=round(time.time() - t0), + entries=listing.count("\n")) + except subprocess.CalledProcessError as e: + status["error"] = (e.stderr or str(e))[-400:] + except Exception as e: # noqa: BLE001 + status["error"] = str(e)[-400:] + if not status["ok"] and dump.exists(): + dump.unlink() # never keep a broken dump around as if it were good + dumps = sorted(OUT.glob("live_*.dump")) + for old in dumps[:-KEEP]: + old.unlink() + status["kept"] = [p.name for p in sorted(OUT.glob("live_*.dump"))] + (OUT / "status.json").write_text(json.dumps(status, indent=1), encoding="utf-8") + print(json.dumps(status, indent=1)) + return 0 if status["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/research/daily_report.py b/research/daily_report.py index 315fa21..2c7d862 100644 --- a/research/daily_report.py +++ b/research/daily_report.py @@ -199,6 +199,18 @@ def health_section(paper, live, today, alerts) -> list[str]: except Exception as e: # noqa: BLE001 lines.append(f"- {name}: ERROR {str(e)[:120]}") alerts.append(f"Could not read the {name} database: {str(e)[:80]}") + bk = Path(os.environ.get("LIVE_BACKUP_DIR", r"C:\Users\Admin\momentum-backups")) / "status.json" + if bk.exists(): + b = json.loads(bk.read_text(encoding="utf-8")) + age_h = (datetime.now(ET) - datetime.fromisoformat(b["at"])).total_seconds() / 3600 + lines.append(f"- live DB backup: {'ok' if b.get('ok') else 'FAILED'} {b['at'][:16]} " + f"({b.get('size_mb', '?')} MB, {len(b.get('kept', []))} kept)") + if not b.get("ok"): + alerts.append(f"Last live-DB backup FAILED: {str(b.get('error'))[:120]}") + elif age_h > 36: + alerts.append(f"Last good live-DB backup is {age_h:.0f} h old (nightly task not running?).") + else: + alerts.append("No live-DB backup status found (the Momentum LiveDbBackup task has never run).") pw = ROOT / "data" / "prwire" / f"{today}.jsonl" if pw.exists(): rows = [json.loads(x) for x in pw.read_text(encoding="utf-8").splitlines() if x.strip()] @@ -213,7 +225,8 @@ def health_section(paper, live, today, alerts) -> list[str]: def main(argv=None) -> str: argv = sys.argv[1:] if argv is None else argv - today = argv[0] if argv else datetime.now(ET).strftime("%Y-%m-%d") + dates = [a for a in argv if not a.startswith("--")] + today = dates[0] if dates else datetime.now(ET).strftime("%Y-%m-%d") alerts: list[str] = [] body: list[str] = [] live = paper = None @@ -237,8 +250,20 @@ def main(argv=None) -> str: OUT.mkdir(parents=True, exist_ok=True) (OUT / f"daily_{today}.md").write_text(md, encoding="utf-8") print(md) + if alerts and "--no-telegram" not in argv: + notify(today, alerts, body) return md +def notify(today: str, alerts: list[str], body: list[str]) -> bool: + """Phone push via the stack's Telegram bot (credentials in .env). Sent only when there are alerts.""" + from runtime.notifier import send_telegram + env = dotenv_values(ROOT / ".env") + status = next((b for b in body if b.startswith("- **")), "").replace("*", "").lstrip("- ") + text = (f"Momentum {today}: {len(alerts)} alert(s)\n" + "\n".join(f"- {a}" for a in alerts) + + (f"\nLive test: {status}" if status else "") + f"\nReport: data/reports/daily_{today}.md") + return send_telegram(text[:3900], token=env.get("TELEGRAM_BOT_TOKEN"), chat_id=env.get("TELEGRAM_CHAT_ID")) + + if __name__ == "__main__": main()