From 6c6d0cd799288950345ef0dd23820cd52504119a Mon Sep 17 00:00:00 2001 From: momentum dev Date: Fri, 24 Jul 2026 22:25:14 -0500 Subject: [PATCH] =?UTF-8?q?feat:=20weekly=20execution=20tallies=20?= =?UTF-8?q?=E2=80=94=20midday-entry=20net=20+=20market=5Fexit=20post-run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read-only nightly-scheduler job (Friday-gated): buckets realized P&L by entry time (open/late-morning/midday) and, for each market_exit close, computes the post-exit run-up capturable under a breakeven stop (0 if price undercuts entry first; intrabar ties count as stopped). Emits a weekly_tally event + Telegram summary. Evidence accumulator for the dead no_new_entries_after cutoff and the weakness-exit winner-clipping question — no live trading path touched. Co-Authored-By: Claude Fable 5 --- deploy/docker/truenas-compose.flat.yaml | 2 +- deploy/docker/truenas-compose.yml | 3 + research/weekly_tallies.py | 206 +++++++++++++++++++++ tests/unit/research/test_weekly_tallies.py | 64 +++++++ 4 files changed, 274 insertions(+), 1 deletion(-) create mode 100644 research/weekly_tallies.py create mode 100644 tests/unit/research/test_weekly_tallies.py diff --git a/deploy/docker/truenas-compose.flat.yaml b/deploy/docker/truenas-compose.flat.yaml index 218650f..f296832 100644 --- a/deploy/docker/truenas-compose.flat.yaml +++ b/deploy/docker/truenas-compose.flat.yaml @@ -569,7 +569,7 @@ services: command: - /bin/sh - -c - - "echo \"[scheduler] up; fires 20:00 ET Mon-Fri\"\nwhile true; do\n if [ \"$$(date +%H:%M)\" = \"20:00\" ] && [ \"$$(date +%u)\" -le 5 ]; then\n python -m research.premarket_eval || echo \"[scheduler] eval FAILED\"\n python -m research.ingestion.market_context || echo \"[scheduler] market_context FAILED\"\n python -m research.eod_reconcile || echo \"[scheduler] reconcile FAILED\"\n python eod_replay.py || echo \"[scheduler] eod_replay FAILED\"\n python -m research.labeler build || echo \"[scheduler] labeler build FAILED\"\n python -m research.labeler build-tod1 || echo \"[scheduler] labeler build-tod1 FAILED\"\n python nightly_tune.py || echo \"[scheduler] nightly_tune FAILED\"\n python -m research.regime build-features || echo \"[scheduler] regime build FAILED\"\n python -m research.regime score --model threshold_v1 || echo \"[scheduler] regime threshold FAILED\"\n python -m research.regime score --model meta_v1 || echo \"[scheduler] regime meta FAILED\"\n python -m research.experiments run || echo \"[scheduler] experiments run FAILED\"\n python -m research.experiments adjudicate || echo \"[scheduler] experiments adjudicate FAILED\"\n sleep 120\n fi\n sleep 20\ndone\n" + - "echo \"[scheduler] up; fires 20:00 ET Mon-Fri\"\nwhile true; do\n if [ \"$$(date +%H:%M)\" = \"20:00\" ] && [ \"$$(date +%u)\" -le 5 ]; then\n python -m research.premarket_eval || echo \"[scheduler] eval FAILED\"\n python -m research.ingestion.market_context || echo \"[scheduler] market_context FAILED\"\n python -m research.eod_reconcile || echo \"[scheduler] reconcile FAILED\"\n python eod_replay.py || echo \"[scheduler] eod_replay FAILED\"\n python -m research.labeler build || echo \"[scheduler] labeler build FAILED\"\n python -m research.labeler build-tod1 || echo \"[scheduler] labeler build-tod1 FAILED\"\n python nightly_tune.py || echo \"[scheduler] nightly_tune FAILED\"\n python -m research.regime build-features || echo \"[scheduler] regime build FAILED\"\n python -m research.regime score --model threshold_v1 || echo \"[scheduler] regime threshold FAILED\"\n python -m research.regime score --model meta_v1 || echo \"[scheduler] regime meta FAILED\"\n python -m research.experiments run || echo \"[scheduler] experiments run FAILED\"\n python -m research.experiments adjudicate || echo \"[scheduler] experiments adjudicate FAILED\"\n python -m research.weekly_tallies || echo \"[scheduler] weekly_tallies FAILED\"\n sleep 120\n fi\n sleep 20\ndone\n" environment: ALPACA_API_KEY: ALPACA_BACKFILL_FEED: sip diff --git a/deploy/docker/truenas-compose.yml b/deploy/docker/truenas-compose.yml index 8394446..9e5d5d2 100644 --- a/deploy/docker/truenas-compose.yml +++ b/deploy/docker/truenas-compose.yml @@ -233,6 +233,9 @@ services: # newly-logged fires, then adjudicate against pre-registered bars. python -m research.experiments run || echo "[scheduler] experiments run FAILED" python -m research.experiments adjudicate || echo "[scheduler] experiments adjudicate FAILED" + # Weekly execution tallies (read-only): midday-entry net + market_exit + # post-run. Script self-gates to Friday; other nights it no-ops. + python -m research.weekly_tallies || echo "[scheduler] weekly_tallies FAILED" sleep 120 fi sleep 20 diff --git a/research/weekly_tallies.py b/research/weekly_tallies.py new file mode 100644 index 0000000..e79aae2 --- /dev/null +++ b/research/weekly_tallies.py @@ -0,0 +1,206 @@ +"""Weekly execution tallies — two standing stats the 2026-07-24 trade review flagged. + +READ-ONLY: queries the event store + minute bars, emits one summary event, optional +Telegram. Never touches live trading state. + + MIDDAY NET — realized P&L bucketed by ENTRY time (open ≤10:30 ET, late-morning + ≤12:00, midday after). Accumulating evidence for wiring the dead + no_new_entries_after cutoff: if midday stays net-negative for a few + weeks, wire it; until then just count. + EXIT POST-RUN — for each market_exit (weakness exit) close: how much run-up was + CAPTURABLE after the exit, i.e. the high reached before price first + dipped below entry (a breakeven stop survives until then; a dip below + entry first means the "money left" was never reachable → 0). + +Entry time = first order_submitted event for (day, symbol); re-entries on the same +symbol/day share that first submit time (close enough for a bucket tally). + +CLI: python -m research.weekly_tallies [--days 7] [--force] +Runs only on Friday (ET) unless --force — the nightly scheduler chain calls it daily. +""" + +from __future__ import annotations + +import argparse +import json +import os +from datetime import datetime, time, timedelta, timezone +from zoneinfo import ZoneInfo + +ET = ZoneInfo("America/New_York") + +OPEN_CUTOFF = time(10, 30) +MIDDAY_CUTOFF = time(12, 0) + + +def _bucket(entry_et: time) -> str: + if entry_et <= OPEN_CUTOFF: + return "open" + if entry_et <= MIDDAY_CUTOFF: + return "late_morning" + return "midday" + + +def capturable_left_pct(bars: list[dict], exit_ts_et: datetime, + entry_price: float, exit_price: float) -> float: + """Run-up (% of exit price) to the highest high after exit BEFORE the first bar + whose low undercuts entry price. ``bars`` are chronological dicts with tz-aware + ET ``ts``, ``high``, ``low``. Dip below entry before any new high -> 0.0. + Intrabar order is unknowable from minute bars, so a bar that both undercuts + entry and makes a new high counts as stopped first (pessimistic on purpose — + 'capturable' must not depend on winning an intrabar coin flip).""" + best = exit_price + for b in bars: + if b["ts"] <= exit_ts_et: + continue + if b["low"] < entry_price: + break + if b["high"] > best: + best = b["high"] + return max(0.0, (best - exit_price) / exit_price * 100.0) + + +def compute_tallies(closes: list[dict], bars_for) -> dict: + """Pure core. ``closes``: dicts with symbol, day (date), entry_ts_et / + exit_ts_et (tz-aware datetimes), entry_price, exit_price, exit_reason, + realized_pnl. ``bars_for(symbol, day)`` -> chronological bar dicts + (tz-aware ET ts, high, low). Returns the tally dict.""" + buckets: dict[str, dict] = {k: {"n": 0, "net_pnl": 0.0} + for k in ("open", "late_morning", "midday")} + post_runs: list[dict] = [] + for c in closes: + b = _bucket(c["entry_ts_et"].timetz().replace(tzinfo=None)) + buckets[b]["n"] += 1 + buckets[b]["net_pnl"] = round(buckets[b]["net_pnl"] + c["realized_pnl"], 2) + if c["exit_reason"] == "market_exit": + left = capturable_left_pct(bars_for(c["symbol"], c["day"]), + c["exit_ts_et"], c["entry_price"], c["exit_price"]) + post_runs.append({"symbol": c["symbol"], "day": str(c["day"]), + "pnl": c["realized_pnl"], "left_pct": round(left, 1)}) + clipped = [p for p in post_runs if p["left_pct"] >= 5.0] + return { + "n_trades": len(closes), + "entry_buckets": buckets, + "market_exit_post_runs": post_runs, + "n_market_exits": len(post_runs), + "n_clipped_ge_5pct": len(clipped), + "median_left_pct": (sorted(p["left_pct"] for p in post_runs)[len(post_runs) // 2] + if post_runs else 0.0), + } + + +def _load_closes(con, since_et: datetime) -> list[dict]: + """events.timestamp is stored ET-naive (matches the live loop's wall clock).""" + rows = con.execute( + "SELECT timestamp, payload_json FROM events " + "WHERE event_type = 'position_closed' AND timestamp >= ? ORDER BY timestamp", + [since_et.replace(tzinfo=None)], + ).fetchall() + submits = con.execute( + "SELECT timestamp, payload_json FROM events " + "WHERE event_type = 'order_submitted' AND timestamp >= ? ORDER BY timestamp", + [since_et.replace(tzinfo=None)], + ).fetchall() + first_submit: dict[tuple, datetime] = {} + for ts, pj in submits: + p = pj if isinstance(pj, dict) else json.loads(pj or "{}") + key = (ts.date(), p.get("symbol")) + first_submit.setdefault(key, ts) + closes = [] + for ts, pj in rows: + p = pj if isinstance(pj, dict) else json.loads(pj or "{}") + sym = p.get("symbol") + if not sym or p.get("realized_pnl") is None: + continue + sub = first_submit.get((ts.date(), sym)) or ts + closes.append({ + "symbol": sym, "day": ts.date(), + "entry_ts_et": sub.replace(tzinfo=ET), + "exit_ts_et": ts.replace(tzinfo=ET), + "entry_price": float(p.get("entry_price") or 0), + "exit_price": float(p.get("exit_price") or 0), + "exit_reason": p.get("exit_reason") or "", + "realized_pnl": float(p["realized_pnl"]), + }) + return closes + + +def _bars_for(con): + """minute_bars.timestamp is stored UTC-naive — convert to tz-aware ET.""" + def fetch(symbol: str, day) -> list[dict]: + rows = con.execute( + "SELECT timestamp, high, low FROM minute_bars " + "WHERE symbol = ? AND session_date = ? AND is_regular_hours " + "ORDER BY timestamp", + [symbol, day], + ).fetchall() + return [{"ts": t.replace(tzinfo=timezone.utc).astimezone(ET), + "high": float(h), "low": float(l)} for t, h, l in rows] + return fetch + + +def _summary_text(t: dict) -> str: + eb = t["entry_buckets"] + lines = [f"Weekly tallies ({t['n_trades']} trades):"] + for k in ("open", "late_morning", "midday"): + lines.append(f" entry {k}: {eb[k]['n']} trades net ${eb[k]['net_pnl']:+.0f}") + lines.append(f" market_exit post-run: {t['n_market_exits']} exits, " + f"{t['n_clipped_ge_5pct']} clipped >=5% capturable, " + f"median left {t['median_left_pct']:.1f}%") + return "\n".join(lines) + + +def run(con, store, *, days: int = 7, now_et: datetime | None = None, notify=None) -> dict: + from storage.event_schema import EventMode, RiskRuleTriggeredEvent + + now_et = now_et or datetime.now(ET) + since = now_et - timedelta(days=days) + closes = _load_closes(con, since) + tallies = compute_tallies(closes, _bars_for(con)) + msg = _summary_text(tallies) + store.emit(RiskRuleTriggeredEvent( + timestamp=datetime.now(), mode=EventMode.PAPER, + correlation_id="weekly_tallies", message=msg.splitlines()[0], + rule_type="weekly_tally", rule_value=float(tallies["n_trades"]), + current_state=tallies, action_taken="reported", + )) + if notify is not None: + try: + notify(f"\U0001F4CA {msg}") + except Exception: # noqa: BLE001 — an alert failure must not break the job + pass + print(f"[weekly_tallies] {msg}", flush=True) + return tallies + + +def main() -> int: + from dotenv import load_dotenv + load_dotenv() + from runtime.env_overrides import load as _load_overrides + _load_overrides() + + parser = argparse.ArgumentParser() + parser.add_argument("--days", type=int, default=7) + parser.add_argument("--force", action="store_true", + help="run even when it is not Friday ET") + args = parser.parse_args() + + now_et = datetime.now(ET) + if now_et.weekday() != 4 and not args.force: + print("[weekly_tallies] not Friday — skipping (use --force to override)", flush=True) + return 0 + + from storage.db import get_connection + from storage.event_store import EventStore + con = get_connection(os.environ.get("WATCHER_EVENT_DB_PATH", "momentum")) + store = EventStore(os.environ.get("WATCHER_EVENT_DB_PATH", "momentum")) + try: + from runtime.notifier import send_telegram + except Exception: # noqa: BLE001 + send_telegram = None + run(con, store, days=args.days, now_et=now_et, notify=send_telegram) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/research/test_weekly_tallies.py b/tests/unit/research/test_weekly_tallies.py new file mode 100644 index 0000000..40d6048 --- /dev/null +++ b/tests/unit/research/test_weekly_tallies.py @@ -0,0 +1,64 @@ +"""Pure-core tests for research.weekly_tallies (no DB, no broker).""" + +from datetime import date, datetime +from zoneinfo import ZoneInfo + +from research.weekly_tallies import capturable_left_pct, compute_tallies + +ET = ZoneInfo("America/New_York") +DAY = date(2026, 7, 24) + + +def _ts(h, m): + return datetime(2026, 7, 24, h, m, tzinfo=ET) + + +def _bar(h, m, high, low): + return {"ts": _ts(h, m), "high": high, "low": low} + + +def _close(sym, entry_hm, exit_hm, entry_px, exit_px, reason, pnl): + return { + "symbol": sym, "day": DAY, + "entry_ts_et": _ts(*entry_hm), "exit_ts_et": _ts(*exit_hm), + "entry_price": entry_px, "exit_price": exit_px, + "exit_reason": reason, "realized_pnl": pnl, + } + + +def test_capturable_left_clean_runup(): + # ADVB-like: after exit at 7.0 the stock runs to 9.4 without undercutting entry + bars = [_bar(9, 45, 7.2, 6.9), _bar(10, 0, 8.0, 7.1), _bar(15, 44, 9.4, 8.8)] + left = capturable_left_pct(bars, _ts(9, 41), entry_price=6.73, exit_price=7.0) + assert round(left, 1) == round((9.4 - 7.0) / 7.0 * 100, 1) + + +def test_capturable_left_zero_when_dip_below_entry_first(): + # WLDS-like: dips below entry before the big run -> BE stop dies, nothing capturable + bars = [_bar(10, 10, 2.8, 2.55), _bar(15, 36, 4.35, 2.6)] + left = capturable_left_pct(bars, _ts(10, 2), entry_price=2.73, exit_price=2.78) + assert left == 0.0 + + +def test_capturable_ignores_bars_before_exit(): + bars = [_bar(9, 30, 99.0, 1.0), _bar(11, 0, 3.0, 2.9)] + left = capturable_left_pct(bars, _ts(10, 0), entry_price=2.0, exit_price=2.5) + assert round(left, 1) == 20.0 + + +def test_compute_tallies_buckets_and_post_runs(): + closes = [ + _close("AAA", (9, 40), (9, 50), 5.0, 5.5, "stop_loss", 200.0), + _close("BBB", (11, 0), (11, 30), 3.0, 3.1, "stop_loss", 50.0), + _close("CCC", (12, 47), (15, 40), 15.76, 15.36, "market_exit", -370.0), + ] + # recovery above entry without undercutting it -> capturable run-up + bars = {"CCC": [_bar(15, 50, 16.5, 15.8)]} + t = compute_tallies(closes, lambda sym, day: bars.get(sym, [])) + assert t["n_trades"] == 3 + assert t["entry_buckets"]["open"] == {"n": 1, "net_pnl": 200.0} + assert t["entry_buckets"]["late_morning"] == {"n": 1, "net_pnl": 50.0} + assert t["entry_buckets"]["midday"] == {"n": 1, "net_pnl": -370.0} + assert t["n_market_exits"] == 1 + run = t["market_exit_post_runs"][0] + assert run["symbol"] == "CCC" and run["left_pct"] > 0