feat: live-test scorecard and frozen strategy settings

Pre-registers the real-money test that starts 2026-09-24, the first session without commission drag. PASS needs at least 80 round trips with a positive mean R and positive P&L. FAIL is any session the equity floor halts. Changing any strategy setting in live_test_freeze.json restarts the count, and the scorecard reports that drift. The settings were frozen, not tuned, because a study of 239 real fills re-simulated under the current exit found nothing worth changing: - Capping how far above the trigger an entry may fill lost R at every cap from 0.25% to 10%, with either skip or resting-limit handling. Fills more than 5% above the trigger were the best bucket (+0.66R). - No break-time factor (spread, price, VWAP distance, stop width, L1 bid/ask size, premarket range) kept its sign across both date halves and both accounts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 10:53:39 -05:00
1 parent 42ca523053
commit e2e9687c45
@@ -0,0 +1,149 @@
+"""Live-test scorecard: is the frozen strategy making money on the real account?
+
+The test was pre-registered on 2026-09-25, after the commission leak was closed
+(first clean session 2026-09-24):
+
+  PASS  after >= 80 round trips since the freeze, mean R > 0 AND realized P&L > 0.
+  FAIL  the equity-floor kill switch halts a session.
+  Otherwise the test is still running. Any change to a strategy setting in
+  research/live_test_freeze.json restarts the count, so this report flags drift.
+
+Why 80: per-trade R ranges roughly -1 to +10 with a stdev of ~1.4, so under ~80
+trades a positive mean cannot be told apart from zero. Why no filters were
+changed at the freeze: the 2026-09-25 study (239 real fills, re-simulated under
+the current exit) found that capping the entry chase lost R at every cap, and
+that no break-time factor, including the L1/L2 book, held its sign across both
+halves of the sample.
+
+Run against the LIVE (tradier) database:
+    DATABASE_URL=<tradier dsn> PG_SKIP_SCHEMA=1 python -m research.live_scorecard
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import random
+from pathlib import Path
+
+from research.multi_schema import open_research_db
+
+FREEZE_FILE = Path(__file__).with_name("live_test_freeze.json")
+TARGET_TRADES = 80
+# safety knobs: changing these does not change the strategy under test
+SAFETY_KEYS = {"TRADING_EQUITY_FLOOR", "TRADING_DEFAULT_EQUITY"}
+
+
+def load_freeze(path: Path = FREEZE_FILE) -> dict:
+    return json.loads(path.read_text())
+
+
+def config_drift(frozen: dict, current: dict) -> dict:
+    """{key: (frozen, current)} for strategy settings that differ from the freeze."""
+    keys = set(frozen) | {k for k in current if k.startswith(("TRADING_", "TRIGGER_"))}
+    return {k: (frozen.get(k), current.get(k)) for k in sorted(keys - SAFETY_KEYS)
+            if frozen.get(k) != current.get(k)}
+
+
+def bootstrap_mean_ci(xs: list[float], n: int = 4000, seed: int = 7) -> tuple[float, float]:
+    if not xs:
+        return (float("nan"), float("nan"))
+    rng = random.Random(seed)
+    means = sorted(sum(rng.choices(xs, k=len(xs))) / len(xs) for _ in range(n))
+    return means[int(0.05 * n)], means[int(0.95 * n) - 1]
+
+
+def verdict(r_values: list[float], pnl: float, floor_halts: int,
+            target: int = TARGET_TRADES) -> str:
+    if floor_halts:
+        return "FAIL: the equity floor halted trading"
+    if len(r_values) < target:
+        return f"RUNNING: {len(r_values)}/{target} trades"
+    mean_r = sum(r_values) / len(r_values)
+    if mean_r > 0 and pnl > 0:
+        return "PASS: positive per-trade R and positive P&L"
+    return "FAIL: not profitable after the full sample"
+
+
+def load_trades(con, since: str) -> list[dict]:
+    """Closed round trips since ``since`` with R measured against the stop the
+    order was submitted with (position_closed carries the final, trailed stop)."""
+    closes = con.execute(
+        "SELECT timestamp, payload_json FROM events WHERE event_type='position_closed' "
+        "AND timestamp >= ? ORDER BY timestamp", [since]).fetchall()
+    out = []
+    for ts, payload in closes:
+        p = json.loads(payload or "{}")
+        sym = p.get("symbol")
+        row = con.execute(
+            "SELECT payload_json FROM events WHERE event_type='order_approval_requested' "
+            "AND payload_json LIKE ? AND timestamp::date = ?::date AND timestamp < ? "
+            "ORDER BY timestamp DESC LIMIT 1", [f'%"symbol": "{sym}"%', ts, ts]).fetchone()
+        stop0 = None
+        if row:
+            stop0 = (json.loads(row[0]).get("execution_request") or {}).get("stop_loss_price")
+        entry, exit_p = p.get("entry_price"), p.get("exit_price")
+        r = None
+        if stop0 and entry and exit_p is not None and float(entry) > float(stop0):
+            r = (float(exit_p) - float(entry)) / (float(entry) - float(stop0))
+        out.append({"ts": ts, "symbol": sym, "pnl": float(p.get("realized_pnl") or 0.0),
+                    "r": r, "reason": p.get("exit_reason")})
+    return out
+
+
+def equity_path(con, since: str) -> tuple[float | None, float | None]:
+    rows = con.execute(
+        "SELECT payload_json FROM events WHERE event_type='account_summary_updated' "
+        "AND timestamp >= ? ORDER BY timestamp", [since]).fetchall()
+    eq = [json.loads(r[0]).get("total_equity") for r in rows]
+    eq = [float(e) for e in eq if e]
+    return (eq[0], eq[-1]) if eq else (None, None)
+
+
+def floor_halts(con, since: str) -> int:
+    """Sessions the kill switch halted during market hours (the midnight re-check
+    that fires while equity is still below the floor is not a new halt)."""
+    rows = con.execute(
+        "SELECT timestamp, message FROM events WHERE event_type='risk_rule_triggered' "
+        "AND timestamp >= ? AND message LIKE 'EQUITY FLOOR%'", [since]).fetchall()
+    return len({ts.date() for ts, _ in rows if 9 <= ts.hour < 16})
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="live-test scorecard since the freeze")
+    ap.add_argument("--since", help="override the freeze date (YYYY-MM-DD)")
+    args = ap.parse_args()
+    freeze = load_freeze()
+    since = args.since or freeze["frozen_on"]
+    con = open_research_db("market")
+
+    trades = load_trades(con, since)
+    rs = [t["r"] for t in trades if t["r"] is not None]
+    pnl = sum(t["pnl"] for t in trades)
+    halts = floor_halts(con, since)
+    eq0, eq1 = equity_path(con, since)
+    current = dict(con.execute("SELECT key, value FROM config_overrides").fetchall())
+    drift = config_drift(freeze["settings"], current)
+
+    print(f"Live test since {since}  ({len(trades)} round trips)\n")
+    for t in trades:
+        r = f"{t['r']:+.2f}R" if t["r"] is not None else "   n/a"
+        print(f"  {str(t['ts'])[:16]}  {t['symbol']:6s} {r}  ${t['pnl']:+8.2f}  {t['reason']}")
+    if rs:
+        lo, hi = bootstrap_mean_ci(rs)
+        wins = sum(1 for r in rs if r > 0)
+        print(f"\nmean R {sum(rs) / len(rs):+.3f}  (90% CI {lo:+.3f} .. {hi:+.3f})"
+              f"  win rate {wins}/{len(rs)}")
+    print(f"realized P&L ${pnl:+.2f}")
+    if eq0 is not None:
+        print(f"equity ${eq0:.2f} -> ${eq1:.2f}  (difference from realized P&L = fees + deposits/refunds)")
+    print(f"floor halts: {halts}")
+    if drift:
+        print("\nSETTINGS CHANGED SINCE THE FREEZE (the count restarts from the change):")
+        for k, (a, b) in drift.items():
+            print(f"  {k}: {a!r} -> {b!r}")
+    print(f"\n{verdict(rs, pnl, halts)}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,68 @@
+{
+  "frozen_on": "2026-09-24",
+  "source": "config_overrides on the tradier live DB (port 5434), snapshotted 2026-09-25",
+  "note": "Strategy knobs only. Safety knobs (equity floor, default equity) may change without resetting the test.",
+  "settings": {
+    "NEWS_CATALYST_SCORE_ENABLED": "1",
+    "NEWS_DILUTION_VETO_ENABLED": "1",
+    "NEWS_RANK_BOOST": "1.0",
+    "TRADING_AUTO_APPROVE": "1",
+    "TRADING_BACKOUT_COOLDOWN_SECONDS": "180",
+    "TRADING_BLOCK_REENTRY_AFTER_EXIT": "1",
+    "TRADING_DEFAULT_EQUITY": "200",
+    "TRADING_ENTRY_CONFIRM_BY_MINUTE": "0",
+    "TRADING_ENTRY_FILL_MODEL": "marketable",
+    "TRADING_ENTRY_INVALIDATE_PCT": "0.015",
+    "TRADING_ENTRY_ORDER_TYPE": "limit",
+    "TRADING_ENTRY_TIMEOUT_BARS": "2",
+    "TRADING_EOD_FLATTEN_ENABLED": "1",
+    "TRADING_EOD_FLATTEN_TIME": "15:55",
+    "TRADING_EQUITY_FLOOR": "215",
+    "TRADING_EXECUTION_ENABLED": "1",
+    "TRADING_EXIT_BREAKEVEN_PCT": "0.0",
+    "TRADING_EXIT_BREAKEVEN_R": "1.0",
+    "TRADING_EXIT_MANAGE_INTERVAL_SECONDS": "4",
+    "TRADING_EXIT_PROFIT_TIERS": "",
+    "TRADING_EXIT_THREAD_ENABLED": "1",
+    "TRADING_EXIT_THREAD_INTERVAL_SECONDS": "1.0",
+    "TRADING_EXIT_TRAIL_AFTER_R": "1.0",
+    "TRADING_EXIT_TRAIL_MODE": "pct",
+    "TRADING_EXIT_TRAIL_PCT": "0.12",
+    "TRADING_EXIT_TRAIL_R_STEP": "0.0",
+    "TRADING_LIQUIDITY_MAX_VOLUME_PCT": "0.025",
+    "TRADING_MAX_BACKOUTS_PER_SYMBOL": "3",
+    "TRADING_MAX_CONCURRENT_POSITIONS": "10",
+    "TRADING_MAX_DAILY_LOSS_PCT": "0.05",
+    "TRADING_MAX_FRESH_ENTRIES_PER_DAY": "5",
+    "TRADING_MAX_GROSS_NOTIONAL_PCT": "0.60",
+    "TRADING_MAX_ORDERS_PER_TICK": "1",
+    "TRADING_MAX_POSITION_PCT": "0.16",
+    "TRADING_MAX_RISK_DOLLARS": "1000",
+    "TRADING_MIN_DAY_VOLUME": "300000",
+    "TRADING_MIN_ENTRY_GAP_PCT": "0.10",
+    "TRADING_MIN_QUALITY_SCORE": "0.20",
+    "TRADING_PRICE_MAX": "20.0",
+    "TRADING_PRICE_MIN": "2.0",
+    "TRADING_REENTRY_MIN_LOSS_PCT": "0.01",
+    "TRADING_REGIME_GATE_ENABLED": "0",
+    "TRADING_REGIME_GATE_P_MIN": "0.5",
+    "TRADING_REQUIRE_ABOVE_VWAP": "1",
+    "TRADING_REWARD_MULTIPLE": "10.0",
+    "TRADING_RISK_PER_TRADE_PCT": "0.015",
+    "TRADING_SESSION_ONLY": "1",
+    "TRADING_TRIGGER_SLIP_PCT": "0.004",
+    "TRADING_UNIFIED_ENTRY": "1",
+    "TRIGGER_ENABLED": "1",
+    "TRIGGER_GAP_MAX": "35",
+    "TRIGGER_GAP_MIN": "10.0",
+    "TRIGGER_GAP_SQUEEZE_MIN": "150",
+    "TRIGGER_GAP_SQUEEZE_RVOL": "5.0",
+    "TRIGGER_INTERVAL_SECONDS": "2",
+    "TRIGGER_MAX_ARMED": "10",
+    "TRIGGER_MIN_DOLLAR_VOL": "8000",
+    "TRIGGER_PRICE_MAX_AGE": "12",
+    "TRIGGER_RVOL_MIN": "2.0",
+    "WATCHER_PRICE_MAX": "20",
+    "WATCHER_PRICE_MIN": "2"
+  }
+}
@@ -0,0 +1,30 @@
+"""Live-test scorecard: pure verdict / drift logic."""
+
+from research.live_scorecard import config_drift, load_freeze, verdict
+
+
+def test_running_until_target():
+    assert verdict([0.5] * 10, 5.0, 0, target=80).startswith("RUNNING: 10/80")
+
+
+def test_floor_halt_fails_immediately():
+    assert verdict([2.0] * 100, 50.0, 1).startswith("FAIL: the equity floor")
+
+
+def test_pass_needs_positive_r_and_pnl():
+    assert verdict([0.2] * 80, 10.0, 0).startswith("PASS")
+    assert verdict([0.2] * 80, -1.0, 0).startswith("FAIL")
+    assert verdict([-0.1] * 80, 5.0, 0).startswith("FAIL")
+
+
+def test_drift_ignores_safety_knobs_and_catches_new_strategy_keys():
+    frozen = {"TRADING_EXIT_TRAIL_PCT": "0.12", "TRADING_EQUITY_FLOOR": "215"}
+    current = {"TRADING_EXIT_TRAIL_PCT": "0.12", "TRADING_EQUITY_FLOOR": "190",
+               "TRIGGER_GAP_MAX": "40", "OLLAMA_MODEL": "x"}
+    assert config_drift(frozen, current) == {"TRIGGER_GAP_MAX": (None, "40")}
+
+
+def test_freeze_file_is_valid():
+    f = load_freeze()
+    assert f["frozen_on"] == "2026-09-24"
+    assert f["settings"]["TRADING_EXIT_TRAIL_PCT"] == "0.12"