diff --git a/research/live_scorecard.py b/research/live_scorecard.py new file mode 100644 index 0000000..1341056 --- /dev/null +++ b/research/live_scorecard.py @@ -0,0 +1,149 @@ +"""Live-test scorecard: is the frozen strategy making money on the real account? + +The test was pre-registered on 2026-09-25, after the commission leak was closed +(first clean session 2026-09-24): + + PASS after >= 80 round trips since the freeze, mean R > 0 AND realized P&L > 0. + FAIL the equity-floor kill switch halts a session. + Otherwise the test is still running. Any change to a strategy setting in + research/live_test_freeze.json restarts the count, so this report flags drift. + +Why 80: per-trade R ranges roughly -1 to +10 with a stdev of ~1.4, so under ~80 +trades a positive mean cannot be told apart from zero. Why no filters were +changed at the freeze: the 2026-09-25 study (239 real fills, re-simulated under +the current exit) found that capping the entry chase lost R at every cap, and +that no break-time factor, including the L1/L2 book, held its sign across both +halves of the sample. + +Run against the LIVE (tradier) database: + DATABASE_URL= PG_SKIP_SCHEMA=1 python -m research.live_scorecard +""" + +from __future__ import annotations + +import argparse +import json +import random +from pathlib import Path + +from research.multi_schema import open_research_db + +FREEZE_FILE = Path(__file__).with_name("live_test_freeze.json") +TARGET_TRADES = 80 +# safety knobs: changing these does not change the strategy under test +SAFETY_KEYS = {"TRADING_EQUITY_FLOOR", "TRADING_DEFAULT_EQUITY"} + + +def load_freeze(path: Path = FREEZE_FILE) -> dict: + return json.loads(path.read_text()) + + +def config_drift(frozen: dict, current: dict) -> dict: + """{key: (frozen, current)} for strategy settings that differ from the freeze.""" + keys = set(frozen) | {k for k in current if k.startswith(("TRADING_", "TRIGGER_"))} + return {k: (frozen.get(k), current.get(k)) for k in sorted(keys - SAFETY_KEYS) + if frozen.get(k) != current.get(k)} + + +def bootstrap_mean_ci(xs: list[float], n: int = 4000, seed: int = 7) -> tuple[float, float]: + if not xs: + return (float("nan"), float("nan")) + rng = random.Random(seed) + means = sorted(sum(rng.choices(xs, k=len(xs))) / len(xs) for _ in range(n)) + return means[int(0.05 * n)], means[int(0.95 * n) - 1] + + +def verdict(r_values: list[float], pnl: float, floor_halts: int, + target: int = TARGET_TRADES) -> str: + if floor_halts: + return "FAIL: the equity floor halted trading" + if len(r_values) < target: + return f"RUNNING: {len(r_values)}/{target} trades" + mean_r = sum(r_values) / len(r_values) + if mean_r > 0 and pnl > 0: + return "PASS: positive per-trade R and positive P&L" + return "FAIL: not profitable after the full sample" + + +def load_trades(con, since: str) -> list[dict]: + """Closed round trips since ``since`` with R measured against the stop the + order was submitted with (position_closed carries the final, trailed stop).""" + closes = con.execute( + "SELECT timestamp, payload_json FROM events WHERE event_type='position_closed' " + "AND timestamp >= ? ORDER BY timestamp", [since]).fetchall() + out = [] + for ts, payload in closes: + p = json.loads(payload or "{}") + sym = p.get("symbol") + row = con.execute( + "SELECT payload_json FROM events WHERE event_type='order_approval_requested' " + "AND payload_json LIKE ? AND timestamp::date = ?::date AND timestamp < ? " + "ORDER BY timestamp DESC LIMIT 1", [f'%"symbol": "{sym}"%', ts, ts]).fetchone() + stop0 = None + if row: + stop0 = (json.loads(row[0]).get("execution_request") or {}).get("stop_loss_price") + entry, exit_p = p.get("entry_price"), p.get("exit_price") + r = None + if stop0 and entry and exit_p is not None and float(entry) > float(stop0): + r = (float(exit_p) - float(entry)) / (float(entry) - float(stop0)) + out.append({"ts": ts, "symbol": sym, "pnl": float(p.get("realized_pnl") or 0.0), + "r": r, "reason": p.get("exit_reason")}) + return out + + +def equity_path(con, since: str) -> tuple[float | None, float | None]: + rows = con.execute( + "SELECT payload_json FROM events WHERE event_type='account_summary_updated' " + "AND timestamp >= ? ORDER BY timestamp", [since]).fetchall() + eq = [json.loads(r[0]).get("total_equity") for r in rows] + eq = [float(e) for e in eq if e] + return (eq[0], eq[-1]) if eq else (None, None) + + +def floor_halts(con, since: str) -> int: + """Sessions the kill switch halted during market hours (the midnight re-check + that fires while equity is still below the floor is not a new halt).""" + rows = con.execute( + "SELECT timestamp, message FROM events WHERE event_type='risk_rule_triggered' " + "AND timestamp >= ? AND message LIKE 'EQUITY FLOOR%'", [since]).fetchall() + return len({ts.date() for ts, _ in rows if 9 <= ts.hour < 16}) + + +def main() -> None: + ap = argparse.ArgumentParser(description="live-test scorecard since the freeze") + ap.add_argument("--since", help="override the freeze date (YYYY-MM-DD)") + args = ap.parse_args() + freeze = load_freeze() + since = args.since or freeze["frozen_on"] + con = open_research_db("market") + + trades = load_trades(con, since) + rs = [t["r"] for t in trades if t["r"] is not None] + pnl = sum(t["pnl"] for t in trades) + halts = floor_halts(con, since) + eq0, eq1 = equity_path(con, since) + current = dict(con.execute("SELECT key, value FROM config_overrides").fetchall()) + drift = config_drift(freeze["settings"], current) + + print(f"Live test since {since} ({len(trades)} round trips)\n") + for t in trades: + r = f"{t['r']:+.2f}R" if t["r"] is not None else " n/a" + print(f" {str(t['ts'])[:16]} {t['symbol']:6s} {r} ${t['pnl']:+8.2f} {t['reason']}") + if rs: + lo, hi = bootstrap_mean_ci(rs) + wins = sum(1 for r in rs if r > 0) + print(f"\nmean R {sum(rs) / len(rs):+.3f} (90% CI {lo:+.3f} .. {hi:+.3f})" + f" win rate {wins}/{len(rs)}") + print(f"realized P&L ${pnl:+.2f}") + if eq0 is not None: + print(f"equity ${eq0:.2f} -> ${eq1:.2f} (difference from realized P&L = fees + deposits/refunds)") + print(f"floor halts: {halts}") + if drift: + print("\nSETTINGS CHANGED SINCE THE FREEZE (the count restarts from the change):") + for k, (a, b) in drift.items(): + print(f" {k}: {a!r} -> {b!r}") + print(f"\n{verdict(rs, pnl, halts)}") + + +if __name__ == "__main__": + main() diff --git a/research/live_test_freeze.json b/research/live_test_freeze.json new file mode 100644 index 0000000..f9d723a --- /dev/null +++ b/research/live_test_freeze.json @@ -0,0 +1,68 @@ +{ + "frozen_on": "2026-09-24", + "source": "config_overrides on the tradier live DB (port 5434), snapshotted 2026-09-25", + "note": "Strategy knobs only. Safety knobs (equity floor, default equity) may change without resetting the test.", + "settings": { + "NEWS_CATALYST_SCORE_ENABLED": "1", + "NEWS_DILUTION_VETO_ENABLED": "1", + "NEWS_RANK_BOOST": "1.0", + "TRADING_AUTO_APPROVE": "1", + "TRADING_BACKOUT_COOLDOWN_SECONDS": "180", + "TRADING_BLOCK_REENTRY_AFTER_EXIT": "1", + "TRADING_DEFAULT_EQUITY": "200", + "TRADING_ENTRY_CONFIRM_BY_MINUTE": "0", + "TRADING_ENTRY_FILL_MODEL": "marketable", + "TRADING_ENTRY_INVALIDATE_PCT": "0.015", + "TRADING_ENTRY_ORDER_TYPE": "limit", + "TRADING_ENTRY_TIMEOUT_BARS": "2", + "TRADING_EOD_FLATTEN_ENABLED": "1", + "TRADING_EOD_FLATTEN_TIME": "15:55", + "TRADING_EQUITY_FLOOR": "215", + "TRADING_EXECUTION_ENABLED": "1", + "TRADING_EXIT_BREAKEVEN_PCT": "0.0", + "TRADING_EXIT_BREAKEVEN_R": "1.0", + "TRADING_EXIT_MANAGE_INTERVAL_SECONDS": "4", + "TRADING_EXIT_PROFIT_TIERS": "", + "TRADING_EXIT_THREAD_ENABLED": "1", + "TRADING_EXIT_THREAD_INTERVAL_SECONDS": "1.0", + "TRADING_EXIT_TRAIL_AFTER_R": "1.0", + "TRADING_EXIT_TRAIL_MODE": "pct", + "TRADING_EXIT_TRAIL_PCT": "0.12", + "TRADING_EXIT_TRAIL_R_STEP": "0.0", + "TRADING_LIQUIDITY_MAX_VOLUME_PCT": "0.025", + "TRADING_MAX_BACKOUTS_PER_SYMBOL": "3", + "TRADING_MAX_CONCURRENT_POSITIONS": "10", + "TRADING_MAX_DAILY_LOSS_PCT": "0.05", + "TRADING_MAX_FRESH_ENTRIES_PER_DAY": "5", + "TRADING_MAX_GROSS_NOTIONAL_PCT": "0.60", + "TRADING_MAX_ORDERS_PER_TICK": "1", + "TRADING_MAX_POSITION_PCT": "0.16", + "TRADING_MAX_RISK_DOLLARS": "1000", + "TRADING_MIN_DAY_VOLUME": "300000", + "TRADING_MIN_ENTRY_GAP_PCT": "0.10", + "TRADING_MIN_QUALITY_SCORE": "0.20", + "TRADING_PRICE_MAX": "20.0", + "TRADING_PRICE_MIN": "2.0", + "TRADING_REENTRY_MIN_LOSS_PCT": "0.01", + "TRADING_REGIME_GATE_ENABLED": "0", + "TRADING_REGIME_GATE_P_MIN": "0.5", + "TRADING_REQUIRE_ABOVE_VWAP": "1", + "TRADING_REWARD_MULTIPLE": "10.0", + "TRADING_RISK_PER_TRADE_PCT": "0.015", + "TRADING_SESSION_ONLY": "1", + "TRADING_TRIGGER_SLIP_PCT": "0.004", + "TRADING_UNIFIED_ENTRY": "1", + "TRIGGER_ENABLED": "1", + "TRIGGER_GAP_MAX": "35", + "TRIGGER_GAP_MIN": "10.0", + "TRIGGER_GAP_SQUEEZE_MIN": "150", + "TRIGGER_GAP_SQUEEZE_RVOL": "5.0", + "TRIGGER_INTERVAL_SECONDS": "2", + "TRIGGER_MAX_ARMED": "10", + "TRIGGER_MIN_DOLLAR_VOL": "8000", + "TRIGGER_PRICE_MAX_AGE": "12", + "TRIGGER_RVOL_MIN": "2.0", + "WATCHER_PRICE_MAX": "20", + "WATCHER_PRICE_MIN": "2" + } +} diff --git a/tests/unit/research/test_live_scorecard.py b/tests/unit/research/test_live_scorecard.py new file mode 100644 index 0000000..a08ca0e --- /dev/null +++ b/tests/unit/research/test_live_scorecard.py @@ -0,0 +1,30 @@ +"""Live-test scorecard: pure verdict / drift logic.""" + +from research.live_scorecard import config_drift, load_freeze, verdict + + +def test_running_until_target(): + assert verdict([0.5] * 10, 5.0, 0, target=80).startswith("RUNNING: 10/80") + + +def test_floor_halt_fails_immediately(): + assert verdict([2.0] * 100, 50.0, 1).startswith("FAIL: the equity floor") + + +def test_pass_needs_positive_r_and_pnl(): + assert verdict([0.2] * 80, 10.0, 0).startswith("PASS") + assert verdict([0.2] * 80, -1.0, 0).startswith("FAIL") + assert verdict([-0.1] * 80, 5.0, 0).startswith("FAIL") + + +def test_drift_ignores_safety_knobs_and_catches_new_strategy_keys(): + frozen = {"TRADING_EXIT_TRAIL_PCT": "0.12", "TRADING_EQUITY_FLOOR": "215"} + current = {"TRADING_EXIT_TRAIL_PCT": "0.12", "TRADING_EQUITY_FLOOR": "190", + "TRIGGER_GAP_MAX": "40", "OLLAMA_MODEL": "x"} + assert config_drift(frozen, current) == {"TRIGGER_GAP_MAX": (None, "40")} + + +def test_freeze_file_is_valid(): + f = load_freeze() + assert f["frozen_on"] == "2026-09-24" + assert f["settings"]["TRADING_EXIT_TRAIL_PCT"] == "0.12"