Self-tuning loop: per-setup report + nightly re-sweep + live learned-params

(a) learn_params.per_setup_report: permanent per-setup P&L/earliness breakdown
    in the harness (forces all-setups so the ORB-vs-rest comparison is visible).
(b) nightly_tune.py + nightly_tune.sh: backfill latest bars, sweep ORB params
    over accumulated gapper sessions, write the best PROFITABLE config to
    data/learned_params.json. run_live_paper reads it on boot and applies the
    tuned ready_score / min_bars (explicit env still wins); won't overwrite with
    a losing or thin-sample config. Cron-scheduled after the close.

Verified: 75/75 tests; boot logs the learned params and applies them.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

learn_params.py

@@ -20,6 +20,8 @@ the live loop accumulate them) for a robust optimum.
 from __future__ import annotations
 
 import itertools
+import os
+from collections import defaultdict
 
 import pandas as pd
 from dotenv import load_dotenv
@@ -99,6 +101,43 @@ def run_combo(sessions, *, warmup, min_bars, eval_every, ready, gap_min, rvol_mi
     }
 
 
+def per_setup_report(sessions, *, warmup=6, eval_every=1, ready=65, min_bars=3,
+                     gap_min=0.05, rvol_min=2.0):
+    """Break trades down by setup type — shows which setups actually make money.
+
+    Always runs with ALL setups enabled (ignores STRATEGY_SETUPS) so the
+    comparison is visible even when the live config is restricted to ORB-only.
+    """
+    saved = os.environ.pop("STRATEGY_SETUPS", None)
+    try:
+        eng = BacktestEngine(
+            Config(), warmup_bars=warmup, eval_every=eval_every, ready_score_pct=ready,
+            min_bars=min_bars,
+            criteria=SetupCriteria(gap_pct_min=gap_min, relative_volume_min=rvol_min),
+        )
+        by = defaultdict(lambda: {"n": 0, "win": 0, "R": 0.0, "pnl": 0.0, "em": []})
+        for symbol, _sess, bars, pc, adv in sessions:
+            for t in eng.run(bars, symbol, previous_close=pc, avg_daily_volume=adv).trades:
+                b = by[t.setup_type or "?"]
+                b["n"] += 1
+                b["win"] += 1 if (t.realized_pnl or 0) > 0 else 0
+                b["R"] += t.r_multiple or 0.0
+                b["pnl"] += t.realized_pnl or 0.0
+                b["em"].append(_entry_min(bars, t.entry_time))
+    finally:
+        if saved is not None:
+            os.environ["STRATEGY_SETUPS"] = saved
+
+    print("\n=== per-setup breakdown (which setups actually make money) ===")
+    print(f"{'setup':24s} {'trades':>6} {'win%':>5} {'avgR':>6} {'pnl':>9} {'entry_min':>9}")
+    for st, b in sorted(by.items(), key=lambda x: x[1]["pnl"], reverse=True):
+        em = [e for e in b["em"] if e == e]
+        em_avg = (sum(em) / len(em)) if em else float("nan")
+        print(f"{st:24s} {b['n']:>6} {b['win']/max(1,b['n'])*100:>4.0f}% "
+              f"{b['R']/max(1,b['n']):>+6.2f} {b['pnl']:>9.0f} {em_avg:>9.1f}")
+    print("  -> keep only the profitable setups via STRATEGY_SETUPS in .env")
+
+
 def main():
     con = open_research_db("market")
     sessions = load_sessions(con)
@@ -145,6 +184,8 @@ def main():
     print(f"  pnl {base['pnl']:.0f}, win {base['win']*100:.0f}%, R {base['avgR']:.2f}, "
           f"{base['trades']} trades, entry@{base['entry_min']:.1f}min")
 
+    per_setup_report(sessions)
+
 
 if __name__ == "__main__":
     main()

nightly_tune.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""Nightly self-tuning loop.
+
+Run after the close: backfill the latest bars, sweep the ORB strategy parameters
+over the accumulated gapper sessions, and write the best config to
+data/learned_params.json. run_live_paper.py reads that file on boot, so the
+strategy re-tunes itself as data grows — no manual intervention.
+
+    PYTHONPATH=. /home/philip/.venvs/momentum/bin/python nightly_tune.py
+
+Only writes a config that is actually PROFITABLE on the sample (>0 P&L, >=5
+trades); otherwise it leaves the existing learned_params.json untouched so a thin
+or bad day can't degrade the live config.
+"""
+
+from __future__ import annotations
+
+import itertools
+import json
+import os
+from datetime import datetime, timezone
+
+from dotenv import load_dotenv
+
+from research.multi_schema import open_research_db
+
+load_dotenv()
+
+
+def main():
+    # Tune the profitable setup only.
+    os.environ["STRATEGY_SETUPS"] = "opening_range_break"
+
+    # 1) accumulate the latest sessions (best-effort; never abort tuning on this)
+    try:
+        import backfill_history
+        backfill_history.main()
+    except Exception as exc:  # noqa: BLE001
+        print(f"backfill skipped: {exc}", flush=True)
+
+    # imported after backfill so STRATEGY_SETUPS is already set for classify_setup
+    from learn_params import load_sessions, run_combo
+
+    con = open_research_db("market")
+    sessions = load_sessions(con)
+    print(f"tuning over {len(sessions)} gapper sessions", flush=True)
+    if len(sessions) < 10:
+        print("too few sessions — keeping current learned_params.json")
+        return
+
+    grid = {
+        "warmup": [3, 6],
+        "min_bars": [3, 5],
+        "eval_every": [1],
+        "ready": [55, 60, 65, 70],
+        "gap_min": [0.05],
+        "rvol_min": [2.0],
+    }
+    keys = list(grid)
+    best = None
+    for combo in itertools.product(*grid.values()):
+        p = dict(zip(keys, combo))
+        if p["warmup"] < p["min_bars"]:
+            continue
+        m = run_combo(sessions, **p)
+        if m["trades"] < 5 or m["pnl"] <= 0:
+            continue
+        # score: profit, lightly favouring earlier entries on ties
+        score = m["pnl"] - (m["entry_min"] * 50.0)
+        if best is None or score > best[2]:
+            best = (p, m, score)
+
+    if best is None:
+        print("no profitable ORB config on this sample — keeping current params")
+        return
+
+    p, m, _ = best
+    out = {
+        "min_bars": p["min_bars"],
+        "ready_score_pct": float(p["ready"]),
+        "setups": "opening_range_break",
+        "as_of": datetime.now(timezone.utc).isoformat(),
+        "sessions": len(sessions),
+        "pnl": round(m["pnl"]),
+        "win": round(m["win"], 3),
+        "avg_R": round(m["avgR"], 3),
+        "entry_min": round(m["entry_min"], 1),
+    }
+    path = os.path.join(os.environ.get("DATA_DIR", "./data"), "learned_params.json")
+    os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+    with open(path, "w") as f:
+        json.dump(out, f, indent=2)
+    print(f"wrote {path}:\n{json.dumps(out, indent=2)}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

nightly_tune.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Nightly self-tuning: backfill latest bars, sweep ORB params, write
+# data/learned_params.json (which run_live_paper.py reads on boot).
+#
+# Schedule it after the close. Examples:
+#   crontab -e   ->   30 17 * * 1-5  /mnt/c/code/momentum/nightly_tune.sh
+#   (WSL: cron may need `sudo service cron start`, or use Windows Task Scheduler
+#    to run:  wsl bash /mnt/c/code/momentum/nightly_tune.sh )
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+VENV_PY="/home/philip/.venvs/momentum/bin/python"
+cd "$ROOT" || exit 1
+mkdir -p "$ROOT/data"
+echo "=== nightly_tune $(date) ===" >> "$ROOT/data/nightly_tune.log"
+PYTHONPATH="$ROOT" "$VENV_PY" nightly_tune.py >> "$ROOT/data/nightly_tune.log" 2>&1

run_live_paper.py

@@ -25,6 +25,7 @@ still work against whatever data already exists.
 from __future__ import annotations
 
 import argparse
+import json
 import os
 import sys
 import time
@@ -72,6 +73,16 @@ def _now_session_date():
     return session_date
 
 
+def _load_learned() -> dict:
+    """Nightly-tuned params from data/learned_params.json (empty if absent)."""
+    path = os.path.join(os.environ.get("DATA_DIR", "./data"), "learned_params.json")
+    try:
+        with open(path) as fh:
+            return json.load(fh)
+    except Exception:  # noqa: BLE001
+        return {}
+
+
 def _db_target_desc() -> str:
     """Human-readable Postgres target for the boot log (password masked)."""
     url = os.environ.get("DATABASE_URL")
@@ -169,6 +180,13 @@ def build_runtime(args: argparse.Namespace) -> dict:
         )
     )
 
+    _learned = _load_learned()
+    if _learned:
+        print(f"[boot] learned params: ready_score={_learned.get('ready_score_pct')} "
+              f"min_bars={_learned.get('min_bars')} setups={_learned.get('setups')} "
+              f"(tuned {str(_learned.get('as_of', ''))[:10]} over {_learned.get('sessions')} "
+              f"sessions; backtest pnl {_learned.get('pnl')}, entry@{_learned.get('entry_min')}min)")
+
     watcher = Watcher(
         store,
         ResearchWatchlistProvider(
@@ -179,7 +197,11 @@ def build_runtime(args: argparse.Namespace) -> dict:
             session_id=session_id,
             mode=EventMode.PAPER,
             max_symbols=int(os.environ.get("WATCHER_MAX_SYMBOLS", "25")),
-            min_bars=int(os.environ.get("WATCHER_MIN_BARS", "10")),
+            # Defaults come from the nightly self-tuner (learned_params.json);
+            # an explicit env var still wins.
+            min_bars=int(os.environ.get("WATCHER_MIN_BARS", str(_learned.get("min_bars", 10)))),
+            ready_score_pct=float(os.environ.get(
+                "WATCHER_READY_SCORE", str(_learned.get("ready_score_pct", 60.0)))),
             min_quality=float(os.environ.get("WATCHER_MIN_QUALITY", "0.30")),
         ),
     )