diff --git a/learn_params.py b/learn_params.py index 0a638fe..b96f500 100644 --- a/learn_params.py +++ b/learn_params.py @@ -20,6 +20,8 @@ the live loop accumulate them) for a robust optimum. from __future__ import annotations import itertools +import os +from collections import defaultdict import pandas as pd from dotenv import load_dotenv @@ -99,6 +101,43 @@ def run_combo(sessions, *, warmup, min_bars, eval_every, ready, gap_min, rvol_mi } +def per_setup_report(sessions, *, warmup=6, eval_every=1, ready=65, min_bars=3, + gap_min=0.05, rvol_min=2.0): + """Break trades down by setup type — shows which setups actually make money. + + Always runs with ALL setups enabled (ignores STRATEGY_SETUPS) so the + comparison is visible even when the live config is restricted to ORB-only. + """ + saved = os.environ.pop("STRATEGY_SETUPS", None) + try: + eng = BacktestEngine( + Config(), warmup_bars=warmup, eval_every=eval_every, ready_score_pct=ready, + min_bars=min_bars, + criteria=SetupCriteria(gap_pct_min=gap_min, relative_volume_min=rvol_min), + ) + by = defaultdict(lambda: {"n": 0, "win": 0, "R": 0.0, "pnl": 0.0, "em": []}) + for symbol, _sess, bars, pc, adv in sessions: + for t in eng.run(bars, symbol, previous_close=pc, avg_daily_volume=adv).trades: + b = by[t.setup_type or "?"] + b["n"] += 1 + b["win"] += 1 if (t.realized_pnl or 0) > 0 else 0 + b["R"] += t.r_multiple or 0.0 + b["pnl"] += t.realized_pnl or 0.0 + b["em"].append(_entry_min(bars, t.entry_time)) + finally: + if saved is not None: + os.environ["STRATEGY_SETUPS"] = saved + + print("\n=== per-setup breakdown (which setups actually make money) ===") + print(f"{'setup':24s} {'trades':>6} {'win%':>5} {'avgR':>6} {'pnl':>9} {'entry_min':>9}") + for st, b in sorted(by.items(), key=lambda x: x[1]["pnl"], reverse=True): + em = [e for e in b["em"] if e == e] + em_avg = (sum(em) / len(em)) if em else float("nan") + print(f"{st:24s} {b['n']:>6} {b['win']/max(1,b['n'])*100:>4.0f}% " + f"{b['R']/max(1,b['n']):>+6.2f} {b['pnl']:>9.0f} {em_avg:>9.1f}") + print(" -> keep only the profitable setups via STRATEGY_SETUPS in .env") + + def main(): con = open_research_db("market") sessions = load_sessions(con) @@ -145,6 +184,8 @@ def main(): print(f" pnl {base['pnl']:.0f}, win {base['win']*100:.0f}%, R {base['avgR']:.2f}, " f"{base['trades']} trades, entry@{base['entry_min']:.1f}min") + per_setup_report(sessions) + if __name__ == "__main__": main() diff --git a/nightly_tune.py b/nightly_tune.py new file mode 100644 index 0000000..467ea73 --- /dev/null +++ b/nightly_tune.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Nightly self-tuning loop. + +Run after the close: backfill the latest bars, sweep the ORB strategy parameters +over the accumulated gapper sessions, and write the best config to +data/learned_params.json. run_live_paper.py reads that file on boot, so the +strategy re-tunes itself as data grows — no manual intervention. + + PYTHONPATH=. /home/philip/.venvs/momentum/bin/python nightly_tune.py + +Only writes a config that is actually PROFITABLE on the sample (>0 P&L, >=5 +trades); otherwise it leaves the existing learned_params.json untouched so a thin +or bad day can't degrade the live config. +""" + +from __future__ import annotations + +import itertools +import json +import os +from datetime import datetime, timezone + +from dotenv import load_dotenv + +from research.multi_schema import open_research_db + +load_dotenv() + + +def main(): + # Tune the profitable setup only. + os.environ["STRATEGY_SETUPS"] = "opening_range_break" + + # 1) accumulate the latest sessions (best-effort; never abort tuning on this) + try: + import backfill_history + backfill_history.main() + except Exception as exc: # noqa: BLE001 + print(f"backfill skipped: {exc}", flush=True) + + # imported after backfill so STRATEGY_SETUPS is already set for classify_setup + from learn_params import load_sessions, run_combo + + con = open_research_db("market") + sessions = load_sessions(con) + print(f"tuning over {len(sessions)} gapper sessions", flush=True) + if len(sessions) < 10: + print("too few sessions — keeping current learned_params.json") + return + + grid = { + "warmup": [3, 6], + "min_bars": [3, 5], + "eval_every": [1], + "ready": [55, 60, 65, 70], + "gap_min": [0.05], + "rvol_min": [2.0], + } + keys = list(grid) + best = None + for combo in itertools.product(*grid.values()): + p = dict(zip(keys, combo)) + if p["warmup"] < p["min_bars"]: + continue + m = run_combo(sessions, **p) + if m["trades"] < 5 or m["pnl"] <= 0: + continue + # score: profit, lightly favouring earlier entries on ties + score = m["pnl"] - (m["entry_min"] * 50.0) + if best is None or score > best[2]: + best = (p, m, score) + + if best is None: + print("no profitable ORB config on this sample — keeping current params") + return + + p, m, _ = best + out = { + "min_bars": p["min_bars"], + "ready_score_pct": float(p["ready"]), + "setups": "opening_range_break", + "as_of": datetime.now(timezone.utc).isoformat(), + "sessions": len(sessions), + "pnl": round(m["pnl"]), + "win": round(m["win"], 3), + "avg_R": round(m["avgR"], 3), + "entry_min": round(m["entry_min"], 1), + } + path = os.path.join(os.environ.get("DATA_DIR", "./data"), "learned_params.json") + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + with open(path, "w") as f: + json.dump(out, f, indent=2) + print(f"wrote {path}:\n{json.dumps(out, indent=2)}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/nightly_tune.sh b/nightly_tune.sh new file mode 100644 index 0000000..293a473 --- /dev/null +++ b/nightly_tune.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Nightly self-tuning: backfill latest bars, sweep ORB params, write +# data/learned_params.json (which run_live_paper.py reads on boot). +# +# Schedule it after the close. Examples: +# crontab -e -> 30 17 * * 1-5 /mnt/c/code/momentum/nightly_tune.sh +# (WSL: cron may need `sudo service cron start`, or use Windows Task Scheduler +# to run: wsl bash /mnt/c/code/momentum/nightly_tune.sh ) +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VENV_PY="/home/philip/.venvs/momentum/bin/python" +cd "$ROOT" || exit 1 +mkdir -p "$ROOT/data" +echo "=== nightly_tune $(date) ===" >> "$ROOT/data/nightly_tune.log" +PYTHONPATH="$ROOT" "$VENV_PY" nightly_tune.py >> "$ROOT/data/nightly_tune.log" 2>&1 diff --git a/run_live_paper.py b/run_live_paper.py index 9ee1a6f..40319f5 100644 --- a/run_live_paper.py +++ b/run_live_paper.py @@ -25,6 +25,7 @@ still work against whatever data already exists. from __future__ import annotations import argparse +import json import os import sys import time @@ -72,6 +73,16 @@ def _now_session_date(): return session_date +def _load_learned() -> dict: + """Nightly-tuned params from data/learned_params.json (empty if absent).""" + path = os.path.join(os.environ.get("DATA_DIR", "./data"), "learned_params.json") + try: + with open(path) as fh: + return json.load(fh) + except Exception: # noqa: BLE001 + return {} + + def _db_target_desc() -> str: """Human-readable Postgres target for the boot log (password masked).""" url = os.environ.get("DATABASE_URL") @@ -169,6 +180,13 @@ def build_runtime(args: argparse.Namespace) -> dict: ) ) + _learned = _load_learned() + if _learned: + print(f"[boot] learned params: ready_score={_learned.get('ready_score_pct')} " + f"min_bars={_learned.get('min_bars')} setups={_learned.get('setups')} " + f"(tuned {str(_learned.get('as_of', ''))[:10]} over {_learned.get('sessions')} " + f"sessions; backtest pnl {_learned.get('pnl')}, entry@{_learned.get('entry_min')}min)") + watcher = Watcher( store, ResearchWatchlistProvider( @@ -179,7 +197,11 @@ def build_runtime(args: argparse.Namespace) -> dict: session_id=session_id, mode=EventMode.PAPER, max_symbols=int(os.environ.get("WATCHER_MAX_SYMBOLS", "25")), - min_bars=int(os.environ.get("WATCHER_MIN_BARS", "10")), + # Defaults come from the nightly self-tuner (learned_params.json); + # an explicit env var still wins. + min_bars=int(os.environ.get("WATCHER_MIN_BARS", str(_learned.get("min_bars", 10)))), + ready_score_pct=float(os.environ.get( + "WATCHER_READY_SCORE", str(_learned.get("ready_score_pct", 60.0)))), min_quality=float(os.environ.get("WATCHER_MIN_QUALITY", "0.30")), ), )