"""Score morning picks against bars — the human/LLM forecast experiment judge. PRE-REGISTERED BAR (locked 2026-08-19 BEFORE any picks existed): Per forecaster, a track record PASSES only if ALL of: 1. n >= 40 valid picks across >= 30 distinct sessions 2. mean net return >= +0.5% per pick 3. median net return > 0 (no skew-carried means) 4. positive mean in BOTH halves of the sample (split at the median session) Scoring (fixed, same for every forecaster): entry = first regular-hours bar OPEN on the pick's session exit = last regular-hours bar CLOSE that session (EOD) net = direction-signed return minus the repo NOCHASE round-trip cost (4% sub-$3, 2% otherwise); shorts are scored but flagged (cash account cannot trade them) Only picks with valid=true (logged before 09:30 ET) count. Reference baseline: the bot's replay fire population runs -0.34R fire-weighted; a forecaster passing this bar is the first positive prospective signal in the project's history and earns a forward sizing experiment — never a direct live gate. Usage: python -m research.pick_scoring # score everything to date python -m research.pick_scoring --date 2026-08-20 # one session's detail """ from __future__ import annotations import argparse import statistics from collections import defaultdict from research.multi_schema import open_research_db from storage.event_store import EventStore COST_SUB3, COST_OTHER = 0.04, 0.02 # labeler.py NOCHASE round-trip def _cost(price: float) -> float: return COST_SUB3 if price < 3.0 else COST_OTHER def score_pick(con, symbol: str, session_date, direction: str) -> dict | None: """Net EOD return for one pick from minute_bars. None when no RTH bars.""" rows = con.execute( "SELECT open, close FROM minute_bars WHERE symbol=? AND session_date=? " "AND is_regular_hours ORDER BY timestamp", [symbol, session_date]).fetchall() if not rows: return None entry = float(rows[0][0]) exit_px = float(rows[-1][1]) if entry <= 0: return None raw = exit_px / entry - 1.0 signed = raw if direction == "long" else -raw return dict(entry=entry, exit=exit_px, raw=raw, net=signed - _cost(entry)) def evaluate(events_con, research_con, for_date=None) -> dict: from research.pick_journal import ensure_schema ensure_schema(events_con) where, params = "WHERE valid", [] if for_date is not None: where += " AND session_date = ?" params.append(for_date) picks = events_con.execute( f"SELECT session_date, forecaster, symbol, direction, conviction " f"FROM human_picks {where} ORDER BY session_date, picked_at", params).fetchall() by_fc: dict[str, list] = defaultdict(list) unscored = 0 for d, fc, sym, direction, conv in picks: s = score_pick(research_con, sym, d, direction) if s is None: unscored += 1 continue by_fc[fc].append(dict(d=d, sym=sym, direction=direction, conv=conv, **s)) out = {"unscored": unscored, "forecasters": {}} for fc, rows in by_fc.items(): nets = [r["net"] for r in rows] sessions = sorted({r["d"] for r in rows}) mid = sessions[len(sessions) // 2] if sessions else None h1 = [r["net"] for r in rows if r["d"] < mid] if mid else [] h2 = [r["net"] for r in rows if r["d"] >= mid] if mid else [] mean = statistics.mean(nets) med = statistics.median(nets) ok = (len(nets) >= 40 and len(sessions) >= 30 and mean >= 0.005 and med > 0 and h1 and h2 and statistics.mean(h1) > 0 and statistics.mean(h2) > 0) out["forecasters"][fc] = dict( n=len(nets), sessions=len(sessions), mean=round(mean, 5), median=round(med, 5), win=round(sum(1 for v in nets if v > 0) / len(nets), 3), h1=round(statistics.mean(h1), 5) if h1 else None, h2=round(statistics.mean(h2), 5) if h2 else None, shorts=sum(1 for r in rows if r["direction"] == "short"), bar="PASS" if ok else ("accruing" if len(nets) < 40 else "FAIL"), picks=[dict(d=str(r["d"]), sym=r["sym"], dir=r["direction"], net=round(r["net"], 4)) for r in rows[-10:]]) return out def main(): ap = argparse.ArgumentParser() ap.add_argument("--date", default=None) args = ap.parse_args() store = EventStore() research = open_research_db("market") res = evaluate(store.con, research, for_date=args.date) print(f"unscored (no bars): {res['unscored']}") for fc, s in sorted(res["forecasters"].items()): print(f"\n{fc}: n={s['n']} sessions={s['sessions']} " f"mean={s['mean']*100:+.2f}% med={s['median']*100:+.2f}% " f"win={s['win']*100:.0f}% h1={s['h1']} h2={s['h2']} " f"shorts={s['shorts']} | BAR: {s['bar']}") for p in s["picks"]: print(f" {p['d']} {p['sym']:6} {p['dir']:5} net={p['net']*100:+.2f}%") if __name__ == "__main__": main()