momentum-system/research/pick_scoring.py

"""Score morning picks against bars — the human/LLM forecast experiment judge.

PRE-REGISTERED BAR (locked 2026-08-19 BEFORE any picks existed):
  Per forecaster, a track record PASSES only if ALL of:
    1. n >= 40 valid picks across >= 30 distinct sessions
    2. mean net return >= +0.5% per pick
    3. median net return > 0 (no skew-carried means)
    4. positive mean in BOTH halves of the sample (split at the median session)
  Scoring (fixed, same for every forecaster):
    entry = first regular-hours bar OPEN on the pick's session
    exit  = last regular-hours bar CLOSE that session (EOD)
    net   = direction-signed return minus the repo NOCHASE round-trip cost
            (4% sub-$3, 2% otherwise); shorts are scored but flagged
            (cash account cannot trade them)
  Only picks with valid=true (logged before 09:30 ET) count. Reference
  baseline: the bot's replay fire population runs -0.34R fire-weighted; a
  forecaster passing this bar is the first positive prospective signal in
  the project's history and earns a forward sizing experiment — never a
  direct live gate.

Usage:
    python -m research.pick_scoring            # score everything to date
    python -m research.pick_scoring --date 2026-08-20   # one session's detail
"""
from __future__ import annotations

import argparse
import statistics
from collections import defaultdict

from research.multi_schema import open_research_db
from storage.event_store import EventStore

COST_SUB3, COST_OTHER = 0.04, 0.02   # labeler.py NOCHASE round-trip


def _cost(price: float) -> float:
    return COST_SUB3 if price < 3.0 else COST_OTHER


def score_pick(con, symbol: str, session_date, direction: str) -> dict | None:
    """Net EOD return for one pick from minute_bars. None when no RTH bars."""
    rows = con.execute(
        "SELECT open, close FROM minute_bars WHERE symbol=? AND session_date=? "
        "AND is_regular_hours ORDER BY timestamp", [symbol, session_date]).fetchall()
    if not rows:
        return None
    entry = float(rows[0][0])
    exit_px = float(rows[-1][1])
    if entry <= 0:
        return None
    raw = exit_px / entry - 1.0
    signed = raw if direction == "long" else -raw
    return dict(entry=entry, exit=exit_px, raw=raw,
                net=signed - _cost(entry))


def evaluate(events_con, research_con, for_date=None) -> dict:
    from research.pick_journal import ensure_schema
    ensure_schema(events_con)
    where, params = "WHERE valid", []
    if for_date is not None:
        where += " AND session_date = ?"
        params.append(for_date)
    picks = events_con.execute(
        f"SELECT session_date, forecaster, symbol, direction, conviction "
        f"FROM human_picks {where} ORDER BY session_date, picked_at",
        params).fetchall()
    by_fc: dict[str, list] = defaultdict(list)
    unscored = 0
    for d, fc, sym, direction, conv in picks:
        s = score_pick(research_con, sym, d, direction)
        if s is None:
            unscored += 1
            continue
        by_fc[fc].append(dict(d=d, sym=sym, direction=direction, conv=conv, **s))
    out = {"unscored": unscored, "forecasters": {}}
    for fc, rows in by_fc.items():
        nets = [r["net"] for r in rows]
        sessions = sorted({r["d"] for r in rows})
        mid = sessions[len(sessions) // 2] if sessions else None
        h1 = [r["net"] for r in rows if r["d"] < mid] if mid else []
        h2 = [r["net"] for r in rows if r["d"] >= mid] if mid else []
        mean = statistics.mean(nets)
        med = statistics.median(nets)
        ok = (len(nets) >= 40 and len(sessions) >= 30 and mean >= 0.005
              and med > 0 and h1 and h2
              and statistics.mean(h1) > 0 and statistics.mean(h2) > 0)
        out["forecasters"][fc] = dict(
            n=len(nets), sessions=len(sessions),
            mean=round(mean, 5), median=round(med, 5),
            win=round(sum(1 for v in nets if v > 0) / len(nets), 3),
            h1=round(statistics.mean(h1), 5) if h1 else None,
            h2=round(statistics.mean(h2), 5) if h2 else None,
            shorts=sum(1 for r in rows if r["direction"] == "short"),
            bar="PASS" if ok else ("accruing" if len(nets) < 40 else "FAIL"),
            picks=[dict(d=str(r["d"]), sym=r["sym"], dir=r["direction"],
                        net=round(r["net"], 4)) for r in rows[-10:]])
    return out


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--date", default=None)
    args = ap.parse_args()
    store = EventStore()
    research = open_research_db("market")
    res = evaluate(store.con, research, for_date=args.date)
    print(f"unscored (no bars): {res['unscored']}")
    for fc, s in sorted(res["forecasters"].items()):
        print(f"\n{fc}: n={s['n']} sessions={s['sessions']} "
              f"mean={s['mean']*100:+.2f}% med={s['median']*100:+.2f}% "
              f"win={s['win']*100:.0f}% h1={s['h1']} h2={s['h2']} "
              f"shorts={s['shorts']} | BAR: {s['bar']}")
        for p in s["picks"]:
            print(f"   {p['d']} {p['sym']:6} {p['dir']:5} net={p['net']*100:+.2f}%")


if __name__ == "__main__":
    main()