momentum-system/youtube_claims/recap/evaluate.py

"""Score the local-model extraction against the GOLD extractions (data/ross/extracted/, done by
a strong model on the same bundles). Per video: trade count, traded tickers, entry/exit price
(+-2%), setup label, passed-on tickers, market temperature, video kind, net P&L.

  python -m youtube_claims.recap.evaluate --run v1                 # extract (cached) + score
  python -m youtube_claims.recap.evaluate --run v1 --model tools:latest --host http://192.168.1.5:11434 --num-ctx 0
  python -m youtube_claims.recap.evaluate --run v1 --score-only    # re-score cached outputs

Model outputs are cached per run under data/ross/eval/<run>/ so scoring changes are free."""

from __future__ import annotations

import argparse
import json
import time
from collections import Counter
from pathlib import Path

from youtube_claims.recap import settings


def _close(a, b, tol=0.02) -> bool:
    return a is not None and b is not None and abs(a - b) <= tol * max(abs(b), 1e-9)


def _tick(x: dict) -> str | None:
    return (x.get("ticker_guess") or x.get("ticker_heard") or "").upper().replace(" ", "") or None


def pair_trades(gold: list[dict], pred: list[dict]) -> list[tuple[dict, dict]]:
    """Match predicted to gold trades on the same ticker, greedily by agreement (setup, entry
    price, exit price), ties broken by order. A trade matches at most once."""
    pairs, used = [], set()
    cand = []
    for gi, g in enumerate(gold):
        for pi, p in enumerate(pred):
            if _tick(g) and _tick(g) == _tick(p):
                score = (g.get("setup") == p.get("setup")) + _close(p.get("entry_price"), g.get("entry_price")) \
                    + _close(p.get("exit_price"), g.get("exit_price"))
                cand.append((-score, abs(gi - pi), gi, pi))
    gdone = set()
    for _, _, gi, pi in sorted(cand):
        if gi in gdone or pi in used:
            continue
        gdone.add(gi); used.add(pi)
        pairs.append((gold[gi], pred[pi]))
    return pairs


def score_video(gold: dict, pred: dict) -> dict:
    gt, pt = gold.get("trades") or [], pred.get("trades") or []
    pairs = pair_trades(gt, pt)
    s = Counter({k: 0 for k in ("entry_price_ok", "entry_price_gold", "entry_price_pred", "exit_price_ok",
                                "exit_price_gold", "exit_price_pred", "setup_ok", "outcome_ok", "temp_gold",
                                "temp_ok", "pnl_gold", "pnl_ok")})
    s["videos"] = 1
    s["gold_trades"], s["pred_trades"], s["matched_trades"] = len(gt), len(pt), len(pairs)
    s["trade_count_exact"] = int(len(gt) == len(pt))
    s["trade_count_abs_err"] = abs(len(gt) - len(pt))
    gts, pts = {_tick(t) for t in gt} - {None}, {_tick(t) for t in pt} - {None}
    s["tick_tp"], s["tick_gold"], s["tick_pred"] = len(gts & pts), len(gts), len(pts)
    for field in ("entry_price", "exit_price"):
        for g, p in pairs:
            gv, pv = g.get(field), p.get(field)
            if gv is not None:
                s[f"{field}_gold"] += 1
            if pv is not None:
                s[f"{field}_pred"] += 1
            if _close(pv, gv):
                s[f"{field}_ok"] += 1
    for g, p in pairs:
        s["setup_ok"] += int(g.get("setup") == p.get("setup"))
        s["outcome_ok"] += int(g.get("outcome") == p.get("outcome"))
    gp = {_tick(x) for x in gold.get("passed_on") or []} - {None}
    pp = {_tick(x) for x in pred.get("passed_on") or []} - {None}
    s["pass_tp"], s["pass_gold"], s["pass_pred"] = len(gp & pp), len(gp), len(pp)
    gtemp = (gold.get("day") or {}).get("market_temperature")
    ptemp = (pred.get("day") or {}).get("market_temperature")
    if gtemp:
        s["temp_gold"] += 1
        s["temp_ok"] += int(gtemp == ptemp)
    s["kind_ok"] = int(gold.get("video_kind") == pred.get("video_kind"))
    gp_, pp_ = (gold.get("day") or {}).get("net_pnl"), (pred.get("day") or {}).get("net_pnl")
    if gp_ is not None:
        s["pnl_gold"] += 1
        s["pnl_ok"] += int(_close(pp_, gp_))
    s["rules_gold"], s["rules_pred"] = len(gold.get("rules_stated") or []), len(pred.get("rules_stated") or [])
    return dict(s)


def _pr(tp, npred, ngold) -> str:
    p = tp / npred if npred else 0.0
    r = tp / ngold if ngold else 0.0
    f = 2 * p * r / (p + r) if p + r else 0.0
    return f"P {p:.2f}  R {r:.2f}  F1 {f:.2f}  ({tp}/{npred} pred, {tp}/{ngold} gold)"


def summarize(per_video: dict[str, dict]) -> tuple[str, dict]:
    tot = Counter()
    for s in per_video.values():
        tot.update(s)
    n = tot["videos"] or 1
    lines = [
        f"videos scored: {tot['videos']}",
        f"trades (matched on ticker): {_pr(tot['matched_trades'], tot['pred_trades'], tot['gold_trades'])}",
        f"trade count exact: {tot['trade_count_exact']}/{tot['videos']}   mean |err| {tot['trade_count_abs_err'] / n:.2f}",
        f"traded tickers: {_pr(tot['tick_tp'], tot['tick_pred'], tot['tick_gold'])}",
        f"entry price +-2% (matched trades): {_pr(tot['entry_price_ok'], tot['entry_price_pred'], tot['entry_price_gold'])}",
        f"exit price +-2% (matched trades):  {_pr(tot['exit_price_ok'], tot['exit_price_pred'], tot['exit_price_gold'])}",
        f"setup label agrees: {tot['setup_ok']}/{tot['matched_trades']}   outcome agrees: {tot['outcome_ok']}/{tot['matched_trades']}",
        f"passed-on tickers: {_pr(tot['pass_tp'], tot['pass_pred'], tot['pass_gold'])}",
        f"market temperature: {tot['temp_ok']}/{tot['temp_gold']}   video kind: {tot['kind_ok']}/{tot['videos']}",
        f"net P&L +-2%: {tot['pnl_ok']}/{tot['pnl_gold']}   rules: {tot['rules_pred']} pred vs {tot['rules_gold']} gold",
    ]
    return "\n".join(lines), dict(tot)


def main() -> None:
    ap = argparse.ArgumentParser(description="Score recap extraction vs the gold set")
    ap.add_argument("--run", required=True, help="name of this configuration (cache dir)")
    ap.add_argument("--model")
    ap.add_argument("--host")
    ap.add_argument("--num-ctx", type=int, help="0 = the model's default context")
    ap.add_argument("--think", help="false|true|low|medium|high (model-dependent)")
    ap.add_argument("--chunk-chars", type=int)
    ap.add_argument("--system-file", help="alternative system prompt (prompt iteration)")
    ap.add_argument("--split", choices=["0", "1"], help="two focused calls per chunk (default: settings)")
    ap.add_argument("--limit", type=int)
    ap.add_argument("--only", help="comma-separated substrings of bundle names")
    ap.add_argument("--score-only", action="store_true")
    a = ap.parse_args()
    from youtube_claims.recap.extract import extract_bundle
    d = settings.dirs()
    gold_dir, bundles = d["extracted"], d["bundles"]
    out_dir = settings.DATA_DIR / "eval" / a.run
    out_dir.mkdir(parents=True, exist_ok=True)
    think = None if a.think is None else {"false": False, "true": True}.get(a.think.lower(), a.think)
    golds = sorted(gold_dir.glob("*.json"))
    if a.only:
        keys = a.only.split(",")
        golds = [g for g in golds if any(k in g.stem for k in keys)]
    if a.limit:
        golds = golds[:a.limit]
    per, secs = {}, []
    for g in golds:
        cache = out_dir / g.name
        if not cache.exists() and not a.score_only:
            b = (bundles / f"{g.stem}.txt").read_text(encoding="utf-8")
            t0 = time.time()
            try:
                rec, run = extract_bundle(b, model=a.model, host=a.host, num_ctx=a.num_ctx, think=think,
                                          chunk_chars=a.chunk_chars,
                                          split=None if a.split is None else a.split == "1",
                                          system=Path(a.system_file).read_text(encoding="utf-8")
                                          if a.system_file else None)
            except Exception as e:  # noqa: BLE001 -- score the rest; a failure scores as empty
                print(g.stem, "FAILED:", e, flush=True)
                rec, run = {"trades": [], "passed_on": [], "rules_stated": [], "day": {}}, {"error": str(e)}
            run["wall_seconds"] = round(time.time() - t0, 1)
            cache.write_text(json.dumps({**rec, "_run": run}, indent=1), encoding="utf-8")
        if not cache.exists():
            continue
        pred = json.loads(cache.read_text(encoding="utf-8"))
        gold = json.loads(g.read_text(encoding="utf-8"))
        per[g.stem] = score_video(gold, pred)
        r = pred.get("_run") or {}
        if r.get("wall_seconds"):
            secs.append(r["wall_seconds"])
        s = per[g.stem]
        print(f"{g.stem}: trades {s['pred_trades']}/{s['gold_trades']} matched {s['matched_trades']}  "
              f"tickers {s['tick_tp']}/{s['tick_gold']}  passed {s['pass_tp']}/{s['pass_gold']}  "
              f"temp {'ok' if s.get('temp_ok') else '-'}  {r.get('wall_seconds', '?')}s  drops {r.get('stats', {})}",
              flush=True)
    text, tot = summarize(per)
    if secs:
        text += f"\nextraction seconds/video: mean {sum(secs) / len(secs):.0f}, max {max(secs):.0f}"
    print("\n" + text)
    (out_dir / "_summary.json").write_text(json.dumps({"totals": tot, "per_video": per, "text": text}, indent=1))


if __name__ == "__main__":
    main()