momentum-system/youtube_claims/recap/report.py

"""Cumulative gap report: Ross's trades vs what our system saw/did (the pilot's report.py
sections) plus "what changed since last run". compute()/render()/diff_metrics() are pure;
write_report() loads from Postgres and writes data/ross/reports/YYYY-MM-DD.{md,json}."""

from __future__ import annotations

import datetime as dt
import json
import re
import statistics
from collections import Counter, defaultdict
from pathlib import Path

GOOD = ("price_verified", "price_verified_loose", "price_matched_scanner")
BUCKETS = ["pre-7:00", "7:00-9:30", "9:30-9:45", "9:45+"]


def _f(x):
    try:
        return float(x) if x is not None else None
    except (TypeError, ValueError):
        return None


def _bucket(hhmm: str | None) -> str | None:
    if not hhmm:
        return None
    return ("pre-7:00" if hhmm < "07:00" else "7:00-9:30" if hhmm < "09:30"
            else "9:30-9:45" if hhmm < "09:45" else "9:45+")


def _disc(view: dict | None) -> str | None:
    ts = [t for side in ("paper", "live") for t, *_ in ((view or {}).get(side) or {}).get("discovered", [])]
    return min(ts) if ts else None


def _describe(xs: list[float]) -> dict:
    if not xs:
        return {"count": 0}
    xs = sorted(xs)
    q = statistics.quantiles(xs, n=4) if len(xs) > 1 else [xs[0]] * 3
    return {"count": len(xs), "mean": round(statistics.mean(xs), 2), "min": round(xs[0], 2),
            "p25": round(q[0], 2), "median": round(q[1], 2), "p75": round(q[2], 2), "max": round(xs[-1], 2)}


def trade_facts(t: dict) -> dict:
    """The per-trade columns the pilot report aggregated (also the CSV row)."""
    o = t.get("our_view") or {}
    tp = o.get("tape") or {}
    prev, e = _f(tp.get("prev_close")), _f(t.get("entry_price"))
    how = (t.get("resolve") or "").split(":")[0]
    et = t.get("entry_time_tape") if how in GOOD else None
    disc = _disc(o)
    states = [s for side in ("paper", "live") for _, s in ((o.get(side) or {}).get("states") or [])]
    blocks = [b for side in ("paper", "live") for b, _ in ((o.get(side) or {}).get("top_blocks") or [])]
    sc = t.get("stock_context") or {}
    return {
        "video": t.get("video_id"), "date": str(t.get("session_date") or o.get("date") or ""),
        "sym": t.get("symbol"), "resolve": how, "setup": t.get("setup"), "outcome": t.get("outcome"),
        "pnl": _f(t.get("pnl")), "entry": e, "entry_t": et, "entry_bucket": _bucket(et),
        "gap_at_entry": (e / prev - 1) if (e and prev) else None,
        "catalyst": sc.get("catalyst"), "discovered": disc,
        "discovered_before_entry": bool(disc and et and disc <= et),
        "ours_traded": any(((o.get(s) or {}).get("orders")) for s in ("paper", "live")),
        "out_of_population": next((s for s in states if "out_of_population" in s), None),
        "top_block": blocks[0] if blocks else None,
    }


def compute(videos: list[dict], trades: list[dict], passed: list[dict]) -> dict:
    """All report numbers as a JSON-able dict (the diff compares two of these)."""
    T = [trade_facts(t) for t in trades]
    V = [r for r in T if r["resolve"] in GOOD]
    m: dict = {
        "videos": len(videos), "trades": len(T), "passed_on": len(passed),
        "video_ids": sorted(v["video_id"] for v in videos),
        "video_kinds": dict(Counter(v.get("video_kind") for v in videos)),
        "extractors": dict(Counter(v.get("extractor") for v in videos)),
        "resolution": dict(Counter(r["resolve"] for r in T).most_common()),
        "verified": len(V),
        "entry_buckets": {b: sum(r["entry_bucket"] == b for r in V) for b in BUCKETS + [None]},
    }
    m["entry_buckets"]["unlocated"] = m["entry_buckets"].pop(None)
    wl: dict = defaultdict(Counter)
    pn: dict = defaultdict(list)
    for r in V:
        if r["entry_bucket"]:
            wl[r["entry_bucket"]][r["outcome"] or "unstated"] += 1
            if r["pnl"] is not None:
                pn[r["entry_bucket"]].append(r["pnl"])
    m["outcome_by_bucket"] = {b: dict(wl[b]) for b in BUCKETS if b in wl}
    m["pnl_by_bucket"] = {b: {"count": len(pn[b]), "sum": round(sum(pn[b])), "median": round(statistics.median(pn[b]))}
                          for b in BUCKETS if pn.get(b)}
    m["setups"] = dict(Counter(r["setup"] for r in V).most_common())
    m["setups_all"] = dict(Counter(r["setup"] for r in T).most_common())
    m["coverage"] = {"verified": len(V), "discovered": sum(r["discovered"] is not None for r in V),
                     "discovered_before_entry": sum(r["discovered_before_entry"] for r in V),
                     "we_ordered": sum(r["ours_traded"] for r in V),
                     "out_of_population": sum(r["out_of_population"] is not None for r in V)}
    prices = [r["entry"] for r in V if r["entry"] is not None]
    m["price_at_entry"] = _describe(prices)
    m["gap_at_entry"] = _describe([r["gap_at_entry"] for r in V if r["gap_at_entry"] is not None])
    m["entries_under_2"] = sum(p < 2 for p in prices)
    m["entries_over_20"] = sum(p > 20 for p in prices)
    m["oop_reasons"] = Counter(re.sub(r"[\d.]+", "#", r["out_of_population"].split("out_of_population", 1)[1])[:80]
                               for r in V if r["out_of_population"]).most_common(8)
    m["top_blocks"] = Counter(r["top_block"] for r in V if r["top_block"]).most_common(6)
    m["catalysts"] = Counter((r["catalyst"] if isinstance(r["catalyst"], str) else "none/unstated").lower()[:30]
                             for r in V).most_common(10)
    pr = []
    for p in passed:
        tp = ((p.get("our_view") or {}).get("tape") or {})
        if p.get("symbol") and tp.get("open"):
            pr.append((tp.get("open_to_hod_pct"), (tp["close"] / tp["open"] - 1) if tp.get("close") else None))
    oh = [a for a, _ in pr if a is not None]
    oc = [b for _, b in pr if b is not None]
    m["passed_tape"] = {"with_tape": len(pr), "median_open_to_hod": round(statistics.median(oh), 4) if oh else None,
                        "median_open_to_close": round(statistics.median(oc), 4) if oc else None}
    temps = {}
    for v in videos:
        if v.get("session_date") and v.get("market_temperature"):
            temps[str(v["session_date"])] = v["market_temperature"]
    m["temperature_by_day"] = dict(sorted(temps.items()))
    m["temperature"] = dict(Counter(temps.values()))
    m["trade_keys"] = sorted(f"{r['video']}|{r['sym'] or '?'}|{r['setup']}|{r['entry']}" for r in T)
    return m


def diff_metrics(prev: dict | None, cur: dict) -> list[str]:
    """Human lines describing what changed between two compute() results."""
    if not prev:
        return ["First report -- no previous run to compare with."]
    out = []
    new_v = sorted(set(cur.get("video_ids", [])) - set(prev.get("video_ids", [])))
    gone_v = sorted(set(prev.get("video_ids", [])) - set(cur.get("video_ids", [])))
    if new_v:
        out.append(f"New videos ({len(new_v)}): " + ", ".join(new_v))
    if gone_v:
        out.append(f"Videos no longer in the set ({len(gone_v)}): " + ", ".join(gone_v))
    for k in ("videos", "trades", "passed_on", "verified"):
        if prev.get(k) != cur.get(k):
            out.append(f"{k}: {prev.get(k)} -> {cur.get(k)} ({cur.get(k, 0) - (prev.get(k) or 0):+d})")
    for group in ("coverage", "entry_buckets", "temperature", "setups", "resolution"):
        a, b = prev.get(group) or {}, cur.get(group) or {}
        for k in sorted(set(a) | set(b), key=str):
            if a.get(k, 0) != b.get(k, 0):
                out.append(f"{group}.{k}: {a.get(k, 0)} -> {b.get(k, 0)}")
    pa, pb = (prev.get("price_at_entry") or {}).get("median"), (cur.get("price_at_entry") or {}).get("median")
    if pa != pb:
        out.append(f"median entry price: {pa} -> {pb}")
    new_t = sorted(set(cur.get("trade_keys", [])) - set(prev.get("trade_keys", [])))
    if new_t:
        out.append(f"New trades ({len(new_t)}): " + "; ".join(k.split("|", 1)[1] for k in new_t[:20]))
    return out or ["No change since the previous run."]


def _table(rows: list[list], head: list[str]) -> str:
    out = ["| " + " | ".join(head) + " |", "|" + "---|" * len(head)]
    out += ["| " + " | ".join("" if c is None else str(c) for c in r) + " |" for r in rows]
    return "\n".join(out)


def render(m: dict, changes: list[str], *, today: str, new_trades: list[dict] | None = None,
           eval_text: str | None = None) -> str:
    pct = lambda x: "-" if x is None else f"{x:.1%}"  # noqa: E731
    cov = m["coverage"]
    L = [f"# Ross recap vs our bot -- {today}", "",
         f"Videos {m['videos']} ({', '.join(f'{k} {v}' for k, v in m['video_kinds'].items())}), "
         f"trades {m['trades']}, passed-on {m['passed_on']}. Extractors: "
         + ", ".join(f"{k} {v}" for k, v in m["extractors"].items()) + ".", "",
         "## What changed since last run", ""] + [f"- {c}" for c in changes] + [""]
    if new_trades:
        L += ["## New trades this run", "", _table(
            [[t.get("session_date"), t.get("symbol") or f"?{t.get('ticker_guess')}", t.get("resolve"), t.get("setup"),
              t.get("entry_price"), t.get("outcome"), t.get("entry_time_tape"), _disc(t.get("our_view")),
              "yes" if any(((t.get("our_view") or {}).get(s) or {}).get("orders") for s in ("paper", "live")) else ""]
             for t in new_trades],
            ["date", "symbol", "resolve", "setup", "entry", "outcome", "entry@tape", "we discovered", "we ordered"]), ""]
    L += ["## Ticker resolution (trades)", "",
          ", ".join(f"{k} {v}" for k, v in m["resolution"].items()) or "-", "",
          f"Price-verified trades: {m['verified']} of {m['trades']}", "",
          "## When he entered (verified, entry located on tape)", "",
          _table([[b, m["entry_buckets"].get(b, 0)] for b in BUCKETS + ["unlocated"]], ["window", "trades"]), "",
          "## Win/loss by entry window (self-reported)", ""]
    outs = sorted({o for d in m["outcome_by_bucket"].values() for o in d})
    L += [_table([[b] + [m["outcome_by_bucket"][b].get(o, 0) for o in outs] for b in m["outcome_by_bucket"]],
                 ["window"] + outs) if outs else "-", "",
          "## Stated $ P&L by entry window", "",
          _table([[b, v["count"], v["sum"], v["median"]] for b, v in m["pnl_by_bucket"].items()],
                 ["window", "count", "sum", "median"]) if m["pnl_by_bucket"] else "-", "",
          "## Setups (verified trades)", "", ", ".join(f"{k} {v}" for k, v in m["setups"].items()) or "-", "",
          "## Our coverage of his verified trades", "",
          f"- discovered by our scanner at all: {cov['discovered']} / {cov['verified']}",
          f"- discovered BEFORE his entry: {cov['discovered_before_entry']}",
          f"- we placed an order that day: {cov['we_ordered']}",
          f"- rejected as out_of_population: {cov['out_of_population']}", "",
          "## Price and gap at his entry", "",
          f"- price: {m['price_at_entry']}",
          f"- gap vs prior close: {m['gap_at_entry']}",
          f"- entries under $2: {m['entries_under_2']}, over $20: {m['entries_over_20']}", "",
          "## out_of_population reasons", ""] + [f"- {k} ({v})" for k, v in m["oop_reasons"]] + [
          "", "## Our top block reasons on his names", ""] + [f"- {k} ({v})" for k, v in m["top_blocks"]] + [
          "", "## Catalysts he cited", "", ", ".join(f"{k} {v}" for k, v in m["catalysts"]) or "-", "",
          "## Passed-on names: what happened after he passed", "",
          f"{m['passed_tape']['with_tape']} with tape; median open->HOD {pct(m['passed_tape']['median_open_to_hod'])}, "
          f"open->close {pct(m['passed_tape']['median_open_to_close'])}", "",
          "## Market temperature", "", ", ".join(f"{k} {v}" for k, v in m["temperature"].items()) or "-", "",
          ", ".join(f"{d} {t}" for d, t in list(m["temperature_by_day"].items())[-15:]), ""]
    if eval_text:
        L += ["## Extraction quality (local model vs gold set)", "", "```", eval_text, "```", ""]
    return "\n".join(L)


def previous_metrics(reports_dir: Path, today: str) -> dict | None:
    prior = sorted(p for p in reports_dir.glob("*.json") if p.stem < today)
    return json.loads(prior[-1].read_text(encoding="utf-8")) if prior else None


def write_report(reports_dir: Path, *, new_video_ids: list[str] | None = None, eval_text: str | None = None,
                 today: str | None = None) -> Path:
    from youtube_claims.recap import store
    today = today or dt.date.today().isoformat()
    videos, trades, passed = store.load_rows()
    m = compute(videos, trades, passed)
    changes = diff_metrics(previous_metrics(reports_dir, today), m)
    new_trades = [t for t in trades if t["video_id"] in set(new_video_ids or [])]
    md = render(m, changes, today=today, new_trades=new_trades, eval_text=eval_text)
    out = reports_dir / f"{today}.md"
    out.write_text(md, encoding="utf-8")
    (reports_dir / f"{today}.json").write_text(json.dumps(m, default=str, indent=1), encoding="utf-8")
    return out