momentum-system/youtube_claims/recap/nightly.py

"""The nightly pass (run on the owner's Windows workstation, ~20:30 ET weekdays):

  1. list the channel's newest uploads (yt-dlp flat listing) and drop the ones already in
     transcripts.recap_videos
  2. per new video: metadata -> audio -> faster-whisper -> bundle (meta + our scanner list +
     transcript) -> extraction (the gold file if the pilot has one, else local Ollama with the
     verbatim-quote check) -> join (tape-verified symbol + our paper/live view) -> Postgres
  3. write the cumulative report data/ross/reports/YYYY-MM-DD.md (+ .json for the next diff)

  python -m youtube_claims.recap.nightly                 # normal nightly run
  python -m youtube_claims.recap.nightly --backfill      # also ingest every local bundle not in the DB
  python -m youtube_claims.recap.nightly --reprocess ID  # force one video through again
  python -m youtube_claims.recap.nightly --report-only

Each video is independent: a failure is recorded on its recap_videos row and retried next
night (up to RECAP_MAX_ATTEMPTS), never aborting the run."""

from __future__ import annotations

import argparse
import datetime as dt
import json
import os
import sys
import time
import traceback

from youtube_claims.recap import settings


def log(*a):
    print(dt.datetime.now().strftime("%H:%M:%S"), *a, flush=True)


def _stem(meta: dict) -> str:
    return f"{meta['upload_date']}_{meta['id']}"


def extraction_for(stem: str, bundle: str, *, force_llm: bool = False) -> tuple[dict, str, dict | None]:
    """(record, extractor, run_meta). A pilot gold file wins; else the cached local-model
    output; else a fresh Ollama extraction (cached to data/ross/llm/)."""
    from youtube_claims.recap.extract import extract_bundle
    d = settings.dirs()
    gold = d["extracted"] / f"{stem}.json"
    if gold.exists() and not force_llm:
        return json.loads(gold.read_text(encoding="utf-8")), "gold", None
    cache = d["llm"] / f"{stem}.json"
    if cache.exists() and not force_llm:
        rec = json.loads(cache.read_text(encoding="utf-8"))
        run = rec.pop("_run", None)
        return rec, (run or {}).get("model") or settings.OLLAMA_MODEL, run
    rec, run = extract_bundle(bundle, log=log)
    cache.write_text(json.dumps({**rec, "_run": run}, indent=1), encoding="utf-8")
    return rec, run["model"], run


def process(meta: dict, ourside, *, force_llm: bool = False) -> dict:
    """One video end to end. Returns a small summary for the log."""
    from youtube_claims.recap import channel, store
    from youtube_claims.recap.join import join_record
    vid, stem = meta["id"], _stem(meta)
    d = settings.dirs()
    t0 = time.time()
    scanner = channel.scanner_for(meta["upload_date"], ourside)
    bfile = d["bundles"] / f"{stem}.txt"
    if bfile.exists():              # built before (pilot or an earlier night): reuse as-is
        bundle = bfile.read_text(encoding="utf-8")
        src = channel.bundle_source(bundle)
    else:
        src, segs = channel.best_transcript(vid)
        if not segs:
            log(f"{vid}: downloading audio")
            audio = channel.download_audio(vid)
            log(f"{vid}: transcribing {audio.name}")
            segs = channel.transcribe(vid, audio)
            src = "whisper-small"
            try:
                audio.unlink()      # the transcript is what we keep; audio is ~10 MB/video
            except OSError:
                pass
        bundle = channel.make_bundle(meta, src, segs, scanner)
        bfile.write_text(bundle, encoding="utf-8")
    t_tx = time.time() - t0
    t1 = time.time()
    rec, extractor, run = extraction_for(stem, bundle, force_llm=force_llm)
    rec.setdefault("video_id", vid)
    rec.setdefault("upload_date", meta["upload_date"])
    t_ex = time.time() - t1
    t2 = time.time()
    trades, passed = join_record(rec, ourside, scanner)
    t_join = time.time() - t2
    run = {**(run or {}), "timing_s": {"transcribe": round(t_tx), "extract": round(t_ex), "join": round(t_join)}}
    store.upsert_video(rec, meta, extractor=extractor, transcript_source=src, run_meta=run,
                       trades=trades, passed=passed)
    return {"kind": rec.get("video_kind"), "trades": len(trades), "passed": len(passed), "extractor": extractor,
            "seconds": round(time.time() - t0)}


def _local_bundle_metas() -> list[dict]:
    """Every pilot bundle on disk as a meta dict (for --backfill)."""
    out = []
    for f in sorted(settings.dirs()["bundles"].glob("*.txt")):
        head = f.read_text(encoding="utf-8").splitlines()[:3]
        kv = dict(ln.split(": ", 1) for ln in head if ": " in ln)
        mf = settings.dirs()["transcripts"] / f"{kv.get('video_id')}.meta.json"
        dur = json.loads(mf.read_text(encoding="utf-8")).get("duration") if mf.exists() else None
        out.append({"id": kv["video_id"], "upload_date": kv["upload_date"], "title": kv.get("title", ""),
                    "duration": dur})
    return out


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description="Nightly Ross recap learning loop")
    ap.add_argument("--backfill", action="store_true", help="ingest every local bundle not yet in the DB")
    ap.add_argument("--reprocess", action="append", default=[], help="video id to force through again")
    ap.add_argument("--force-llm", action="store_true", help="ignore gold/cached extractions for processed videos")
    ap.add_argument("--report-only", action="store_true")
    ap.add_argument("--no-list", action="store_true", help="skip the channel listing (offline)")
    ap.add_argument("--max-new", type=int, default=settings.MAX_NEW_PER_RUN)
    a = ap.parse_args(argv)

    from youtube_claims.recap import channel, report, store
    from youtube_claims.recap.ourside import OurSide
    d = settings.dirs()
    store.apply_schema()
    known = store.video_status()
    max_attempts = int(os.getenv("RECAP_MAX_ATTEMPTS", "3"))
    processed: list[str] = []
    failures: list[str] = []

    if not a.report_only:
        local = _local_bundle_metas()
        local_ids = {m["id"] for m in local}
        todo: list[dict] = []
        if not a.no_list:
            try:
                ups = channel.list_uploads()
                log(f"channel listing: {len(ups)} newest uploads")
            except Exception as e:  # noqa: BLE001 -- still ingest/report what we have
                log(f"channel listing FAILED: {e}")
                failures.append(f"listing: {e}")
                ups = []
            for u in ups:
                k = known.get(u["id"])
                if k and (k["status"] in ("done", "skipped") or
                          (k["status"] == "failed" and (k.get("attempts") or 0) >= max_attempts)):
                    continue
                if u["id"] in a.reprocess:
                    continue
                try:
                    m = channel.video_meta(u["id"], u["title"], u.get("duration"))
                except Exception as e:  # noqa: BLE001
                    log(f"{u['id']}: metadata failed: {e}")
                    failures.append(f"{u['id']} meta: {e}")
                    continue
                if not m.get("upload_date") or m["upload_date"] < settings.MIN_UPLOAD_DATE:
                    continue
                todo.append(m)
        if a.backfill:
            have = {m["id"] for m in todo}
            todo += [m for m in local
                     if m["id"] not in have and (known.get(m["id"]) or {}).get("status") != "done"]
        for vid in a.reprocess:
            todo.append(channel.video_meta(vid))
        todo.sort(key=lambda m: m["upload_date"])
        new_from_channel = [m for m in todo if m["id"] not in local_ids]
        if len(new_from_channel) > a.max_new:          # politeness cap on downloads per night
            drop = {m["id"] for m in new_from_channel[:-a.max_new]}
            todo = [m for m in todo if m["id"] not in drop]
            log(f"capped downloads at {a.max_new}; {len(drop)} older new videos wait for the next run")
        log(f"{len(todo)} video(s) to process")
        ours = OurSide()
        try:
            for m in todo:
                try:
                    s = process(m, ours, force_llm=a.force_llm and m["id"] in a.reprocess)
                    processed.append(m["id"])
                    log(f"{m['id']} {m['upload_date']} {m.get('title', '')[:50]!r}: {s}")
                except Exception as e:  # noqa: BLE001 -- one bad video never aborts the night
                    log(f"{m['id']}: FAILED {type(e).__name__}: {e}")
                    traceback.print_exc()
                    failures.append(f"{m['id']}: {type(e).__name__}: {str(e)[:200]}")
                    store.mark(m["id"], "failed", error=f"{type(e).__name__}: {e}", meta=m)
        finally:
            ours.close()

    ev = settings.DATA_DIR / "eval" / "_best.txt"
    out = report.write_report(d["reports"], new_video_ids=processed,
                              eval_text=ev.read_text(encoding="utf-8") if ev.exists() else None)
    log(f"report: {out}  (processed {len(processed)}, failures {len(failures)})")
    for f in failures:
        log("  failure:", f)
    return 1 if failures and not processed else 0


if __name__ == "__main__":
    sys.exit(main())