"""The nightly pass (run on the owner's Windows workstation, ~20:30 ET weekdays): 1. list the channel's newest uploads (yt-dlp flat listing) and drop the ones already in transcripts.recap_videos 2. per new video: metadata -> audio -> faster-whisper -> bundle (meta + our scanner list + transcript) -> extraction (the gold file if the pilot has one, else local Ollama with the verbatim-quote check) -> join (tape-verified symbol + our paper/live view) -> Postgres 3. write the cumulative report data/ross/reports/YYYY-MM-DD.md (+ .json for the next diff) python -m youtube_claims.recap.nightly # normal nightly run python -m youtube_claims.recap.nightly --backfill # also ingest every local bundle not in the DB python -m youtube_claims.recap.nightly --reprocess ID # force one video through again python -m youtube_claims.recap.nightly --report-only Each video is independent: a failure is recorded on its recap_videos row and retried next night (up to RECAP_MAX_ATTEMPTS), never aborting the run.""" from __future__ import annotations import argparse import datetime as dt import json import os import sys import time import traceback from youtube_claims.recap import settings def log(*a): print(dt.datetime.now().strftime("%H:%M:%S"), *a, flush=True) def _stem(meta: dict) -> str: return f"{meta['upload_date']}_{meta['id']}" def extraction_for(stem: str, bundle: str, *, force_llm: bool = False) -> tuple[dict, str, dict | None]: """(record, extractor, run_meta). A pilot gold file wins; else the cached local-model output; else a fresh Ollama extraction (cached to data/ross/llm/).""" from youtube_claims.recap.extract import extract_bundle d = settings.dirs() gold = d["extracted"] / f"{stem}.json" if gold.exists() and not force_llm: return json.loads(gold.read_text(encoding="utf-8")), "gold", None cache = d["llm"] / f"{stem}.json" if cache.exists() and not force_llm: rec = json.loads(cache.read_text(encoding="utf-8")) run = rec.pop("_run", None) return rec, (run or {}).get("model") or settings.OLLAMA_MODEL, run rec, run = extract_bundle(bundle, log=log) cache.write_text(json.dumps({**rec, "_run": run}, indent=1), encoding="utf-8") return rec, run["model"], run def process(meta: dict, ourside, *, force_llm: bool = False) -> dict: """One video end to end. Returns a small summary for the log.""" from youtube_claims.recap import channel, store from youtube_claims.recap.join import join_record vid, stem = meta["id"], _stem(meta) d = settings.dirs() t0 = time.time() scanner = channel.scanner_for(meta["upload_date"], ourside) bfile = d["bundles"] / f"{stem}.txt" if bfile.exists(): # built before (pilot or an earlier night): reuse as-is bundle = bfile.read_text(encoding="utf-8") src = channel.bundle_source(bundle) else: src, segs = channel.best_transcript(vid) if not segs: log(f"{vid}: downloading audio") audio = channel.download_audio(vid) log(f"{vid}: transcribing {audio.name}") segs = channel.transcribe(vid, audio) src = "whisper-small" try: audio.unlink() # the transcript is what we keep; audio is ~10 MB/video except OSError: pass bundle = channel.make_bundle(meta, src, segs, scanner) bfile.write_text(bundle, encoding="utf-8") t_tx = time.time() - t0 t1 = time.time() rec, extractor, run = extraction_for(stem, bundle, force_llm=force_llm) rec.setdefault("video_id", vid) rec.setdefault("upload_date", meta["upload_date"]) t_ex = time.time() - t1 t2 = time.time() trades, passed = join_record(rec, ourside, scanner) t_join = time.time() - t2 run = {**(run or {}), "timing_s": {"transcribe": round(t_tx), "extract": round(t_ex), "join": round(t_join)}} store.upsert_video(rec, meta, extractor=extractor, transcript_source=src, run_meta=run, trades=trades, passed=passed) return {"kind": rec.get("video_kind"), "trades": len(trades), "passed": len(passed), "extractor": extractor, "seconds": round(time.time() - t0)} def _local_bundle_metas() -> list[dict]: """Every pilot bundle on disk as a meta dict (for --backfill).""" out = [] for f in sorted(settings.dirs()["bundles"].glob("*.txt")): head = f.read_text(encoding="utf-8").splitlines()[:3] kv = dict(ln.split(": ", 1) for ln in head if ": " in ln) mf = settings.dirs()["transcripts"] / f"{kv.get('video_id')}.meta.json" dur = json.loads(mf.read_text(encoding="utf-8")).get("duration") if mf.exists() else None out.append({"id": kv["video_id"], "upload_date": kv["upload_date"], "title": kv.get("title", ""), "duration": dur}) return out def main(argv=None) -> int: ap = argparse.ArgumentParser(description="Nightly Ross recap learning loop") ap.add_argument("--backfill", action="store_true", help="ingest every local bundle not yet in the DB") ap.add_argument("--reprocess", action="append", default=[], help="video id to force through again") ap.add_argument("--force-llm", action="store_true", help="ignore gold/cached extractions for processed videos") ap.add_argument("--report-only", action="store_true") ap.add_argument("--no-list", action="store_true", help="skip the channel listing (offline)") ap.add_argument("--max-new", type=int, default=settings.MAX_NEW_PER_RUN) a = ap.parse_args(argv) from youtube_claims.recap import channel, report, store from youtube_claims.recap.ourside import OurSide d = settings.dirs() store.apply_schema() known = store.video_status() max_attempts = int(os.getenv("RECAP_MAX_ATTEMPTS", "3")) processed: list[str] = [] failures: list[str] = [] if not a.report_only: local = _local_bundle_metas() local_ids = {m["id"] for m in local} todo: list[dict] = [] if not a.no_list: try: ups = channel.list_uploads() log(f"channel listing: {len(ups)} newest uploads") except Exception as e: # noqa: BLE001 -- still ingest/report what we have log(f"channel listing FAILED: {e}") failures.append(f"listing: {e}") ups = [] for u in ups: k = known.get(u["id"]) if k and (k["status"] in ("done", "skipped") or (k["status"] == "failed" and (k.get("attempts") or 0) >= max_attempts)): continue if u["id"] in a.reprocess: continue try: m = channel.video_meta(u["id"], u["title"], u.get("duration")) except Exception as e: # noqa: BLE001 log(f"{u['id']}: metadata failed: {e}") failures.append(f"{u['id']} meta: {e}") continue if not m.get("upload_date") or m["upload_date"] < settings.MIN_UPLOAD_DATE: continue todo.append(m) if a.backfill: have = {m["id"] for m in todo} todo += [m for m in local if m["id"] not in have and (known.get(m["id"]) or {}).get("status") != "done"] for vid in a.reprocess: todo.append(channel.video_meta(vid)) todo.sort(key=lambda m: m["upload_date"]) new_from_channel = [m for m in todo if m["id"] not in local_ids] if len(new_from_channel) > a.max_new: # politeness cap on downloads per night drop = {m["id"] for m in new_from_channel[:-a.max_new]} todo = [m for m in todo if m["id"] not in drop] log(f"capped downloads at {a.max_new}; {len(drop)} older new videos wait for the next run") log(f"{len(todo)} video(s) to process") ours = OurSide() try: for m in todo: try: s = process(m, ours, force_llm=a.force_llm and m["id"] in a.reprocess) processed.append(m["id"]) log(f"{m['id']} {m['upload_date']} {m.get('title', '')[:50]!r}: {s}") except Exception as e: # noqa: BLE001 -- one bad video never aborts the night log(f"{m['id']}: FAILED {type(e).__name__}: {e}") traceback.print_exc() failures.append(f"{m['id']}: {type(e).__name__}: {str(e)[:200]}") store.mark(m["id"], "failed", error=f"{type(e).__name__}: {e}", meta=m) finally: ours.close() ev = settings.DATA_DIR / "eval" / "_best.txt" out = report.write_report(d["reports"], new_video_ids=processed, eval_text=ev.read_text(encoding="utf-8") if ev.exists() else None) log(f"report: {out} (processed {len(processed)}, failures {len(failures)})") for f in failures: log(" failure:", f) return 1 if failures and not processed else 0 if __name__ == "__main__": sys.exit(main())