momentum devandClaude Opus 4.8
4a54b2adcc
A descriptive ingestion layer feeding the downstream reasoning model: monitors YouTube
playlists, transcribes new uploads, and extracts structured, verifiable CLAIM records
(with provenance + a mandatory verbatim quote). It records WHAT was claimed; it never
judges correctness or tradeability -- all judgment stays downstream, in one place.
Built per the operator's spec. Two clocks coordinated through Postgres (§2): a 30-min
detection poll + a paced concurrency-1 transcription/extraction worker. Reuses the app's
infra: shared Postgres (isolated `transcripts` schema), the app's Ollama for extraction
(free, on-LAN, private -- transcripts never leave the network), the config.py host resolver.
youtube_claims/schema.sql playlists / videos / claims (§7-8); claims.verbatim_quote NOT NULL
youtube_claims/db.py psycopg2; claim-by-atomic-status-flip under FOR UPDATE SKIP LOCKED
youtube_claims/detection.py playlistItems.list over urllib; dedupe by video_id (§5)
youtube_claims/extraction.py local Ollama -> descriptive claims; drops any claim lacking an
exact verbatim quote present in the source window (anti-hallucination)
youtube_claims/transcription.py yt-dlp -> faster-whisper (medium/cpu/int8, option A;
initial_prompt=watchlist) with youtube-transcript-api captions fallback
youtube_claims/worker.py paced drain; per-video state machine + retries/backoff
youtube_claims/run.py APScheduler poll + worker loop in one process; CLI for setup
VERIFIED LIVE: schema applied; playlist "Anatomy of a Trade" (Ross Cameron) added; poll
ingested all 12 videos (1 API unit); Ollama extraction pulled clean descriptive claims
(TC/DXST/CF) with exact verbatim quotes from a real transcript. Whisper transcription pending
ffmpeg (`apt install ffmpeg`, needs sudo) + the yt-dlp/faster-whisper deps (installing via uv).
Secrets (YOUTUBE_API_KEY) live in gitignored .env only. Whisper is option A (CPU); env-flippable
to option B (LAN GPU large-v3) later.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
65 lines
2.4 KiB
Python
65 lines
2.4 KiB
Python
"""Config — env-driven (§10 defaults). Reuses the app's DATABASE_URL and Ollama host;
|
|
secrets (YOUTUBE_API_KEY) come from .env only, never hard-coded."""
|
|
|
|
import os
|
|
|
|
_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
# load .env once (the same file the trading app uses; gitignored)
|
|
try:
|
|
from dotenv import load_dotenv
|
|
load_dotenv(os.path.join(_ROOT, ".env"))
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
|
|
# --- §10 defaults (env-overridable) ---
|
|
POLL_INTERVAL_MINUTES = int(os.getenv("YT_POLL_INTERVAL_MINUTES", "30"))
|
|
PLAYLIST_PAGE_SIZE = 50
|
|
WORKER_CONCURRENCY = 1 # deliberate — §13 (yt-dlp politeness)
|
|
INTER_VIDEO_DELAY_SECONDS = int(os.getenv("YT_INTER_VIDEO_DELAY_SECONDS", "60"))
|
|
MAX_RETRIES = int(os.getenv("YT_MAX_RETRIES", "3"))
|
|
RETRY_BACKOFF_BASE_SEC = 30 # 30, 60, 120...
|
|
YT_DLP_FORMAT = "bestaudio"
|
|
YOUTUBE_API_QUOTA_DAILY = 10000
|
|
TRANSCRIPT_DIR = os.getenv("YT_TRANSCRIPT_DIR", os.path.join(_ROOT, "data", "transcripts"))
|
|
PG_SCHEMA = os.getenv("YT_PG_SCHEMA", "transcripts")
|
|
|
|
# Whisper — option A (CPU on this WSL box; no local GPU). Flip to large-v3/cuda/float16
|
|
# via env if we later move the worker to the LAN GPU host (option B).
|
|
WHISPER_MODEL = os.getenv("YT_WHISPER_MODEL", "medium")
|
|
WHISPER_DEVICE = os.getenv("YT_WHISPER_DEVICE", "cpu")
|
|
WHISPER_COMPUTE_TYPE = os.getenv("YT_WHISPER_COMPUTE_TYPE", "int8")
|
|
|
|
# Extraction LLM — local Ollama (free, on-LAN, private; transcripts never leave the network).
|
|
OLLAMA_MODEL = os.getenv("YT_EXTRACT_MODEL", "qwen2.5:7b-instruct")
|
|
|
|
|
|
def youtube_api_key() -> str:
|
|
k = os.getenv("YOUTUBE_API_KEY")
|
|
if not k:
|
|
raise RuntimeError("YOUTUBE_API_KEY not set — put it in .env")
|
|
return k
|
|
|
|
|
|
def database_url() -> str:
|
|
u = os.getenv("DATABASE_URL")
|
|
if not u:
|
|
raise RuntimeError("DATABASE_URL not set — the pipeline shares the app's Postgres")
|
|
return u
|
|
|
|
|
|
def ollama_host() -> str:
|
|
"""Reuse the app's resolver so the pipeline self-heals across the WSL host-flip."""
|
|
default = os.getenv("OLLAMA_HOST", "http://192.168.1.5:30068")
|
|
try:
|
|
from config import _resolve_ollama_host
|
|
return _resolve_ollama_host(default)
|
|
except Exception: # noqa: BLE001
|
|
return default
|
|
|
|
|
|
def watchlist() -> list[str]:
|
|
"""Tickers/names to bias Whisper's initial_prompt + focus extraction (§6, §12.4)."""
|
|
raw = os.getenv("YT_WATCHLIST", "")
|
|
return [w.strip() for w in raw.split(",") if w.strip()]
|