momentum devandClaude Opus 4.8
4a54b2adcc
A descriptive ingestion layer feeding the downstream reasoning model: monitors YouTube
playlists, transcribes new uploads, and extracts structured, verifiable CLAIM records
(with provenance + a mandatory verbatim quote). It records WHAT was claimed; it never
judges correctness or tradeability -- all judgment stays downstream, in one place.
Built per the operator's spec. Two clocks coordinated through Postgres (§2): a 30-min
detection poll + a paced concurrency-1 transcription/extraction worker. Reuses the app's
infra: shared Postgres (isolated `transcripts` schema), the app's Ollama for extraction
(free, on-LAN, private -- transcripts never leave the network), the config.py host resolver.
youtube_claims/schema.sql playlists / videos / claims (§7-8); claims.verbatim_quote NOT NULL
youtube_claims/db.py psycopg2; claim-by-atomic-status-flip under FOR UPDATE SKIP LOCKED
youtube_claims/detection.py playlistItems.list over urllib; dedupe by video_id (§5)
youtube_claims/extraction.py local Ollama -> descriptive claims; drops any claim lacking an
exact verbatim quote present in the source window (anti-hallucination)
youtube_claims/transcription.py yt-dlp -> faster-whisper (medium/cpu/int8, option A;
initial_prompt=watchlist) with youtube-transcript-api captions fallback
youtube_claims/worker.py paced drain; per-video state machine + retries/backoff
youtube_claims/run.py APScheduler poll + worker loop in one process; CLI for setup
VERIFIED LIVE: schema applied; playlist "Anatomy of a Trade" (Ross Cameron) added; poll
ingested all 12 videos (1 API unit); Ollama extraction pulled clean descriptive claims
(TC/DXST/CF) with exact verbatim quotes from a real transcript. Whisper transcription pending
ffmpeg (`apt install ffmpeg`, needs sudo) + the yt-dlp/faster-whisper deps (installing via uv).
Secrets (YOUTUBE_API_KEY) live in gitignored .env only. Whisper is option A (CPU); env-flippable
to option B (LAN GPU large-v3) later.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
61 lines
2.7 KiB
SQL
61 lines
2.7 KiB
SQL
-- YouTube claim-extraction pipeline schema (§7-8). Isolated in its own schema so it never
|
|
-- collides with the trading tables. {SCHEMA} is substituted at apply time (default: transcripts).
|
|
|
|
CREATE SCHEMA IF NOT EXISTS {SCHEMA};
|
|
|
|
-- config: which playlists to monitor
|
|
CREATE TABLE IF NOT EXISTS {SCHEMA}.playlists (
|
|
playlist_id TEXT PRIMARY KEY,
|
|
label TEXT,
|
|
content_type TEXT DEFAULT 'unknown', -- analysis / news / opinion / promo / unknown
|
|
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
|
added_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
-- videos: one row per detected upload; dedupe key is video_id (§5)
|
|
CREATE TABLE IF NOT EXISTS {SCHEMA}.videos (
|
|
video_id TEXT PRIMARY KEY,
|
|
playlist_id TEXT REFERENCES {SCHEMA}.playlists(playlist_id),
|
|
channel_id TEXT,
|
|
channel_name TEXT,
|
|
title TEXT,
|
|
content_type TEXT DEFAULT 'unknown',
|
|
published_at TIMESTAMPTZ,
|
|
detected_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
poll_latency_seconds INT,
|
|
status TEXT NOT NULL DEFAULT 'pending', -- pending|transcribing|extracting|done|failed
|
|
transcript_source TEXT, -- whisper|captions|NULL
|
|
whisper_model TEXT,
|
|
transcript_path TEXT,
|
|
processed_at TIMESTAMPTZ,
|
|
retry_count INT NOT NULL DEFAULT 0,
|
|
last_error TEXT
|
|
);
|
|
CREATE INDEX IF NOT EXISTS videos_status_idx ON {SCHEMA}.videos(status);
|
|
|
|
-- claims: the core deliverable (§8) — DESCRIPTIVE only, verbatim_quote mandatory
|
|
CREATE TABLE IF NOT EXISTS {SCHEMA}.claims (
|
|
claim_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
video_id TEXT NOT NULL REFERENCES {SCHEMA}.videos(video_id),
|
|
asset_ticker TEXT,
|
|
asset_name TEXT,
|
|
asset_class TEXT, -- equity|crypto|etf|index|commodity|fx|other
|
|
direction TEXT, -- bullish|bearish|neutral|mixed (of the CLAIM, not advice)
|
|
claim_text TEXT,
|
|
verbatim_quote TEXT NOT NULL, -- mandatory chain of custody back to source
|
|
timestamp_start NUMERIC,
|
|
timestamp_end NUMERIC,
|
|
stated_rationale TEXT,
|
|
stated_horizon TEXT,
|
|
extraction_confidence NUMERIC, -- 0-1 PARSE confidence, NOT truth confidence
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
CREATE INDEX IF NOT EXISTS claims_video_idx ON {SCHEMA}.claims(video_id);
|
|
CREATE INDEX IF NOT EXISTS claims_ticker_idx ON {SCHEMA}.claims(asset_ticker);
|
|
|
|
-- daily YouTube API quota counter (§5)
|
|
CREATE TABLE IF NOT EXISTS {SCHEMA}.api_quota (
|
|
day DATE PRIMARY KEY,
|
|
units_used INT NOT NULL DEFAULT 0
|
|
);
|