From 92488b72bc9b230b300161f512fa888a0c15390f Mon Sep 17 00:00:00 2001 From: momentum dev Date: Fri, 17 Jul 2026 09:14:22 -0500 Subject: [PATCH] =?UTF-8?q?youtube:=20YT=5FPLAYLIST=5FURLS=20=E2=80=94=20c?= =?UTF-8?q?omma-separated=20playlists=20via=20env,=20synced=20at=20boot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One playlist per add-playlist CLI call made adding a second playlist a chore. Now YT_PLAYLIST_URLS takes a comma-separated list of playlist URLs (list= param extracted) or bare ids; the transcripts service seeds the playlists table from it at boot (insert-if-missing — hand-set labels are never clobbered), and `python -m youtube_claims.run sync-playlists` does the same on demand. Documented in both compose files; 3 tests. Also: queue the two executor-surfaced defects (OrderFilledEvent side, /api/attribution env hardcode) + the flaky nightly regime build-features step in the plans index; regime_features 07-16 backfilled supervised. Co-Authored-By: Claude Fable 5 --- deploy/docker/truenas-compose.yml | 3 +++ plans/README.md | 6 +++++ tests/unit/test_youtube_playlists.py | 34 ++++++++++++++++++++++++++++ youtube_claims/config.py | 22 ++++++++++++++++++ youtube_claims/db.py | 12 ++++++++++ youtube_claims/run.py | 17 ++++++++++++++ 6 files changed, 94 insertions(+) create mode 100644 tests/unit/test_youtube_playlists.py diff --git a/deploy/docker/truenas-compose.yml b/deploy/docker/truenas-compose.yml index 9640baa..8e2939a 100644 --- a/deploy/docker/truenas-compose.yml +++ b/deploy/docker/truenas-compose.yml @@ -116,6 +116,9 @@ x-app: &app WATCHER_PRICE_MAX: '20' WATCHER_PRICE_MIN: '2' YOUTUBE_API_KEY: '' + # comma-separated playlist URLs (or bare ids); synced into the playlists + # table at transcripts-service boot — edit here to add/track more playlists + YT_PLAYLIST_URLS: '' volumes: - /mnt/volume/Apps/momentum/data:/app/data depends_on: diff --git a/plans/README.md b/plans/README.md index 11400fb..3b313d9 100644 --- a/plans/README.md +++ b/plans/README.md @@ -59,6 +59,12 @@ plan in this series (check the status table above). - Recommended UI execution order: 012 → 013 → 014 → 015. 014's Decision Feed panel half can land before 013; its chart-overlay half can't. 015 consumes replay.py as-is — any plan that changes exit-sim semantics invalidates its honesty claim and must re-verify it. - 015's data supply is the nightly setup_events build (plan 009's accrual) — a missed scheduler night is a missing day on /backtest; the page surfaces it via the "sessions replayed X of Y" line. +## Queued defects (surfaced by plan 013/015 executors — owner-confirmed 2026-07-17, plan files pending) + +- **OrderFilledEvent has no `side` field** (`storage/event_schema.py`; sole emit site `alpaca_paper/execution.py` passes none). /api/chart works around it by joining the sibling order_submitted row — fix at the emitter so every consumer gets it. Live-path change: needs its own plan + tests, do NOT drive-by fix. +- **GET /api/attribution 500s when DATABASE_URL unset** — `research/attribution.py:_connect()` hardcodes `os.environ["DATABASE_URL"]`, bypassing the storage facade's fallback chain everything else uses. Read-side, small. +- **Nightly scheduler's `regime build-features` step silently produced no row two nights running** (07-15 full-outage, 07-16 step-level: setups+SPY landed, regime row didn't — backfilled supervised both times). Pull the scheduler container's stdout at 20:00 ET tonight; suspect the `python -m research.regime` CLI's store bootstrap (`WATCHER_EVENT_DB_PATH` default) diverging from DATABASE_URL in-container. + ## Findings considered and rejected (do not re-audit) - **Retest/pullback entry logic, scale-out partials, close-above-trigger confirmation from folklore** — refuted in `docs/research/entry_conversion_research.md` (external evidence + in-house tests); the research doc's "explicit do-nots". diff --git a/tests/unit/test_youtube_playlists.py b/tests/unit/test_youtube_playlists.py new file mode 100644 index 0000000..9803997 --- /dev/null +++ b/tests/unit/test_youtube_playlists.py @@ -0,0 +1,34 @@ +"""YT_PLAYLIST_URLS parsing + boot sync — playlists come from ONE comma-separated +env var (URLs or bare ids) instead of one add-playlist CLI call each, so adding a +second playlist is editing one line of .env. Sync must never clobber a label the +owner set by hand (insert-if-missing, not upsert).""" + +from youtube_claims import config + + +def test_playlist_urls_parses_urls_and_bare_ids(monkeypatch): + monkeypatch.setenv( + "YT_PLAYLIST_URLS", + "https://www.youtube.com/playlist?list=PLabc123 , " + "https://youtube.com/watch?v=xyz&list=PLdef456&index=2, " + "PLghi789", + ) + assert config.playlist_urls() == ["PLabc123", "PLdef456", "PLghi789"] + + +def test_playlist_urls_empty_and_junk(monkeypatch): + monkeypatch.setenv("YT_PLAYLIST_URLS", " , https://youtube.com/watch?v=noList , ") + assert config.playlist_urls() == [] # a URL with no list= param is not a playlist + monkeypatch.delenv("YT_PLAYLIST_URLS", raising=False) + assert config.playlist_urls() == [] + + +def test_sync_env_playlists_inserts_without_clobbering(monkeypatch): + from youtube_claims import run as run_mod + + monkeypatch.setenv("YT_PLAYLIST_URLS", "PLnew1, https://www.youtube.com/playlist?list=PLnew2") + calls = [] + monkeypatch.setattr(run_mod.db, "ensure_playlist", lambda pid, label: calls.append((pid, label))) + n = run_mod.sync_env_playlists() + assert n == 2 + assert calls == [("PLnew1", "PLnew1"), ("PLnew2", "PLnew2")] diff --git a/youtube_claims/config.py b/youtube_claims/config.py index 656d703..a21897a 100644 --- a/youtube_claims/config.py +++ b/youtube_claims/config.py @@ -73,3 +73,25 @@ def watchlist() -> list[str]: """Tickers/names to bias Whisper's initial_prompt + focus extraction (§6, §12.4).""" raw = os.getenv("YT_WATCHLIST", "") return [w.strip() for w in raw.split(",") if w.strip()] + + +def playlist_urls() -> list[str]: + """Playlist IDs from YT_PLAYLIST_URLS — a comma-separated list of playlist + URLs or bare ids, so adding another playlist is a one-line .env edit + instead of an add-playlist CLI call per id. A URL yields its ``list=`` + query param; a token without ``list=`` and without ``://`` is taken as a + bare id; a URL lacking ``list=`` is not a playlist and is dropped.""" + from urllib.parse import parse_qs, urlparse + + out = [] + for tok in os.getenv("YT_PLAYLIST_URLS", "").split(","): + tok = tok.strip() + if not tok: + continue + if "://" in tok or tok.startswith("www."): + pid = (parse_qs(urlparse(tok).query).get("list") or [None])[0] + if pid: + out.append(pid) + else: + out.append(tok) + return out diff --git a/youtube_claims/db.py b/youtube_claims/db.py index bc36245..3be10b5 100644 --- a/youtube_claims/db.py +++ b/youtube_claims/db.py @@ -37,6 +37,18 @@ def apply_schema() -> None: # --- playlists --- +def ensure_playlist(playlist_id: str, label: str, content_type: str = "unknown") -> None: + """Insert-if-missing (env-sync path): NEVER overwrites a label/content_type + the owner set by hand — that's upsert_playlist's job.""" + with connect() as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO playlists (playlist_id, label, content_type) VALUES (%s,%s,%s) " + "ON CONFLICT (playlist_id) DO NOTHING", + (playlist_id, label, content_type)) + conn.commit() + + def upsert_playlist(playlist_id: str, label: str, content_type: str = "unknown") -> None: with connect() as conn: with conn.cursor() as cur: diff --git a/youtube_claims/run.py b/youtube_claims/run.py index 5f577eb..d6c8043 100644 --- a/youtube_claims/run.py +++ b/youtube_claims/run.py @@ -26,9 +26,22 @@ def _worker_loop(stop: threading.Event) -> None: stop.wait(30) +def sync_env_playlists() -> int: + """Seed the playlists table from YT_PLAYLIST_URLS (comma-separated URLs or + ids). Insert-if-missing only — hand-set labels survive; returns how many + ids were synced. Runs at service boot and via the sync-playlists CLI.""" + pids = config.playlist_urls() + for pid in pids: + db.ensure_playlist(pid, pid) + return len(pids) + + def run_service() -> None: from apscheduler.schedulers.background import BackgroundScheduler db.apply_schema() + n = sync_env_playlists() + if n: + print(f"[service] synced {n} playlist(s) from YT_PLAYLIST_URLS") def _poll(): try: @@ -56,6 +69,7 @@ def main() -> None: ap = argparse.ArgumentParser(description="YouTube -> transcript -> claims pipeline") sub = ap.add_subparsers(dest="cmd", required=True) sub.add_parser("apply-schema") + sub.add_parser("sync-playlists") # seed the playlists table from YT_PLAYLIST_URLS a = sub.add_parser("add-playlist") a.add_argument("playlist_id") a.add_argument("--label", default="") @@ -69,6 +83,9 @@ def main() -> None: if args.cmd == "apply-schema": db.apply_schema() print(f"schema '{config.PG_SCHEMA}' applied") + elif args.cmd == "sync-playlists": + db.apply_schema() + print(f"synced {sync_env_playlists()} playlist(s) from YT_PLAYLIST_URLS") elif args.cmd == "add-playlist": db.apply_schema() db.upsert_playlist(args.playlist_id, args.label, args.content_type)