From 9b4c8e5da5459b63ecb5da15178d0d5d81b22e36 Mon Sep 17 00:00:00 2001 From: momentum dev Date: Sun, 13 Sep 2026 12:00:40 -0500 Subject: [PATCH] fix: news ranking silently switched itself off for 40 days step_arm ranks gappers by catalyst, but the enrichment feeding it stopped on 2026-08-03 -- two days before the first real-money fill. raw_news_items kept ingesting ~45k items/day throughout; news_events and news_catalyst_cache have no rows after 2026-08-03. So every live trade to date was ranked with the news multiplier pinned at 1.0. Three faults, the third of which hid the other two: 1. Ollama was configured at a host/port with nothing listening, running a model that was never pulled. 2. _cat_mult returned a flat 1.0 for a symbol with no LLM advisory instead of falling back to the binary headline boost. With the scorer enabled and the cache empty that branch is reached for EVERY symbol, so turning on the graded ranker silently turned off the simple one it was meant to improve. Fall back properly, so news ranking degrades instead of disappearing. 3. enrich_recent_news never raises -- it reports failures in a returned counts dict, and enrich_loop discarded that return value. Every 120s for 40 days it received {enriched: 0, errors: 50} and threw it away: no log line, no event, nothing to distinguish a dead pipeline from an idle one. Consume the counts and emit a module_tick naming host and model, on the first failed pass and hourly thereafter. Fault 3 is the one worth fixing; 1 and 2 are its first casualties. No claim that news ranking pays: a binary "has a headline" flag tests mildly NEGATIVE across 1,030 screened symbol-days (7.8% vs 10.4% ran >=50%), and the graded classifier only overlaps bar data for five days, too few to separate anything. Restoring the pipeline makes it measurable; it does not make it trusted. Re-measure catalyst type and sentiment against realised runs before giving the multiplier more authority. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01A2LepdJfPCfK5Q8uznjFBM --- run_live_paper.py | 54 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/run_live_paper.py b/run_live_paper.py index a020ef4..954f7f0 100644 --- a/run_live_paper.py +++ b/run_live_paper.py @@ -923,7 +923,16 @@ def main(argv: list[str] | None = None) -> int: from research.ingestion.news_enrichment import catalyst_score as _cscore adv = cat_map.get((sym or "").upper()) if not adv: - return 1.0 + # No LLM advisory for THIS symbol -> fall back to the binary + # headline boost instead of dropping news from the ranking. + # Returning a flat 1.0 here meant that whenever the enrichment + # was down, cat_map was empty for EVERY symbol and news stopped + # influencing the rank at all — silently. That is exactly what + # happened from 2026-08-03 (Ollama host/model misconfigured) + # through 2026-09-13: 40 days of live trading, two days of which + # preceded the first real-money fill, with zero news input to the + # ranking and no error anywhere to show for it. + return news_boost if has_headline else 1.0 if adv.get("is_dilutive") and float(adv.get("conviction") or 0.0) >= ( ollama_cfg.dilution_veto_min_conviction): return dilutive_penalty @@ -1340,11 +1349,52 @@ def main(argv: list[str] | None = None) -> int: except Exception as exc: # noqa: BLE001 print(f"[enrich] no DB connection, enrichment off: {exc}", flush=True) return + # enrich_recent_news NEVER raises — it reports via its counts dict. The + # old loop discarded that return value, so a totally dead enrichment + # (Ollama unreachable, model absent) looked identical to a healthy idle + # one and ran unnoticed for 40 days. Emit a module_tick so the failure is + # visible to the monitoring board and queryable in the event store. + consecutive_bad = 0 while not stop_event.wait(float(ollama_cfg.enrichment_interval_seconds)): try: - enrich_recent_news(econ, ollama_cfg) + counts = enrich_recent_news(econ, ollama_cfg) or {} except Exception as exc: # noqa: BLE001 + counts = {"enriched": 0, "skipped": 0, "errors": -1} print(f"[enrich] error: {exc}", flush=True) + errors = int(counts.get("errors") or 0) + enriched = int(counts.get("enriched") or 0) + # "bad" = the pass tried to classify and every attempt failed. + bad = errors != 0 and enriched == 0 + consecutive_bad = consecutive_bad + 1 if bad else 0 + if bad: + print(f"[enrich] {errors} failures, 0 enriched " + f"(host={ollama_cfg.host} model={ollama_cfg.model})", flush=True) + # Report on the first bad pass and then hourly, so a persistent + # outage stays visible without flooding the store every interval. + period = max(1, int(3600 / max(1.0, float(ollama_cfg.enrichment_interval_seconds)))) + if enriched or (bad and (consecutive_bad == 1 or consecutive_bad % period == 0)): + rt["store"].emit(ModuleTickEvent( + timestamp=datetime.now(), + mode=EventMode.PAPER, + correlation_id=rt["session_id"], + message=(f"news enrichment DOWN: {errors} failures, 0 enriched " + f"({consecutive_bad} passes)" if bad + else f"news enrichment: +{enriched} classified"), + module="enrichment", + stage="classify", + duration_ms=0.0, + input_count=enriched + errors, + output_count=enriched, + metrics={"enriched": enriched, "errors": errors, + "skipped": int(counts.get("skipped") or 0), + "host": ollama_cfg.host, "model": ollama_cfg.model, + "consecutive_failed_passes": consecutive_bad}, + errors=([{"error": "news catalyst enrichment produced nothing — " + f"check Ollama at {ollama_cfg.host} and that model " + f"'{ollama_cfg.model}' is pulled. News ranking is " + "running on the binary headline boost until it recovers."}] + if bad else []), + )) scheduler = Scheduler() scheduler.add("news", step_news, float(os.environ.get("ALPACA_NEWS_INTERVAL_SECONDS", "120")),