diff --git a/run_live_paper.py b/run_live_paper.py index a020ef4..954f7f0 100644 --- a/run_live_paper.py +++ b/run_live_paper.py @@ -923,7 +923,16 @@ def main(argv: list[str] | None = None) -> int: from research.ingestion.news_enrichment import catalyst_score as _cscore adv = cat_map.get((sym or "").upper()) if not adv: - return 1.0 + # No LLM advisory for THIS symbol -> fall back to the binary + # headline boost instead of dropping news from the ranking. + # Returning a flat 1.0 here meant that whenever the enrichment + # was down, cat_map was empty for EVERY symbol and news stopped + # influencing the rank at all — silently. That is exactly what + # happened from 2026-08-03 (Ollama host/model misconfigured) + # through 2026-09-13: 40 days of live trading, two days of which + # preceded the first real-money fill, with zero news input to the + # ranking and no error anywhere to show for it. + return news_boost if has_headline else 1.0 if adv.get("is_dilutive") and float(adv.get("conviction") or 0.0) >= ( ollama_cfg.dilution_veto_min_conviction): return dilutive_penalty @@ -1340,11 +1349,52 @@ def main(argv: list[str] | None = None) -> int: except Exception as exc: # noqa: BLE001 print(f"[enrich] no DB connection, enrichment off: {exc}", flush=True) return + # enrich_recent_news NEVER raises — it reports via its counts dict. The + # old loop discarded that return value, so a totally dead enrichment + # (Ollama unreachable, model absent) looked identical to a healthy idle + # one and ran unnoticed for 40 days. Emit a module_tick so the failure is + # visible to the monitoring board and queryable in the event store. + consecutive_bad = 0 while not stop_event.wait(float(ollama_cfg.enrichment_interval_seconds)): try: - enrich_recent_news(econ, ollama_cfg) + counts = enrich_recent_news(econ, ollama_cfg) or {} except Exception as exc: # noqa: BLE001 + counts = {"enriched": 0, "skipped": 0, "errors": -1} print(f"[enrich] error: {exc}", flush=True) + errors = int(counts.get("errors") or 0) + enriched = int(counts.get("enriched") or 0) + # "bad" = the pass tried to classify and every attempt failed. + bad = errors != 0 and enriched == 0 + consecutive_bad = consecutive_bad + 1 if bad else 0 + if bad: + print(f"[enrich] {errors} failures, 0 enriched " + f"(host={ollama_cfg.host} model={ollama_cfg.model})", flush=True) + # Report on the first bad pass and then hourly, so a persistent + # outage stays visible without flooding the store every interval. + period = max(1, int(3600 / max(1.0, float(ollama_cfg.enrichment_interval_seconds)))) + if enriched or (bad and (consecutive_bad == 1 or consecutive_bad % period == 0)): + rt["store"].emit(ModuleTickEvent( + timestamp=datetime.now(), + mode=EventMode.PAPER, + correlation_id=rt["session_id"], + message=(f"news enrichment DOWN: {errors} failures, 0 enriched " + f"({consecutive_bad} passes)" if bad + else f"news enrichment: +{enriched} classified"), + module="enrichment", + stage="classify", + duration_ms=0.0, + input_count=enriched + errors, + output_count=enriched, + metrics={"enriched": enriched, "errors": errors, + "skipped": int(counts.get("skipped") or 0), + "host": ollama_cfg.host, "model": ollama_cfg.model, + "consecutive_failed_passes": consecutive_bad}, + errors=([{"error": "news catalyst enrichment produced nothing — " + f"check Ollama at {ollama_cfg.host} and that model " + f"'{ollama_cfg.model}' is pulled. News ranking is " + "running on the binary headline boost until it recovers."}] + if bad else []), + )) scheduler = Scheduler() scheduler.add("news", step_news, float(os.environ.get("ALPACA_NEWS_INTERVAL_SECONDS", "120")),