diff --git a/catalyst_eval.py b/catalyst_eval.py new file mode 100644 index 0000000..36099dc --- /dev/null +++ b/catalyst_eval.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""A/B a local Ollama model on catalyst-labeling accuracy against a fixed set. + +Usage: + PYTHONPATH=. python catalyst_eval.py # default model + PYTHONPATH=. python catalyst_eval.py --model qwen3:14b # A/B a bigger model + PYTHONPATH=. python catalyst_eval.py --no-schema # plain JSON format + +Runs the hand-labeled fixture in ``research/ingestion/catalyst_eval.py`` through +the live model and prints type accuracy, is_dilutive precision/recall (the veto's +basis), and the type mistakes. Pick the model that wins here for OLLAMA_MODEL. +""" + +import argparse + +from research.ingestion.catalyst_eval import format_report, run_eval + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--model", default="qwen2.5:7b-instruct") + p.add_argument("--host", default="http://localhost:11434") + p.add_argument("--timeout", type=int, default=30) + p.add_argument("--no-schema", action="store_true", + help="send plain 'format: json' instead of the JSON schema") + args = p.parse_args(argv) + + metrics, _ = run_eval( + host=args.host, model=args.model, timeout=args.timeout, + use_schema=not args.no_schema, + ) + print(format_report(metrics, model=args.model)) + if metrics["scored"] == 0: + print(f"\nNo predictions scored — is Ollama running at {args.host} " + f"with model '{args.model}' pulled?") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/ingestion/catalyst_eval.py b/research/ingestion/catalyst_eval.py new file mode 100644 index 0000000..62fc24d --- /dev/null +++ b/research/ingestion/catalyst_eval.py @@ -0,0 +1,165 @@ +"""Label-quality evaluation for the Ollama catalyst classifier. + +Runs a fixed, hand-labeled set of representative small-cap headlines through +``classify_headline`` and scores the model's labels against the ground truth. +This is the objective basis for (a) confirming the few-shot / schema prompt +helps and (b) A/B-ing models (``qwen3:14b`` vs ``qwen2.5:7b-instruct``) on the +SAME inputs. + +The scoring (`score_predictions`) is PURE — no Ollama, no DB — so it is unit +tested directly. ``run_eval`` is the thin live wrapper that calls the model. + +The single most decision-relevant metric is **is_dilutive precision/recall**: +that flag is what the Phase-2 dilution veto acts on, so a false negative lets a +dilutive trap through and a false positive vetoes a good name. +""" + +from __future__ import annotations + +from research.ingestion.news_enrichment import CatalystAnalysis, classify_headline + +# A representative, hand-labeled fixture spanning the catalyst vocabulary. +# ``sentiment_sign`` is the EXPECTED sign for the stock: +1 bullish, -1 bearish, +# 0 neutral/ambiguous. It is scored with a deadband so near-zero reads pass. +FIXTURE: list[dict] = [ + {"headline": "Acme Bio Announces $30 Million Registered Direct Offering Priced At-The-Market", + "catalyst_type": "offering_dilution", "is_dilutive": True, "sentiment_sign": -1}, + {"headline": "NanoTech Prices Underwritten Public Offering of 8 Million Shares at $2.10", + "catalyst_type": "offering_dilution", "is_dilutive": True, "sentiment_sign": -1}, + {"headline": "QuantumX Files Form S-3 Shelf Registration for Up to $200 Million", + "catalyst_type": "offering_dilution", "is_dilutive": True, "sentiment_sign": -1}, + {"headline": "CureGen Receives FDA Approval for Its Lead Oncology Therapy", + "catalyst_type": "fda_approval", "is_dilutive": False, "sentiment_sign": 1}, + {"headline": "TrialCo Reports Positive Topline Phase 3 Results Meeting Primary Endpoint", + "catalyst_type": "clinical_trial", "is_dilutive": False, "sentiment_sign": 1}, + {"headline": "BioX Drug Fails to Meet Primary Endpoint in Phase 3 Study", + "catalyst_type": "clinical_trial", "is_dilutive": False, "sentiment_sign": -1}, + {"headline": "RetailCo Reports Q4 Revenue Up 35%, Beating Analyst Estimates", + "catalyst_type": "earnings", "is_dilutive": False, "sentiment_sign": 1}, + {"headline": "WidgetCorp Misses Q2 Earnings as Margins Compress", + "catalyst_type": "earnings", "is_dilutive": False, "sentiment_sign": -1}, + {"headline": "TargetCo to Be Acquired by MegaCorp for $12.00 Per Share in Cash", + "catalyst_type": "ma_acquisition", "is_dilutive": False, "sentiment_sign": 1}, + {"headline": "TechCo Signs Multi-Year Supply Agreement With a Major Automaker", + "catalyst_type": "partnership_contract", "is_dilutive": False, "sentiment_sign": 1}, + {"headline": "SoftCo Raises Full-Year Revenue Guidance Above Consensus", + "catalyst_type": "guidance_update", "is_dilutive": False, "sentiment_sign": 1}, + {"headline": "PharmaCo Receives FDA Complete Response Letter for Its Drug Application", + "catalyst_type": "regulatory", "is_dilutive": False, "sentiment_sign": -1}, + {"headline": "MicroCap Announces 1-for-20 Reverse Stock Split to Regain Listing Compliance", + "catalyst_type": "stock_split_reverse", "is_dilutive": False, "sentiment_sign": -1}, + {"headline": "GadgetCo Upgraded to Buy at a Major Bank, Price Target Raised", + "catalyst_type": "analyst_rating", "is_dilutive": False, "sentiment_sign": 1}, + {"headline": "BoardCo Authorizes $100 Million Share Repurchase Program", + "catalyst_type": "insider_buyback", "is_dilutive": False, "sentiment_sign": 1}, + {"headline": "CEO of HoldingCo to Present at an Upcoming Investor Conference", + "catalyst_type": "none", "is_dilutive": False, "sentiment_sign": 0}, +] + + +def _sign(value, deadband: float = 0.15) -> int: + """-1 / 0 / +1 with a neutral deadband so near-zero reads count as neutral.""" + if value is None: + return 0 + if value > deadband: + return 1 + if value < -deadband: + return -1 + return 0 + + +def _field(pred, name): + """Read a field off a CatalystAnalysis or a plain dict (keeps scoring pure).""" + if pred is None: + return None + if isinstance(pred, dict): + return pred.get(name) + return getattr(pred, name, None) + + +def score_predictions(cases: list[tuple]) -> dict: + """Score (expected_dict, predicted) pairs. PURE — no Ollama/DB. + + ``predicted`` may be a ``CatalystAnalysis``, a dict, or ``None`` (model + failed → counted as an error, not scored). Returns accuracy for the + catalyst type and sentiment sign, plus precision/recall/F1 for the + is_dilutive flag (the veto's basis) and the list of type mistakes.""" + n = len(cases) + type_hits = sent_hits = errors = scored = 0 + tp = fp = fn = tn = 0 + confusion: list[dict] = [] + for exp, pred in cases: + if pred is None: + errors += 1 + continue + scored += 1 + p_type = _field(pred, "catalyst_type") + if p_type == exp["catalyst_type"]: + type_hits += 1 + else: + confusion.append({ + "headline": exp.get("headline", ""), + "expected": exp["catalyst_type"], "predicted": p_type, + }) + e_dil, p_dil = bool(exp["is_dilutive"]), bool(_field(pred, "is_dilutive")) + if p_dil and e_dil: + tp += 1 + elif p_dil and not e_dil: + fp += 1 + elif not p_dil and e_dil: + fn += 1 + else: + tn += 1 + if "sentiment_sign" in exp and _sign(_field(pred, "sentiment")) == exp["sentiment_sign"]: + sent_hits += 1 + + def _ratio(a, b): + return round(a / b, 4) if b else None + + precision = _ratio(tp, tp + fp) + recall = _ratio(tp, tp + fn) + f1 = (round(2 * precision * recall / (precision + recall), 4) + if precision and recall else None) + return { + "n": n, "scored": scored, "errors": errors, + "type_accuracy": _ratio(type_hits, scored), + "sentiment_sign_accuracy": _ratio(sent_hits, scored), + "dilutive": {"tp": tp, "fp": fp, "fn": fn, "tn": tn, + "precision": precision, "recall": recall, "f1": f1}, + "confusion": confusion, + } + + +def run_eval(cases: list[dict] | None = None, **kwargs) -> tuple[dict, list[tuple]]: + """Classify each fixture headline live, then score. Returns (metrics, pairs). + + ``kwargs`` pass straight through to ``classify_headline`` (host, model, + timeout, use_schema, ...). A model failure on a case yields ``None`` for + that prediction (counted as an error).""" + cases = cases if cases is not None else FIXTURE + pairs = [ + (c, classify_headline(c["headline"], c.get("snippet", ""), + c.get("tickers", ""), **kwargs)) + for c in cases + ] + return score_predictions(pairs), pairs + + +def format_report(metrics: dict, model: str = "") -> str: + """Human-readable metrics block for the CLI.""" + d = metrics["dilutive"] + def pct(x): + return f"{x * 100:5.1f}%" if isinstance(x, (int, float)) else " n/a" + lines = [ + f"Catalyst label eval — model={model or '?'}", + f" cases: {metrics['n']} scored: {metrics['scored']} errors: {metrics['errors']}", + f" catalyst_type accuracy : {pct(metrics['type_accuracy'])}", + f" sentiment-sign accuracy: {pct(metrics['sentiment_sign_accuracy'])}", + f" is_dilutive P={pct(d['precision'])} R={pct(d['recall'])} F1={pct(d['f1'])}" + f" (tp={d['tp']} fp={d['fp']} fn={d['fn']} tn={d['tn']})", + ] + if metrics["confusion"]: + lines.append(" type misses:") + for m in metrics["confusion"]: + lines.append(f" [{m['expected']} -> {m['predicted']}] {m['headline'][:70]}") + return "\n".join(lines) diff --git a/research/ingestion/news_enrichment.py b/research/ingestion/news_enrichment.py index 52c4ed7..c2f3c8a 100644 --- a/research/ingestion/news_enrichment.py +++ b/research/ingestion/news_enrichment.py @@ -41,6 +41,24 @@ CATALYST_TYPES = { "none", } +# JSON schema handed to Ollama's ``format`` field so the model is constrained at +# decode time (enum-locked catalyst_type, typed numbers/bool) — far fewer malformed +# rows than free-form ``"format": "json"``. Built from CATALYST_TYPES so the two +# never drift. Current Ollama enforces this grammar during generation. +CATALYST_SCHEMA = { + "type": "object", + "properties": { + "catalyst_type": {"type": "string", "enum": sorted(CATALYST_TYPES)}, + "sentiment": {"type": "number", "minimum": -1.0, "maximum": 1.0}, + "conviction": {"type": "number", "minimum": 0.0, "maximum": 1.0}, + "is_dilutive": {"type": "boolean"}, + "rationale": {"type": "string"}, + }, + "required": [ + "catalyst_type", "sentiment", "conviction", "is_dilutive", "rationale", + ], +} + # Cheap stdlib floor for the dilution veto so it still works when Ollama is down. # The LLM CONFIRMS dilution; this regex is the safety net, not the primary signal. _DILUTIVE_RE = re.compile( @@ -95,6 +113,20 @@ def build_prompt(headline: str, snippet: str = "", tickers: str = "") -> str: ' "is_dilutive": true ONLY for confirmed share dilution (offering, ATM, ' "registered direct, warrant, shelf takedown), else false\n" ' "rationale": one short sentence (max ~200 chars)\n\n' + "Examples:\n" + 'Headline: XYZ Therapeutics Announces $50 Million Registered Direct Offering Priced At-The-Market\n' + '{"catalyst_type": "offering_dilution", "sentiment": -0.8, "conviction": 0.9, ' + '"is_dilutive": true, "rationale": "Registered direct ATM offering dilutes ' + 'shareholders; bearish for the stock."}\n' + 'Headline: ABC Pharma Receives FDA Approval for Lead Drug Candidate\n' + '{"catalyst_type": "fda_approval", "sentiment": 0.9, "conviction": 0.9, ' + '"is_dilutive": false, "rationale": "FDA approval is a major de-risking ' + 'catalyst; strongly bullish."}\n' + 'Headline: QRS Inc. Reports Q3 Revenue Up 40%, Beats Analyst Estimates\n' + '{"catalyst_type": "earnings", "sentiment": 0.7, "conviction": 0.75, ' + '"is_dilutive": false, "rationale": "Earnings beat with strong revenue ' + 'growth; bullish."}\n\n' + "Now classify this news:\n" + ctx ) @@ -143,11 +175,16 @@ def classify_headline( timeout: int = 30, temperature: float = 0.3, max_tokens: int = 256, + use_schema: bool = True, ) -> CatalystAnalysis | None: - """Ask a local Ollama model to classify a headline. None on any failure.""" + """Ask a local Ollama model to classify a headline. None on any failure. + + ``use_schema`` (default) constrains decoding with ``CATALYST_SCHEMA``; set it + False to fall back to plain ``"format": "json"`` (e.g. older Ollama).""" if not (headline or snippet): return None prompt = build_prompt(headline, snippet, tickers) + fmt = CATALYST_SCHEMA if use_schema else "json" try: req = urllib.request.Request( f"{host}/api/generate", @@ -156,7 +193,7 @@ def classify_headline( "model": model, "prompt": prompt, "stream": False, - "format": "json", + "format": fmt, "options": { "temperature": temperature, "num_predict": max_tokens, diff --git a/tests/unit/research/test_catalyst_eval.py b/tests/unit/research/test_catalyst_eval.py new file mode 100644 index 0000000..19fc236 --- /dev/null +++ b/tests/unit/research/test_catalyst_eval.py @@ -0,0 +1,97 @@ +"""Tests for the catalyst label-quality eval. + +``score_predictions`` is pure (no Ollama/DB); ``run_eval`` is exercised over a +mocked classifier so the harness logic is covered without a live model. +""" + +from research.ingestion import catalyst_eval as ce +from research.ingestion.news_enrichment import CatalystAnalysis + + +def _exp(ctype, dil, sign, headline="h"): + return {"headline": headline, "catalyst_type": ctype, + "is_dilutive": dil, "sentiment_sign": sign} + + +def test_score_predictions_perfect(): + cases = [ + (_exp("fda_approval", False, 1), CatalystAnalysis("fda_approval", 0.9, 0.9, False)), + (_exp("offering_dilution", True, -1), + CatalystAnalysis("offering_dilution", -0.8, 0.9, True)), + ] + m = ce.score_predictions(cases) + assert m["scored"] == 2 and m["errors"] == 0 + assert m["type_accuracy"] == 1.0 + assert m["sentiment_sign_accuracy"] == 1.0 + assert m["dilutive"]["precision"] == 1.0 and m["dilutive"]["recall"] == 1.0 + assert m["confusion"] == [] + + +def test_score_predictions_counts_none_as_error_not_scored(): + cases = [ + (_exp("earnings", False, 1), None), + (_exp("earnings", False, 1), CatalystAnalysis("earnings", 0.7, 0.7, False)), + ] + m = ce.score_predictions(cases) + assert m["n"] == 2 and m["scored"] == 1 and m["errors"] == 1 + assert m["type_accuracy"] == 1.0 # only the scored one counts + + +def test_score_predictions_dilutive_precision_recall(): + cases = [ + # false negative: real dilution the model missed + (_exp("offering_dilution", True, -1), CatalystAnalysis("other", 0.0, 0.3, False)), + # false positive: model flagged dilution that isn't + (_exp("earnings", False, 1), CatalystAnalysis("earnings", 0.5, 0.6, True)), + # true positive + (_exp("offering_dilution", True, -1), + CatalystAnalysis("offering_dilution", -0.7, 0.8, True)), + # true negative + (_exp("fda_approval", False, 1), CatalystAnalysis("fda_approval", 0.9, 0.9, False)), + ] + d = ce.score_predictions(cases)["dilutive"] + assert (d["tp"], d["fp"], d["fn"], d["tn"]) == (1, 1, 1, 1) + assert d["precision"] == 0.5 and d["recall"] == 0.5 + + +def test_score_predictions_records_type_misses(): + cases = [(_exp("guidance_update", False, 1, "Raises guidance"), + CatalystAnalysis("earnings", 0.4, 0.5, False))] + m = ce.score_predictions(cases) + assert m["type_accuracy"] == 0.0 + assert m["confusion"][0]["expected"] == "guidance_update" + assert m["confusion"][0]["predicted"] == "earnings" + + +def test_sentiment_sign_deadband(): + # predicted sentiment within the deadband counts as neutral + cases = [(_exp("none", False, 0), CatalystAnalysis("none", 0.05, 0.1, False))] + assert ce.score_predictions(cases)["sentiment_sign_accuracy"] == 1.0 + + +def test_run_eval_over_mocked_classifier(monkeypatch): + # a "perfect" model that echoes each fixture's expected labels + def fake_classify(headline, snippet="", tickers="", **kwargs): + for c in ce.FIXTURE: + if c["headline"] == headline: + sign = c["sentiment_sign"] + return CatalystAnalysis( + catalyst_type=c["catalyst_type"], + sentiment=0.8 * sign, conviction=0.8, + is_dilutive=c["is_dilutive"], + ) + return None + + monkeypatch.setattr(ce, "classify_headline", fake_classify) + metrics, pairs = ce.run_eval() + assert len(pairs) == len(ce.FIXTURE) + assert metrics["errors"] == 0 + assert metrics["type_accuracy"] == 1.0 + assert metrics["dilutive"]["precision"] == 1.0 + assert metrics["dilutive"]["recall"] == 1.0 + + +def test_format_report_runs(): + cases = [(_exp("earnings", False, 1), CatalystAnalysis("earnings", 0.6, 0.6, False))] + out = ce.format_report(ce.score_predictions(cases), model="m") + assert "catalyst_type accuracy" in out and "is_dilutive" in out diff --git a/tests/unit/research/test_catalyst_validation.py b/tests/unit/research/test_catalyst_validation.py new file mode 100644 index 0000000..4cb6017 --- /dev/null +++ b/tests/unit/research/test_catalyst_validation.py @@ -0,0 +1,79 @@ +"""Tests for the catalyst→outcome validation harness (pure aggregation only).""" + +import validate_catalysts as vc + + +def _row(otc, oth, has_cat, dil=None, score=None): + return {"open_to_close_pct": otc, "open_to_high_pct": oth, + "has_catalyst": has_cat, "is_dilutive": dil, "catalyst_score": score} + + +def test_outcome_correlation_counts_and_coverage(): + rows = [ + _row(-5.0, 2.0, True, dil=True, score=0.1), + _row(8.0, 12.0, True, dil=False, score=0.8), + _row(1.0, 3.0, False), # no catalyst → excluded from splits + ] + s = vc.outcome_correlation(rows) + assert s["total_gappers"] == 3 + assert s["with_catalyst"] == 2 + assert s["dilutive"]["n"] == 1 + assert s["non_dilutive"]["n"] == 1 + + +def test_outcome_correlation_dilution_split_means(): + rows = [ + _row(-4.0, 1.0, True, dil=True), + _row(-6.0, 1.0, True, dil=True), + _row(10.0, 15.0, True, dil=False), + ] + s = vc.outcome_correlation(rows) + assert s["dilutive"]["mean_open_to_close_pct"] == -5.0 # (-4 + -6)/2 + assert s["non_dilutive"]["mean_open_to_close_pct"] == 10.0 + + +def test_outcome_correlation_score_buckets(): + rows = [ + _row(0.0, 1.0, True, dil=False, score=0.1), # low + _row(0.0, 1.0, True, dil=False, score=0.5), # mid + _row(0.0, 1.0, True, dil=False, score=0.9), # high + _row(0.0, 1.0, True, dil=False, score=None), # none + ] + buckets = vc.outcome_correlation(rows)["by_score_bucket"] + assert set(buckets) == {"low", "mid", "high", "none"} + assert all(b["n"] == 1 for b in buckets.values()) + + +def test_outcome_correlation_empty_when_no_catalysts(): + rows = [_row(3.0, 5.0, False), _row(-2.0, 1.0, False)] + s = vc.outcome_correlation(rows) + assert s["with_catalyst"] == 0 + assert s["dilutive"]["n"] == 0 and s["non_dilutive"]["n"] == 0 + # report renders the "run Phase 1 live first" guidance, no crash + assert "run" in vc._format_report(s, n_sessions=2).lower() + + +def test_session_outcome_basic(): + import pandas as pd + bars = pd.DataFrame({ + "open": [10.0, 11.0, 12.0], + "high": [10.5, 13.0, 12.5], + "low": [9.8, 10.9, 11.5], + "close": [10.4, 12.0, 11.0], + }) + out = vc._session_outcome(bars) + assert out["open_to_close_pct"] == 10.0 # (11.0-10.0)/10.0*100 + assert out["open_to_high_pct"] == 30.0 # (13.0-10.0)/10.0*100 + + +def test_session_outcome_empty(): + import pandas as pd + out = vc._session_outcome(pd.DataFrame()) + assert out == {"open_to_close_pct": None, "open_to_high_pct": None} + + +def test_bucket_thresholds(): + assert vc._bucket(None) == "none" + assert vc._bucket(0.2) == "low" + assert vc._bucket(0.5) == "mid" + assert vc._bucket(0.9) == "high" diff --git a/tests/unit/research/test_news_enrichment.py b/tests/unit/research/test_news_enrichment.py index 3936b76..3508a19 100644 --- a/tests/unit/research/test_news_enrichment.py +++ b/tests/unit/research/test_news_enrichment.py @@ -52,6 +52,42 @@ def test_classify_headline_happy_path(monkeypatch): assert a.is_dilutive is False and a.rationale == "FDA nod" +def test_classify_headline_sends_schema_by_default(monkeypatch): + captured = {} + + def capture(req, *a, **k): + captured["body"] = json.loads(req.data.decode()) + return _ollama_reply({ + "catalyst_type": "earnings", "sentiment": 0.5, + "conviction": 0.6, "is_dilutive": False, "rationale": "beat", + }) + + monkeypatch.setattr(ne.urllib.request, "urlopen", capture) + ne.classify_headline("QRS beats estimates", tickers="QRS") + fmt = captured["body"]["format"] + # schema-constrained decoding: format is the JSON schema, enum-locked + assert isinstance(fmt, dict) + assert fmt["properties"]["catalyst_type"]["enum"] == sorted(ne.CATALYST_TYPES) + assert set(fmt["required"]) == { + "catalyst_type", "sentiment", "conviction", "is_dilutive", "rationale", + } + + +def test_classify_headline_use_schema_false_sends_plain_json(monkeypatch): + captured = {} + + def capture(req, *a, **k): + captured["body"] = json.loads(req.data.decode()) + return _ollama_reply({ + "catalyst_type": "other", "sentiment": 0.0, + "conviction": 0.0, "is_dilutive": False, "rationale": "", + }) + + monkeypatch.setattr(ne.urllib.request, "urlopen", capture) + ne.classify_headline("anything", use_schema=False) + assert captured["body"]["format"] == "json" + + def test_classify_headline_network_down_returns_none(monkeypatch): def boom(*a, **k): raise OSError("connection refused") diff --git a/validate_catalysts.py b/validate_catalysts.py new file mode 100644 index 0000000..93cf945 --- /dev/null +++ b/validate_catalysts.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Validate catalyst labels against realized gapper outcomes — BEFORE automating. + +For every stored gapper session that ALSO has an enriched news catalyst, this +joins the LLM's label (catalyst_type / sentiment / conviction / is_dilutive) to +the day's realized move (open->close % and open->high %) and reports whether the +labels actually separate winners from losers: + + * do is_dilutive-flagged names fade (lower open->close) vs the rest? + * do higher bullish catalyst_score names run further? + +Run this before flipping NEWS_DILUTION_VETO_ENABLED / NEWS_CATALYST_SCORE_ENABLED. + + PYTHONPATH=. python validate_catalysts.py + PYTHONPATH=. python validate_catalysts.py --out /tmp/catalyst_validation.csv + +IMPORTANT — coverage constraint: bars are retained indefinitely, but news is only +ingested LIVE (RSS feeds can't be replayed for past dates). So a session only has +a catalyst here if the system was running live that day. The report prints how +many sessions actually had both; if that's ~0, let Phase 1 advisory run live for a +stretch first to accumulate news, then re-run. +""" + +from __future__ import annotations + +import argparse +import csv +from datetime import datetime, time, timedelta + +from research.ingestion.news_enrichment import catalyst_score +from research.multi_schema import open_research_db +from learn_params import load_sessions + + +def _session_outcome(bars) -> dict: + """open->close % and open->high % from a session's (RTH) minute bars.""" + if bars is None or len(bars) == 0: + return {"open_to_close_pct": None, "open_to_high_pct": None} + o = float(bars["open"].iloc[0]) + if o <= 0: + return {"open_to_close_pct": None, "open_to_high_pct": None} + close = float(bars["close"].iloc[-1]) + high = float(bars["high"].max()) + return { + "open_to_close_pct": round((close - o) / o * 100.0, 3), + "open_to_high_pct": round((high - o) / o * 100.0, 3), + } + + +def _catalyst_for(con, symbol: str, session_date) -> dict | None: + """Latest enriched catalyst for ``symbol`` within +-1 day of the session. + + The +-1 day window ties an overnight/premarket PR to the morning's gap while + staying tight enough not to borrow a catalyst from a different week.""" + lo = datetime.combine(session_date - timedelta(days=1), time(0, 0)) + hi = datetime.combine(session_date + timedelta(days=1), time(0, 0)) + try: + row = con.execute( + "SELECT catalyst_type, sentiment, conviction, is_dilutive, headline " + "FROM news_catalyst_cache WHERE symbol = ? " + "AND enriched_at >= ? AND enriched_at < ? " + "ORDER BY enriched_at DESC LIMIT 1", + [symbol, lo, hi], + ).fetchone() + except Exception: # noqa: BLE001 — harness must never crash on a DB fault + return None + if not row: + return None + return {"catalyst_type": row[0], "sentiment": row[1], "conviction": row[2], + "is_dilutive": bool(row[3]), "headline": row[4]} + + +def _bucket(score) -> str: + if score is None: + return "none" + if score < 0.33: + return "low" + if score < 0.66: + return "mid" + return "high" + + +def _mean(xs): + return round(sum(xs) / len(xs), 3) if xs else None + + +def _group_stats(rows: list[dict]) -> dict: + return { + "n": len(rows), + "mean_open_to_close_pct": _mean( + [r["open_to_close_pct"] for r in rows if r.get("open_to_close_pct") is not None]), + "mean_open_to_high_pct": _mean( + [r["open_to_high_pct"] for r in rows if r.get("open_to_high_pct") is not None]), + } + + +def outcome_correlation(rows: list[dict]) -> dict: + """PURE aggregation: split gapper outcomes by dilution flag and score bucket. + + ``rows`` items need: open_to_close_pct, open_to_high_pct, has_catalyst, + is_dilutive, catalyst_score. No DB/Ollama — unit tested directly.""" + with_cat = [r for r in rows if r.get("has_catalyst")] + buckets: dict[str, list] = {} + for r in with_cat: + buckets.setdefault(_bucket(r.get("catalyst_score")), []).append(r) + return { + "total_gappers": len(rows), + "with_catalyst": len(with_cat), + "dilutive": _group_stats([r for r in with_cat if r.get("is_dilutive")]), + "non_dilutive": _group_stats([r for r in with_cat if not r.get("is_dilutive")]), + "by_score_bucket": {k: _group_stats(v) for k, v in sorted(buckets.items())}, + } + + +def _format_report(summary: dict, n_sessions: int) -> str: + def line(label, s): + c, h = s["mean_open_to_close_pct"], s["mean_open_to_high_pct"] + c = f"{c:+6.2f}%" if c is not None else " n/a" + h = f"{h:+6.2f}%" if h is not None else " n/a" + return f" {label:<14} n={s['n']:<4} open->close {c} open->high {h}" + + out = [ + "Catalyst → outcome validation", + f" sessions with bars: {n_sessions} gappers: {summary['total_gappers']} " + f"with catalyst: {summary['with_catalyst']}", + ] + if summary["with_catalyst"] == 0: + out.append("") + out.append(" No gapper sessions had an enriched catalyst yet. News is ingested") + out.append(" LIVE only — let Phase 1 advisory run for a stretch, then re-run this.") + return "\n".join(out) + out.append(" Dilution split:") + out.append(line("dilutive", summary["dilutive"])) + out.append(line("non-dilutive", summary["non_dilutive"])) + out.append(" Bullish catalyst_score bucket:") + for b, s in summary["by_score_bucket"].items(): + out.append(line(b, s)) + return "\n".join(out) + + +def gather_rows(con, limit_sessions: int | None = None) -> list[dict]: + sessions = load_sessions(con) + if limit_sessions: + sessions = sessions[-limit_sessions:] + rows = [] + for symbol, sess, bars, _pc, _adv in sessions: + outcome = _session_outcome(bars) + cat = _catalyst_for(con, symbol, sess) + rows.append({ + "symbol": symbol, + "session_date": str(sess), + **outcome, + "has_catalyst": cat is not None, + "catalyst_type": cat["catalyst_type"] if cat else None, + "is_dilutive": cat["is_dilutive"] if cat else None, + "catalyst_score": catalyst_score(cat) if cat else None, + "headline": cat["headline"] if cat else None, + }) + return rows + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description="Validate catalyst labels vs gapper outcomes") + p.add_argument("--out", default="catalyst_validation.csv", help="CSV output path") + p.add_argument("--limit-sessions", type=int, default=None, + help="only the most recent N (symbol,date) sessions") + args = p.parse_args(argv) + + con = open_research_db("market") + rows = gather_rows(con, limit_sessions=args.limit_sessions) + n_sessions = len({r["session_date"] for r in rows}) + summary = outcome_correlation(rows) + print(_format_report(summary, n_sessions)) + + if rows: + cols = ["symbol", "session_date", "open_to_close_pct", "open_to_high_pct", + "has_catalyst", "catalyst_type", "is_dilutive", "catalyst_score", "headline"] + with open(args.out, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=cols) + w.writeheader() + for r in rows: + w.writerow({k: r.get(k) for k in cols}) + print(f"\n wrote {len(rows)} rows -> {args.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())