Files
momentum devandClaude Fable 5 dbaeb701e2 conversion(009): committed go/no-go signal probe (Step 5 instrument)
research/conversion/{__init__,dataset,signal_probe}.py — the leakage-safe probe
plan 009 Step 5 re-runs scoped to a live setup_version:

- dataset.load_dataset: setup_events JOIN outcome_labels; label =
  reached_1r_before_minus_1r; features = trigger-time-knowable setup_events
  columns ONLY, dropping >70%-null and <2-distinct columns. HARD leakage rule —
  every outcome_labels column is blocklisted AND asserted absent from X, so a
  schema change can't leak the answer in.
- signal_probe.run_probe: expanding-window CV over ~5 sequential session-date
  blocks (whole days, train strictly precedes test), HistGradientBoostingClassifier
  (max_iter=250, lr=0.05, max_depth=4, l2=1.0, categorical_features=indices),
  per-fold + mean ROC-AUC, top-decile +1R lift vs base, permutation importance on
  the last fold. Drops columns constant within a train slice (HistGBM binning).
  CLI: python3 -m research.conversion.signal_probe [--setup-version v1].

Tests: the critical leakage guard (zero outcome columns in features), AUC>0.5 on a
synthetic signal + ~0.5 on pure noise, and a static live-path guard. All :memory:.
No live-path change.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-14 20:32:18 -04:00

113 lines
5.0 KiB
Python

"""Conversion probe (plans 008/009): the critical leakage guard + signal/noise
sanity + a static live-path guard. Isolated :memory: schema, never prod."""
from datetime import date, datetime, timedelta
import numpy as np
import pytest
from research.conversion.dataset import OUTCOME_LABEL_COLUMNS, LABEL, load_dataset
from research.conversion.signal_probe import run_probe
from storage.db import get_connection
_SE_COLS = ["setup_id", "symbol", "setup_time", "session_date", "setup_name",
"setup_version", "gap_pct", "relative_volume", "above_vwap_flag",
"session_minute_number", "tape_regime_at_trigger",
"spy_intraday_return_pct_at_trigger", "relative_volume_source"]
def _seed(con, rows):
"""rows = list of dicts with feature values + 'label' + row index. Writes a
setup_event and matching outcome_label (with the OTHER outcome columns populated
too, so the leakage guard is tested against real values, not nulls)."""
for i, r in enumerate(rows):
d = r["session_date"]
sid = f"S{i}:{d}"
con.execute(
f"INSERT INTO setup_events ({', '.join(_SE_COLS)}) "
f"VALUES ({', '.join(['?'] * len(_SE_COLS))})",
[sid, "SYM", datetime(d.year, d.month, d.day, 9, 34), d, "orb_completion",
r.get("setup_version", "v1"), r["gap_pct"], r["relative_volume"],
bool(r.get("above_vwap", True)), 5, r.get("tape", "chop"),
r.get("spy_ret", 0.0), r.get("rvol_src", "avg_vol_20d")])
con.execute(
"INSERT INTO outcome_labels (setup_id, label_version, "
"reached_1r_before_minus_1r, reached_2r_before_minus_1r, "
"max_upside_next_60m, trend_day_flag, failed_breakout_flag) "
"VALUES (?, ?, ?, ?, ?, ?, ?)",
[sid, "v1", bool(r["label"]), False, 0.42, True, False])
def _dates(n_days, per_day):
d0 = date(2026, 6, 1)
return [d0 + timedelta(days=i) for i in range(n_days) for _ in range(per_day)]
def test_dataset_has_zero_outcome_columns():
"""THE leakage test: no outcome_labels column — least of all the label itself —
may appear in the feature matrix, even though the join carries them all."""
con = get_connection(":memory:")
try:
rng = np.random.default_rng(0)
rows = [{"session_date": d, "gap_pct": float(rng.random()),
"relative_volume": float(rng.random() * 5), "label": int(rng.integers(2))}
for d in _dates(6, 20)]
_seed(con, rows)
X, y, sdate, cats = load_dataset(con, setup_version="v1")
assert set(X.columns).isdisjoint(OUTCOME_LABEL_COLUMNS)
assert LABEL not in X.columns
assert "max_upside_next_60m" not in X.columns # a populated outcome column
assert "trend_day_flag" not in X.columns and "failed_breakout_flag" not in X.columns
assert len(y) == len(rows) and set(y.unique()) <= {0, 1}
finally:
con.close()
def test_probe_recovers_signal():
"""A feature that genuinely predicts the label -> mean CV AUC well above 0.5."""
con = get_connection(":memory:")
try:
rng = np.random.default_rng(7)
rows = []
for d in _dates(10, 40):
gap = float(rng.random())
label = int((gap + rng.normal(0, 0.15)) > 0.5) # gap_pct carries the signal
rows.append({"session_date": d, "gap_pct": gap,
"relative_volume": float(rng.random() * 5), "label": label})
_seed(con, rows)
res = run_probe(con, setup_version="v1", n_blocks=5, seed=0)
assert res["mean_auc"] is not None and res["mean_auc"] > 0.62
finally:
con.close()
def test_probe_flat_on_pure_noise():
"""Labels independent of every feature -> mean CV AUC ~0.5 (no false signal)."""
con = get_connection(":memory:")
try:
rng = np.random.default_rng(11)
rows = [{"session_date": d, "gap_pct": float(rng.random()),
"relative_volume": float(rng.random() * 5), "label": int(rng.integers(2))}
for d in _dates(10, 40)]
_seed(con, rows)
res = run_probe(con, setup_version="v1", n_blocks=5, seed=0)
assert res["mean_auc"] is not None and 0.40 <= res["mean_auc"] <= 0.60
finally:
con.close()
def test_no_live_path_imports():
"""Static guard: the plan-009 research code never reaches into any live
gate/sizing/exit module (trading_execution / runtime / run_live_paper)."""
import pathlib
root = pathlib.Path(__file__).resolve().parents[3]
files = [root / "research" / "labeler.py",
root / "research" / "ingestion" / "market_context.py",
root / "research" / "conversion" / "dataset.py",
root / "research" / "conversion" / "signal_probe.py"]
forbidden = ("trading_execution", "run_live_paper", "import runtime", "from runtime")
for f in files:
src = f.read_text()
hits = [tok for tok in forbidden if tok in src]
assert not hits, f"{f.name} references a live path: {hits}"