research/conversion/{__init__,dataset,signal_probe}.py — the leakage-safe probe
plan 009 Step 5 re-runs scoped to a live setup_version:
- dataset.load_dataset: setup_events JOIN outcome_labels; label =
reached_1r_before_minus_1r; features = trigger-time-knowable setup_events
columns ONLY, dropping >70%-null and <2-distinct columns. HARD leakage rule —
every outcome_labels column is blocklisted AND asserted absent from X, so a
schema change can't leak the answer in.
- signal_probe.run_probe: expanding-window CV over ~5 sequential session-date
blocks (whole days, train strictly precedes test), HistGradientBoostingClassifier
(max_iter=250, lr=0.05, max_depth=4, l2=1.0, categorical_features=indices),
per-fold + mean ROC-AUC, top-decile +1R lift vs base, permutation importance on
the last fold. Drops columns constant within a train slice (HistGBM binning).
CLI: python3 -m research.conversion.signal_probe [--setup-version v1].
Tests: the critical leakage guard (zero outcome columns in features), AUC>0.5 on a
synthetic signal + ~0.5 on pure noise, and a static live-path guard. All :memory:.
No live-path change.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
113 lines
5.0 KiB
Python
113 lines
5.0 KiB
Python
"""Conversion probe (plans 008/009): the critical leakage guard + signal/noise
|
|
sanity + a static live-path guard. Isolated :memory: schema, never prod."""
|
|
|
|
from datetime import date, datetime, timedelta
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from research.conversion.dataset import OUTCOME_LABEL_COLUMNS, LABEL, load_dataset
|
|
from research.conversion.signal_probe import run_probe
|
|
from storage.db import get_connection
|
|
|
|
_SE_COLS = ["setup_id", "symbol", "setup_time", "session_date", "setup_name",
|
|
"setup_version", "gap_pct", "relative_volume", "above_vwap_flag",
|
|
"session_minute_number", "tape_regime_at_trigger",
|
|
"spy_intraday_return_pct_at_trigger", "relative_volume_source"]
|
|
|
|
|
|
def _seed(con, rows):
|
|
"""rows = list of dicts with feature values + 'label' + row index. Writes a
|
|
setup_event and matching outcome_label (with the OTHER outcome columns populated
|
|
too, so the leakage guard is tested against real values, not nulls)."""
|
|
for i, r in enumerate(rows):
|
|
d = r["session_date"]
|
|
sid = f"S{i}:{d}"
|
|
con.execute(
|
|
f"INSERT INTO setup_events ({', '.join(_SE_COLS)}) "
|
|
f"VALUES ({', '.join(['?'] * len(_SE_COLS))})",
|
|
[sid, "SYM", datetime(d.year, d.month, d.day, 9, 34), d, "orb_completion",
|
|
r.get("setup_version", "v1"), r["gap_pct"], r["relative_volume"],
|
|
bool(r.get("above_vwap", True)), 5, r.get("tape", "chop"),
|
|
r.get("spy_ret", 0.0), r.get("rvol_src", "avg_vol_20d")])
|
|
con.execute(
|
|
"INSERT INTO outcome_labels (setup_id, label_version, "
|
|
"reached_1r_before_minus_1r, reached_2r_before_minus_1r, "
|
|
"max_upside_next_60m, trend_day_flag, failed_breakout_flag) "
|
|
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
[sid, "v1", bool(r["label"]), False, 0.42, True, False])
|
|
|
|
|
|
def _dates(n_days, per_day):
|
|
d0 = date(2026, 6, 1)
|
|
return [d0 + timedelta(days=i) for i in range(n_days) for _ in range(per_day)]
|
|
|
|
|
|
def test_dataset_has_zero_outcome_columns():
|
|
"""THE leakage test: no outcome_labels column — least of all the label itself —
|
|
may appear in the feature matrix, even though the join carries them all."""
|
|
con = get_connection(":memory:")
|
|
try:
|
|
rng = np.random.default_rng(0)
|
|
rows = [{"session_date": d, "gap_pct": float(rng.random()),
|
|
"relative_volume": float(rng.random() * 5), "label": int(rng.integers(2))}
|
|
for d in _dates(6, 20)]
|
|
_seed(con, rows)
|
|
X, y, sdate, cats = load_dataset(con, setup_version="v1")
|
|
assert set(X.columns).isdisjoint(OUTCOME_LABEL_COLUMNS)
|
|
assert LABEL not in X.columns
|
|
assert "max_upside_next_60m" not in X.columns # a populated outcome column
|
|
assert "trend_day_flag" not in X.columns and "failed_breakout_flag" not in X.columns
|
|
assert len(y) == len(rows) and set(y.unique()) <= {0, 1}
|
|
finally:
|
|
con.close()
|
|
|
|
|
|
def test_probe_recovers_signal():
|
|
"""A feature that genuinely predicts the label -> mean CV AUC well above 0.5."""
|
|
con = get_connection(":memory:")
|
|
try:
|
|
rng = np.random.default_rng(7)
|
|
rows = []
|
|
for d in _dates(10, 40):
|
|
gap = float(rng.random())
|
|
label = int((gap + rng.normal(0, 0.15)) > 0.5) # gap_pct carries the signal
|
|
rows.append({"session_date": d, "gap_pct": gap,
|
|
"relative_volume": float(rng.random() * 5), "label": label})
|
|
_seed(con, rows)
|
|
res = run_probe(con, setup_version="v1", n_blocks=5, seed=0)
|
|
assert res["mean_auc"] is not None and res["mean_auc"] > 0.62
|
|
finally:
|
|
con.close()
|
|
|
|
|
|
def test_probe_flat_on_pure_noise():
|
|
"""Labels independent of every feature -> mean CV AUC ~0.5 (no false signal)."""
|
|
con = get_connection(":memory:")
|
|
try:
|
|
rng = np.random.default_rng(11)
|
|
rows = [{"session_date": d, "gap_pct": float(rng.random()),
|
|
"relative_volume": float(rng.random() * 5), "label": int(rng.integers(2))}
|
|
for d in _dates(10, 40)]
|
|
_seed(con, rows)
|
|
res = run_probe(con, setup_version="v1", n_blocks=5, seed=0)
|
|
assert res["mean_auc"] is not None and 0.40 <= res["mean_auc"] <= 0.60
|
|
finally:
|
|
con.close()
|
|
|
|
|
|
def test_no_live_path_imports():
|
|
"""Static guard: the plan-009 research code never reaches into any live
|
|
gate/sizing/exit module (trading_execution / runtime / run_live_paper)."""
|
|
import pathlib
|
|
root = pathlib.Path(__file__).resolve().parents[3]
|
|
files = [root / "research" / "labeler.py",
|
|
root / "research" / "ingestion" / "market_context.py",
|
|
root / "research" / "conversion" / "dataset.py",
|
|
root / "research" / "conversion" / "signal_probe.py"]
|
|
forbidden = ("trading_execution", "run_live_paper", "import runtime", "from runtime")
|
|
for f in files:
|
|
src = f.read_text()
|
|
hits = [tok for tok in forbidden if tok in src]
|
|
assert not hits, f"{f.name} references a live path: {hits}"
|