From 849bd476f9776caaa18fed134e276a487a93cdbc Mon Sep 17 00:00:00 2001 From: momentum dev Date: Fri, 17 Jul 2026 19:50:05 -0500 Subject: [PATCH] adjudicator: random-gate null for keep-or-zero PASSes + register orderflow H6 (quote-size book imbalance) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Null (the H2 lesson): zeroing share f of a negative baseline mechanically yields expR_delta = f*|mean baseline|; a PASS on expR_delta_min now also requires expR_delta_vs_null >= the same margin whenever zeroed_share >= 5%. Exposure-shrinking can no longer masquerade as selection; non-gate experiments unaffected. Honest disclosure: regime_meta_v1's FIRE-weighted pass would not clear this bar either — its standing rests on the session- level walk-forward OOS evidence, which no random gate reproduces. H6: CKS order-flow-imbalance from quote_snapshots bid/ask SIZES — size-weighted book imbalance over the 10-min pre-trigger window, zero ask-tilted books; skip-on-no-coverage; judged under the null from day one. Co-Authored-By: Claude Fable 5 --- experiments/orderflow_h6_ofi_v1.json | 8 +++ research/experiments/adjudicate.py | 29 ++++++++++- research/experiments/replay.py | 48 +++++++++++++++++ tests/unit/experiments/test_adjudicate.py | 42 +++++++++++++++ .../test_orderflow_interventions.py | 52 +++++++++++++++++++ 5 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 experiments/orderflow_h6_ofi_v1.json diff --git a/experiments/orderflow_h6_ofi_v1.json b/experiments/orderflow_h6_ofi_v1.json new file mode 100644 index 0000000..a0efb3e --- /dev/null +++ b/experiments/orderflow_h6_ofi_v1.json @@ -0,0 +1,8 @@ +{ + "id": "orderflow_h6_ofi_v1", + "hypothesis": "Quote-size book imbalance (CKS order-flow-imbalance family — the microstructure shelf's highest-powered predictor, unlocked because quote_snapshots logs bid/ask SIZES): fires entered into an ask-tilted book (size-weighted imbalance < 0 over the 10-min pre-trigger window) face a supply overhang and convert worse; zeroing them lifts kept expR. detail.book_imbalance recorded per fire for threshold/tercile analysis. Coverage: fires without >=5 usable snapshots are SKIPPED (excluded from n); judged under the random-gate null bar from day one.", + "population": {"min_entry_gap_pct": 0.10, "min_day_volume": 300000, "price_min": 2.0, "price_max": 20.0}, + "intervention": {"kind": "quote_imbalance_gate", "baseline_minutes": 10, "min_book_imbalance": 0.0}, + "bars": {"expR_delta_min": 0.05, "min_n": 300}, + "fidelity": {"atr_frac": 0.25, "spread_mult": 3, "adverse_ticks": 1} +} diff --git a/research/experiments/adjudicate.py b/research/experiments/adjudicate.py index 0b34eb8..4f7ecd5 100644 --- a/research/experiments/adjudicate.py +++ b/research/experiments/adjudicate.py @@ -85,12 +85,32 @@ def _meets(key: str, value, metrics: dict) -> bool: def evaluate(runs: list[dict], bars: dict, futility: dict | None): - """Return (verdict, metrics, n). Pure — no DB.""" + """Return (verdict, metrics, n). Pure — no DB. + + RANDOM-GATE NULL (2026-07-17, the orderflow-H2 lesson): a keep-or-zero + gate zeroing share f of a negative-baseline population mechanically + produces expR_delta = f * |mean baseline| — exposure-shrinking that + masquerades as selection. When an experiment has a material zeroed share, + a PASS on ``expR_delta_min`` additionally requires ``expR_delta_vs_null`` + (delta minus that mechanical null) to clear the same margin. Non-gate + experiments (zeroed share ~0) are unaffected: vs_null == delta.""" n = sum(1 for r in runs if r["variant_r"] is not None) referenced = {_metric_name(k) for k in list(bars) + list(futility or {}) if k != "min_n"} metrics = {name: _metric(name, runs) for name in referenced} + if "expR_delta" in metrics and metrics["expR_delta"] is not None: + both = [r for r in runs if r["variant_r"] is not None and r["baseline_r"] is not None] + zeroed = sum(1 for r in both + if r["variant_r"] == 0.0 and r["baseline_r"] != 0.0) + zero_share = (zeroed / len(both)) if both else 0.0 + mean_base = (sum(r["baseline_r"] for r in both) / len(both)) if both else 0.0 + null_delta = zero_share * (0.0 - mean_base) # what a RANDOM gate earns + metrics["zeroed_share"] = round(zero_share, 4) + metrics["null_delta"] = null_delta + metrics["expR_delta_vs_null"] = (metrics["expR_delta"] - null_delta + if zero_share >= 0.05 else metrics["expR_delta"]) + if futility and n >= int(futility.get("min_n", 1 << 60)): conds = [c for c in futility if c != "min_n"] # _meets is False on a None metric, so futility never fires on no data. @@ -105,6 +125,13 @@ def evaluate(runs: list[dict], bars: dict, futility: dict | None): # can't conclude, so INCONCLUSIVE rather than a false FAIL. if any(metrics.get(_metric_name(c)) is None for c in bar_conds): return "INCONCLUSIVE", metrics, n + if ("expR_delta_min" in bars + and metrics.get("expR_delta_vs_null") is not None + and metrics["expR_delta_vs_null"] < float(bars["expR_delta_min"]) + and metrics.get("zeroed_share", 0.0) >= 0.05): + # gate cleared the raw bar but not the random-gate null — the + # "gain" is exposure-shrinking, not selection. Never a PASS. + return "FAIL", metrics, n if bar_conds and all(_meets(c, bars[c], metrics) for c in bar_conds): return "PASS", metrics, n return "FAIL", metrics, n diff --git a/research/experiments/replay.py b/research/experiments/replay.py index 77200ad..d5e3751 100644 --- a/research/experiments/replay.py +++ b/research/experiments/replay.py @@ -658,6 +658,47 @@ def iv_spread_expansion_gate(con, fire: "Fire", baseline: float | None, return baseline, detail +def iv_quote_imbalance_gate(con, fire: "Fire", baseline: float | None, + params: dict) -> tuple[float | None, dict]: + """H6 (Cont-Kukanov-Stoikov order-flow-imbalance family — the shelf's + highest-powered predictor, computable because quote_snapshots logs bid/ask + SIZES): size-weighted book imbalance over the pre-trigger window, + sum(bid_size - ask_size) / sum(bid_size + ask_size) in [-1, +1]. A + bid-tilted book into the trigger = buy-side depth backing the move -> + KEEP; an ask-tilted book = supply overhang -> ZERO. Same coverage design + as iv_spread_expansion_gate: fires without >=5 usable snapshots are + SKIPPED (variant None, excluded from n) — never silently kept.""" + from zoneinfo import ZoneInfo + + minutes = int(params.get("baseline_minutes", 10)) + min_imb = float(params.get("min_book_imbalance", 0.0)) + detail = {"kind": "quote_imbalance_gate", "flagged": False, + "baseline_minutes": minutes, "threshold": min_imb, "book_imbalance": None} + try: + ts = fire.fill_ts if isinstance(fire.fill_ts, datetime) \ + else datetime.fromisoformat(str(fire.fill_ts)) + fill_utc = ts.replace(tzinfo=ZoneInfo("America/New_York")) if ts.tzinfo is None else ts + from datetime import timedelta + rows = con.execute( + "SELECT bid_size, ask_size FROM quote_snapshots " + "WHERE symbol = ? AND ts >= ? AND ts <= ? ORDER BY ts", + [fire.symbol, fill_utc - timedelta(minutes=minutes), fill_utc]).fetchall() + except Exception: # noqa: BLE001 — table absent / query fault = no coverage + detail["skipped"] = "no_quote_coverage" + return None, detail + pairs = [(float(b), float(a)) for b, a in rows if b and a and b > 0 and a > 0] + if len(pairs) < 5: + detail["skipped"] = "no_quote_coverage" + return None, detail + total = sum(b + a for b, a in pairs) + imbalance = sum(b - a for b, a in pairs) / total + detail["book_imbalance"] = round(imbalance, 4) + if imbalance < min_imb: + detail["flagged"] = True + return 0.0, detail + return baseline, detail + + # --------------------------------------------------------------------------- # # Rank (portfolio-level; receipts-only, starves without plan 005) # --------------------------------------------------------------------------- # @@ -754,6 +795,13 @@ def _compute_fire(con, exp: dict, fire: Fire, favorable: dict | None = None, if fire.sim_baseline: d["source"] = "setup_sim" return {"baseline_r": baseline, "variant_r": v, "detail": d} + if kind == "quote_imbalance_gate": + # orderflow H6: CKS book-imbalance from quote sizes; no coverage -> SKIP. + baseline = _sim_baseline_r(fire, bars_after, fidelity) if fire.sim_baseline else fire.baseline_r + v, d = iv_quote_imbalance_gate(con, fire, baseline, params) + if fire.sim_baseline: + d["source"] = "setup_sim" + return {"baseline_r": baseline, "variant_r": v, "detail": d} if kind == "time_stop": v, d = iv_time_stop(fire, bars_after, trigger_high, params, fidelity) elif kind == "probe_add": diff --git a/tests/unit/experiments/test_adjudicate.py b/tests/unit/experiments/test_adjudicate.py index 9537331..195dd38 100644 --- a/tests/unit/experiments/test_adjudicate.py +++ b/tests/unit/experiments/test_adjudicate.py @@ -129,3 +129,45 @@ def test_pm1_bridge_fail_verdict_and_event(store, tmp_path, monkeypatch): assert len(sent) == 1 assert store.con.execute( "SELECT count(*) FROM experiment_runs WHERE experiment_id='pm1_v1'").fetchone()[0] == 5 + + +# -------------------------------------------------------------------------- +# Random-gate null (2026-07-17, the orderflow-H2 lesson): on a negative +# baseline, ANY keep-or-zero gate zeroing share f mechanically produces +# expR_delta = f * |mean baseline|. A PASS must beat that null by the +# pre-registered margin, not just beat zero. +# -------------------------------------------------------------------------- +def _kz_runs(n_zeroed, n_kept, base=-0.30): + """keep-or-zero shaped runs: zeroed fires variant 0.0, kept pass through.""" + runs = [{"baseline_r": base, "variant_r": 0.0, "detail": {}} for _ in range(n_zeroed)] + runs += [{"baseline_r": base, "variant_r": base, "detail": {}} for _ in range(n_kept)] + return runs + + +def test_keep_or_zero_pass_requires_beating_random_gate_null(): + # zero 44% of a -0.30 baseline: raw delta = 0.44*0.30 = +0.132 "passes" the + # old bar, but a random gate produces exactly that -> must be INCONCLUSIVE- + # or FAIL, never PASS. + runs = _kz_runs(440, 560) + verdict, metrics, n = evaluate(runs, {"expR_delta_min": 0.05, "min_n": 300}, None) + assert metrics["expR_delta"] > 0.12 # the mechanical illusion + assert verdict != "PASS" + assert metrics["expR_delta_vs_null"] < 0.05 # and the metric exposes it + + +def test_keep_or_zero_pass_when_gate_truly_selects(): + # gate zeroes fires that were MUCH worse than the kept ones: delta beats + # the random-gate null by more than the margin -> PASS survives. + runs = [{"baseline_r": -1.0, "variant_r": 0.0, "detail": {}} for _ in range(300)] + runs += [{"baseline_r": 0.10, "variant_r": 0.10, "detail": {}} for _ in range(700)] + verdict, metrics, n = evaluate(runs, {"expR_delta_min": 0.05, "min_n": 300}, None) + assert verdict == "PASS" + assert metrics["expR_delta_vs_null"] >= 0.05 + + +def test_non_gate_experiments_unaffected_by_null(): + # exit-style runs (variant rarely exactly 0.0): null share ~0 -> vs_null + # equals delta and the old semantics hold. + runs = [{"baseline_r": -0.2, "variant_r": -0.1, "detail": {}} for _ in range(400)] + verdict, metrics, n = evaluate(runs, {"expR_delta_min": 0.05, "min_n": 300}, None) + assert verdict == "PASS" diff --git a/tests/unit/experiments/test_orderflow_interventions.py b/tests/unit/experiments/test_orderflow_interventions.py index 263b0f2..2cc13a9 100644 --- a/tests/unit/experiments/test_orderflow_interventions.py +++ b/tests/unit/experiments/test_orderflow_interventions.py @@ -194,3 +194,55 @@ def test_no_quote_coverage_is_skipped_not_kept(): v, d = iv_spread_expansion_gate(store.con, f, baseline=0.4, params={"baseline_minutes": 10, "max_expansion_ratio": 2.0}) assert v is None and d.get("skipped") == "no_quote_coverage" + + +# ------------------------------------------------- H6: quote-size imbalance +def _seed_sized_quotes(store, symbol, fill_utc, sizes): + """sizes: list of (bid_size, ask_size) at 15s cadence ending just before fill.""" + from datetime import timedelta + _seed_quotes(store, "_ddl_only_", []) # ensure table + for i, (bs, asz) in enumerate(sizes): + store.con.execute( + "INSERT INTO quote_snapshots (symbol, ts, bid_price, bid_size, ask_price, ask_size) " + "VALUES (?,?,?,?,?,?)", + [symbol, fill_utc - timedelta(seconds=15 * (len(sizes) - i)), 10.0, bs, 10.02, asz]) + + +def test_bid_heavy_book_is_kept(): + from datetime import timezone + from storage.event_store import EventStore + from research.experiments.replay import iv_quote_imbalance_gate + store = EventStore(":memory:") + fill_utc = datetime(2026, 6, 10, 13, 40, tzinfo=timezone.utc) + _seed_sized_quotes(store, "AAA", fill_utc, [(900, 100)] * 8) # imbalance +0.8 + f = _fire() + f = type(f)(**{**f.__dict__, "fill_ts": datetime(2026, 6, 10, 9, 40)}) + v, d = iv_quote_imbalance_gate(store.con, f, baseline=0.4, + params={"baseline_minutes": 10, "min_book_imbalance": 0.0}) + assert v == 0.4 and d["flagged"] is False and d["book_imbalance"] > 0.5 + + +def test_ask_heavy_book_is_zeroed(): + from datetime import timezone + from storage.event_store import EventStore + from research.experiments.replay import iv_quote_imbalance_gate + store = EventStore(":memory:") + fill_utc = datetime(2026, 6, 10, 13, 40, tzinfo=timezone.utc) + _seed_sized_quotes(store, "AAA", fill_utc, [(100, 900)] * 8) # imbalance -0.8 + f = _fire() + f = type(f)(**{**f.__dict__, "fill_ts": datetime(2026, 6, 10, 9, 40)}) + v, d = iv_quote_imbalance_gate(store.con, f, baseline=0.4, + params={"baseline_minutes": 10, "min_book_imbalance": 0.0}) + assert v == 0.0 and d["flagged"] is True and d["book_imbalance"] < -0.5 + + +def test_ofi_no_coverage_is_skipped(): + from storage.event_store import EventStore + from research.experiments.replay import iv_quote_imbalance_gate + store = EventStore(":memory:") + _seed_quotes(store, "OTHER", []) + f = _fire() + f = type(f)(**{**f.__dict__, "fill_ts": datetime(2026, 6, 10, 9, 40)}) + v, d = iv_quote_imbalance_gate(store.con, f, baseline=0.4, + params={"baseline_minutes": 10, "min_book_imbalance": 0.0}) + assert v is None and d.get("skipped") == "no_quote_coverage"