adjudicator: random-gate null for keep-or-zero PASSes + register orderflow H6 (quote-size book imbalance)
Null (the H2 lesson): zeroing share f of a negative baseline mechanically yields expR_delta = f*|mean baseline|; a PASS on expR_delta_min now also requires expR_delta_vs_null >= the same margin whenever zeroed_share >= 5%. Exposure-shrinking can no longer masquerade as selection; non-gate experiments unaffected. Honest disclosure: regime_meta_v1's FIRE-weighted pass would not clear this bar either — its standing rests on the session- level walk-forward OOS evidence, which no random gate reproduces. H6: CKS order-flow-imbalance from quote_snapshots bid/ask SIZES — size-weighted book imbalance over the 10-min pre-trigger window, zero ask-tilted books; skip-on-no-coverage; judged under the null from day one. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5bf089cdcc
commit
849bd476f9
5 files changed
+178
-1
No files matched your search
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"id": "orderflow_h6_ofi_v1",
|
||||||
|
"hypothesis": "Quote-size book imbalance (CKS order-flow-imbalance family — the microstructure shelf's highest-powered predictor, unlocked because quote_snapshots logs bid/ask SIZES): fires entered into an ask-tilted book (size-weighted imbalance < 0 over the 10-min pre-trigger window) face a supply overhang and convert worse; zeroing them lifts kept expR. detail.book_imbalance recorded per fire for threshold/tercile analysis. Coverage: fires without >=5 usable snapshots are SKIPPED (excluded from n); judged under the random-gate null bar from day one.",
|
||||||
|
"population": {"min_entry_gap_pct": 0.10, "min_day_volume": 300000, "price_min": 2.0, "price_max": 20.0},
|
||||||
|
"intervention": {"kind": "quote_imbalance_gate", "baseline_minutes": 10, "min_book_imbalance": 0.0},
|
||||||
|
"bars": {"expR_delta_min": 0.05, "min_n": 300},
|
||||||
|
"fidelity": {"atr_frac": 0.25, "spread_mult": 3, "adverse_ticks": 1}
|
||||||
|
}
|
||||||
@@ -85,12 +85,32 @@ def _meets(key: str, value, metrics: dict) -> bool:
|
|||||||
|
|
||||||
|
|
||||||
def evaluate(runs: list[dict], bars: dict, futility: dict | None):
|
def evaluate(runs: list[dict], bars: dict, futility: dict | None):
|
||||||
"""Return (verdict, metrics, n). Pure — no DB."""
|
"""Return (verdict, metrics, n). Pure — no DB.
|
||||||
|
|
||||||
|
RANDOM-GATE NULL (2026-07-17, the orderflow-H2 lesson): a keep-or-zero
|
||||||
|
gate zeroing share f of a negative-baseline population mechanically
|
||||||
|
produces expR_delta = f * |mean baseline| — exposure-shrinking that
|
||||||
|
masquerades as selection. When an experiment has a material zeroed share,
|
||||||
|
a PASS on ``expR_delta_min`` additionally requires ``expR_delta_vs_null``
|
||||||
|
(delta minus that mechanical null) to clear the same margin. Non-gate
|
||||||
|
experiments (zeroed share ~0) are unaffected: vs_null == delta."""
|
||||||
n = sum(1 for r in runs if r["variant_r"] is not None)
|
n = sum(1 for r in runs if r["variant_r"] is not None)
|
||||||
referenced = {_metric_name(k) for k in list(bars) + list(futility or {})
|
referenced = {_metric_name(k) for k in list(bars) + list(futility or {})
|
||||||
if k != "min_n"}
|
if k != "min_n"}
|
||||||
metrics = {name: _metric(name, runs) for name in referenced}
|
metrics = {name: _metric(name, runs) for name in referenced}
|
||||||
|
|
||||||
|
if "expR_delta" in metrics and metrics["expR_delta"] is not None:
|
||||||
|
both = [r for r in runs if r["variant_r"] is not None and r["baseline_r"] is not None]
|
||||||
|
zeroed = sum(1 for r in both
|
||||||
|
if r["variant_r"] == 0.0 and r["baseline_r"] != 0.0)
|
||||||
|
zero_share = (zeroed / len(both)) if both else 0.0
|
||||||
|
mean_base = (sum(r["baseline_r"] for r in both) / len(both)) if both else 0.0
|
||||||
|
null_delta = zero_share * (0.0 - mean_base) # what a RANDOM gate earns
|
||||||
|
metrics["zeroed_share"] = round(zero_share, 4)
|
||||||
|
metrics["null_delta"] = null_delta
|
||||||
|
metrics["expR_delta_vs_null"] = (metrics["expR_delta"] - null_delta
|
||||||
|
if zero_share >= 0.05 else metrics["expR_delta"])
|
||||||
|
|
||||||
if futility and n >= int(futility.get("min_n", 1 << 60)):
|
if futility and n >= int(futility.get("min_n", 1 << 60)):
|
||||||
conds = [c for c in futility if c != "min_n"]
|
conds = [c for c in futility if c != "min_n"]
|
||||||
# _meets is False on a None metric, so futility never fires on no data.
|
# _meets is False on a None metric, so futility never fires on no data.
|
||||||
@@ -105,6 +125,13 @@ def evaluate(runs: list[dict], bars: dict, futility: dict | None):
|
|||||||
# can't conclude, so INCONCLUSIVE rather than a false FAIL.
|
# can't conclude, so INCONCLUSIVE rather than a false FAIL.
|
||||||
if any(metrics.get(_metric_name(c)) is None for c in bar_conds):
|
if any(metrics.get(_metric_name(c)) is None for c in bar_conds):
|
||||||
return "INCONCLUSIVE", metrics, n
|
return "INCONCLUSIVE", metrics, n
|
||||||
|
if ("expR_delta_min" in bars
|
||||||
|
and metrics.get("expR_delta_vs_null") is not None
|
||||||
|
and metrics["expR_delta_vs_null"] < float(bars["expR_delta_min"])
|
||||||
|
and metrics.get("zeroed_share", 0.0) >= 0.05):
|
||||||
|
# gate cleared the raw bar but not the random-gate null — the
|
||||||
|
# "gain" is exposure-shrinking, not selection. Never a PASS.
|
||||||
|
return "FAIL", metrics, n
|
||||||
if bar_conds and all(_meets(c, bars[c], metrics) for c in bar_conds):
|
if bar_conds and all(_meets(c, bars[c], metrics) for c in bar_conds):
|
||||||
return "PASS", metrics, n
|
return "PASS", metrics, n
|
||||||
return "FAIL", metrics, n
|
return "FAIL", metrics, n
|
||||||
|
|||||||
@@ -658,6 +658,47 @@ def iv_spread_expansion_gate(con, fire: "Fire", baseline: float | None,
|
|||||||
return baseline, detail
|
return baseline, detail
|
||||||
|
|
||||||
|
|
||||||
|
def iv_quote_imbalance_gate(con, fire: "Fire", baseline: float | None,
|
||||||
|
params: dict) -> tuple[float | None, dict]:
|
||||||
|
"""H6 (Cont-Kukanov-Stoikov order-flow-imbalance family — the shelf's
|
||||||
|
highest-powered predictor, computable because quote_snapshots logs bid/ask
|
||||||
|
SIZES): size-weighted book imbalance over the pre-trigger window,
|
||||||
|
sum(bid_size - ask_size) / sum(bid_size + ask_size) in [-1, +1]. A
|
||||||
|
bid-tilted book into the trigger = buy-side depth backing the move ->
|
||||||
|
KEEP; an ask-tilted book = supply overhang -> ZERO. Same coverage design
|
||||||
|
as iv_spread_expansion_gate: fires without >=5 usable snapshots are
|
||||||
|
SKIPPED (variant None, excluded from n) — never silently kept."""
|
||||||
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
|
minutes = int(params.get("baseline_minutes", 10))
|
||||||
|
min_imb = float(params.get("min_book_imbalance", 0.0))
|
||||||
|
detail = {"kind": "quote_imbalance_gate", "flagged": False,
|
||||||
|
"baseline_minutes": minutes, "threshold": min_imb, "book_imbalance": None}
|
||||||
|
try:
|
||||||
|
ts = fire.fill_ts if isinstance(fire.fill_ts, datetime) \
|
||||||
|
else datetime.fromisoformat(str(fire.fill_ts))
|
||||||
|
fill_utc = ts.replace(tzinfo=ZoneInfo("America/New_York")) if ts.tzinfo is None else ts
|
||||||
|
from datetime import timedelta
|
||||||
|
rows = con.execute(
|
||||||
|
"SELECT bid_size, ask_size FROM quote_snapshots "
|
||||||
|
"WHERE symbol = ? AND ts >= ? AND ts <= ? ORDER BY ts",
|
||||||
|
[fire.symbol, fill_utc - timedelta(minutes=minutes), fill_utc]).fetchall()
|
||||||
|
except Exception: # noqa: BLE001 — table absent / query fault = no coverage
|
||||||
|
detail["skipped"] = "no_quote_coverage"
|
||||||
|
return None, detail
|
||||||
|
pairs = [(float(b), float(a)) for b, a in rows if b and a and b > 0 and a > 0]
|
||||||
|
if len(pairs) < 5:
|
||||||
|
detail["skipped"] = "no_quote_coverage"
|
||||||
|
return None, detail
|
||||||
|
total = sum(b + a for b, a in pairs)
|
||||||
|
imbalance = sum(b - a for b, a in pairs) / total
|
||||||
|
detail["book_imbalance"] = round(imbalance, 4)
|
||||||
|
if imbalance < min_imb:
|
||||||
|
detail["flagged"] = True
|
||||||
|
return 0.0, detail
|
||||||
|
return baseline, detail
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
# Rank (portfolio-level; receipts-only, starves without plan 005)
|
# Rank (portfolio-level; receipts-only, starves without plan 005)
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
@@ -754,6 +795,13 @@ def _compute_fire(con, exp: dict, fire: Fire, favorable: dict | None = None,
|
|||||||
if fire.sim_baseline:
|
if fire.sim_baseline:
|
||||||
d["source"] = "setup_sim"
|
d["source"] = "setup_sim"
|
||||||
return {"baseline_r": baseline, "variant_r": v, "detail": d}
|
return {"baseline_r": baseline, "variant_r": v, "detail": d}
|
||||||
|
if kind == "quote_imbalance_gate":
|
||||||
|
# orderflow H6: CKS book-imbalance from quote sizes; no coverage -> SKIP.
|
||||||
|
baseline = _sim_baseline_r(fire, bars_after, fidelity) if fire.sim_baseline else fire.baseline_r
|
||||||
|
v, d = iv_quote_imbalance_gate(con, fire, baseline, params)
|
||||||
|
if fire.sim_baseline:
|
||||||
|
d["source"] = "setup_sim"
|
||||||
|
return {"baseline_r": baseline, "variant_r": v, "detail": d}
|
||||||
if kind == "time_stop":
|
if kind == "time_stop":
|
||||||
v, d = iv_time_stop(fire, bars_after, trigger_high, params, fidelity)
|
v, d = iv_time_stop(fire, bars_after, trigger_high, params, fidelity)
|
||||||
elif kind == "probe_add":
|
elif kind == "probe_add":
|
||||||
|
|||||||
@@ -129,3 +129,45 @@ def test_pm1_bridge_fail_verdict_and_event(store, tmp_path, monkeypatch):
|
|||||||
assert len(sent) == 1
|
assert len(sent) == 1
|
||||||
assert store.con.execute(
|
assert store.con.execute(
|
||||||
"SELECT count(*) FROM experiment_runs WHERE experiment_id='pm1_v1'").fetchone()[0] == 5
|
"SELECT count(*) FROM experiment_runs WHERE experiment_id='pm1_v1'").fetchone()[0] == 5
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Random-gate null (2026-07-17, the orderflow-H2 lesson): on a negative
|
||||||
|
# baseline, ANY keep-or-zero gate zeroing share f mechanically produces
|
||||||
|
# expR_delta = f * |mean baseline|. A PASS must beat that null by the
|
||||||
|
# pre-registered margin, not just beat zero.
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
def _kz_runs(n_zeroed, n_kept, base=-0.30):
|
||||||
|
"""keep-or-zero shaped runs: zeroed fires variant 0.0, kept pass through."""
|
||||||
|
runs = [{"baseline_r": base, "variant_r": 0.0, "detail": {}} for _ in range(n_zeroed)]
|
||||||
|
runs += [{"baseline_r": base, "variant_r": base, "detail": {}} for _ in range(n_kept)]
|
||||||
|
return runs
|
||||||
|
|
||||||
|
|
||||||
|
def test_keep_or_zero_pass_requires_beating_random_gate_null():
|
||||||
|
# zero 44% of a -0.30 baseline: raw delta = 0.44*0.30 = +0.132 "passes" the
|
||||||
|
# old bar, but a random gate produces exactly that -> must be INCONCLUSIVE-
|
||||||
|
# or FAIL, never PASS.
|
||||||
|
runs = _kz_runs(440, 560)
|
||||||
|
verdict, metrics, n = evaluate(runs, {"expR_delta_min": 0.05, "min_n": 300}, None)
|
||||||
|
assert metrics["expR_delta"] > 0.12 # the mechanical illusion
|
||||||
|
assert verdict != "PASS"
|
||||||
|
assert metrics["expR_delta_vs_null"] < 0.05 # and the metric exposes it
|
||||||
|
|
||||||
|
|
||||||
|
def test_keep_or_zero_pass_when_gate_truly_selects():
|
||||||
|
# gate zeroes fires that were MUCH worse than the kept ones: delta beats
|
||||||
|
# the random-gate null by more than the margin -> PASS survives.
|
||||||
|
runs = [{"baseline_r": -1.0, "variant_r": 0.0, "detail": {}} for _ in range(300)]
|
||||||
|
runs += [{"baseline_r": 0.10, "variant_r": 0.10, "detail": {}} for _ in range(700)]
|
||||||
|
verdict, metrics, n = evaluate(runs, {"expR_delta_min": 0.05, "min_n": 300}, None)
|
||||||
|
assert verdict == "PASS"
|
||||||
|
assert metrics["expR_delta_vs_null"] >= 0.05
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_gate_experiments_unaffected_by_null():
|
||||||
|
# exit-style runs (variant rarely exactly 0.0): null share ~0 -> vs_null
|
||||||
|
# equals delta and the old semantics hold.
|
||||||
|
runs = [{"baseline_r": -0.2, "variant_r": -0.1, "detail": {}} for _ in range(400)]
|
||||||
|
verdict, metrics, n = evaluate(runs, {"expR_delta_min": 0.05, "min_n": 300}, None)
|
||||||
|
assert verdict == "PASS"
|
||||||
@@ -194,3 +194,55 @@ def test_no_quote_coverage_is_skipped_not_kept():
|
|||||||
v, d = iv_spread_expansion_gate(store.con, f, baseline=0.4,
|
v, d = iv_spread_expansion_gate(store.con, f, baseline=0.4,
|
||||||
params={"baseline_minutes": 10, "max_expansion_ratio": 2.0})
|
params={"baseline_minutes": 10, "max_expansion_ratio": 2.0})
|
||||||
assert v is None and d.get("skipped") == "no_quote_coverage"
|
assert v is None and d.get("skipped") == "no_quote_coverage"
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------- H6: quote-size imbalance
|
||||||
|
def _seed_sized_quotes(store, symbol, fill_utc, sizes):
|
||||||
|
"""sizes: list of (bid_size, ask_size) at 15s cadence ending just before fill."""
|
||||||
|
from datetime import timedelta
|
||||||
|
_seed_quotes(store, "_ddl_only_", []) # ensure table
|
||||||
|
for i, (bs, asz) in enumerate(sizes):
|
||||||
|
store.con.execute(
|
||||||
|
"INSERT INTO quote_snapshots (symbol, ts, bid_price, bid_size, ask_price, ask_size) "
|
||||||
|
"VALUES (?,?,?,?,?,?)",
|
||||||
|
[symbol, fill_utc - timedelta(seconds=15 * (len(sizes) - i)), 10.0, bs, 10.02, asz])
|
||||||
|
|
||||||
|
|
||||||
|
def test_bid_heavy_book_is_kept():
|
||||||
|
from datetime import timezone
|
||||||
|
from storage.event_store import EventStore
|
||||||
|
from research.experiments.replay import iv_quote_imbalance_gate
|
||||||
|
store = EventStore(":memory:")
|
||||||
|
fill_utc = datetime(2026, 6, 10, 13, 40, tzinfo=timezone.utc)
|
||||||
|
_seed_sized_quotes(store, "AAA", fill_utc, [(900, 100)] * 8) # imbalance +0.8
|
||||||
|
f = _fire()
|
||||||
|
f = type(f)(**{**f.__dict__, "fill_ts": datetime(2026, 6, 10, 9, 40)})
|
||||||
|
v, d = iv_quote_imbalance_gate(store.con, f, baseline=0.4,
|
||||||
|
params={"baseline_minutes": 10, "min_book_imbalance": 0.0})
|
||||||
|
assert v == 0.4 and d["flagged"] is False and d["book_imbalance"] > 0.5
|
||||||
|
|
||||||
|
|
||||||
|
def test_ask_heavy_book_is_zeroed():
|
||||||
|
from datetime import timezone
|
||||||
|
from storage.event_store import EventStore
|
||||||
|
from research.experiments.replay import iv_quote_imbalance_gate
|
||||||
|
store = EventStore(":memory:")
|
||||||
|
fill_utc = datetime(2026, 6, 10, 13, 40, tzinfo=timezone.utc)
|
||||||
|
_seed_sized_quotes(store, "AAA", fill_utc, [(100, 900)] * 8) # imbalance -0.8
|
||||||
|
f = _fire()
|
||||||
|
f = type(f)(**{**f.__dict__, "fill_ts": datetime(2026, 6, 10, 9, 40)})
|
||||||
|
v, d = iv_quote_imbalance_gate(store.con, f, baseline=0.4,
|
||||||
|
params={"baseline_minutes": 10, "min_book_imbalance": 0.0})
|
||||||
|
assert v == 0.0 and d["flagged"] is True and d["book_imbalance"] < -0.5
|
||||||
|
|
||||||
|
|
||||||
|
def test_ofi_no_coverage_is_skipped():
|
||||||
|
from storage.event_store import EventStore
|
||||||
|
from research.experiments.replay import iv_quote_imbalance_gate
|
||||||
|
store = EventStore(":memory:")
|
||||||
|
_seed_quotes(store, "OTHER", [])
|
||||||
|
f = _fire()
|
||||||
|
f = type(f)(**{**f.__dict__, "fill_ts": datetime(2026, 6, 10, 9, 40)})
|
||||||
|
v, d = iv_quote_imbalance_gate(store.con, f, baseline=0.4,
|
||||||
|
params={"baseline_minutes": 10, "min_book_imbalance": 0.0})
|
||||||
|
assert v is None and d.get("skipped") == "no_quote_coverage"
|
||||||
Reference in new issue
Block a user