momentum devandClaude Fable 5
849bd476f9
Null (the H2 lesson): zeroing share f of a negative baseline mechanically yields expR_delta = f*|mean baseline|; a PASS on expR_delta_min now also requires expR_delta_vs_null >= the same margin whenever zeroed_share >= 5%. Exposure-shrinking can no longer masquerade as selection; non-gate experiments unaffected. Honest disclosure: regime_meta_v1's FIRE-weighted pass would not clear this bar either — its standing rests on the session- level walk-forward OOS evidence, which no random gate reproduces. H6: CKS order-flow-imbalance from quote_snapshots bid/ask SIZES — size-weighted book imbalance over the 10-min pre-trigger window, zero ask-tilted books; skip-on-no-coverage; judged under the null from day one. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
225 lines
10 KiB
Python
225 lines
10 KiB
Python
"""Adjudicator — score accumulated runs against pre-registered bars and emit
|
||
verdicts.
|
||
|
||
Per running experiment: compute the metrics its ``bars``/``futility`` reference
|
||
over ``experiment_runs``, decide PASS/FAIL/KILLED/INCONCLUSIVE, append an
|
||
``experiment_verdicts`` row, and — ONLY on a terminal verdict (PASS/FAIL/
|
||
KILLED) — flip ``experiments.status``, emit an ``experiment_verdict`` event,
|
||
and send one Telegram ping. Because a terminal experiment is no longer
|
||
``running``, it is never re-adjudicated: the ping fires exactly once.
|
||
|
||
Metric vocabulary:
|
||
expR_delta mean(variant_r - baseline_r) (intervention edge)
|
||
rate_1r fraction of variant_r > 0
|
||
median_up_60m median(detail.max_upside) (pm1)
|
||
median_dd_60m median(detail.max_drawdown) (pm1)
|
||
big_gap_rate_1r rate_1r over runs with detail.gap_pct >= 0.5 (pm1)
|
||
|
||
Bar keys carry a ``_min`` (metric >= value) or ``_max`` (metric <= value)
|
||
suffix; ``min_n`` is the sample-size gate. n = runs with a non-null variant_r.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from datetime import datetime, timezone
|
||
from statistics import mean, median
|
||
|
||
from research.experiments import registry
|
||
from storage.event_schema import BaseEvent, EventMode, EventType
|
||
from storage.event_store import EventStore
|
||
|
||
_TERMINAL = {"PASS", "FAIL", "KILLED"}
|
||
_STATUS = {"PASS": "passed", "FAIL": "failed", "KILLED": "killed",
|
||
"INCONCLUSIVE": "inconclusive"}
|
||
|
||
|
||
def _f(x):
|
||
try:
|
||
return float(x)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Metrics
|
||
# --------------------------------------------------------------------------- #
|
||
def _metric(name: str, runs: list[dict]):
|
||
have_v = [r for r in runs if r["variant_r"] is not None]
|
||
if name == "expR_delta":
|
||
deltas = [r["variant_r"] - r["baseline_r"] for r in runs
|
||
if r["variant_r"] is not None and r["baseline_r"] is not None]
|
||
return mean(deltas) if deltas else None
|
||
if name == "rate_1r":
|
||
return mean(1.0 if r["variant_r"] > 0 else 0.0 for r in have_v) if have_v else None
|
||
if name == "median_up_60m":
|
||
xs = [(_f((r["detail"] or {}).get("max_upside"))) for r in runs]
|
||
xs = [x for x in xs if x is not None]
|
||
return median(xs) if xs else None
|
||
if name == "median_dd_60m":
|
||
xs = [(_f((r["detail"] or {}).get("max_drawdown"))) for r in runs]
|
||
xs = [x for x in xs if x is not None]
|
||
return median(xs) if xs else None
|
||
if name == "big_gap_rate_1r":
|
||
sub = [r for r in have_v if (_f((r["detail"] or {}).get("gap_pct")) or 0.0) >= 0.5]
|
||
return mean(1.0 if r["variant_r"] > 0 else 0.0 for r in sub) if sub else None
|
||
return None
|
||
|
||
|
||
def _metric_name(key: str) -> str:
|
||
for suf in ("_min", "_max"):
|
||
if key.endswith(suf):
|
||
return key[: -len(suf)]
|
||
return key
|
||
|
||
|
||
def _meets(key: str, value, metrics: dict) -> bool:
|
||
m = metrics.get(_metric_name(key))
|
||
if m is None:
|
||
return False
|
||
if key.endswith("_min"):
|
||
return m >= value
|
||
if key.endswith("_max"):
|
||
return m <= value
|
||
return False
|
||
|
||
|
||
def evaluate(runs: list[dict], bars: dict, futility: dict | None):
|
||
"""Return (verdict, metrics, n). Pure — no DB.
|
||
|
||
RANDOM-GATE NULL (2026-07-17, the orderflow-H2 lesson): a keep-or-zero
|
||
gate zeroing share f of a negative-baseline population mechanically
|
||
produces expR_delta = f * |mean baseline| — exposure-shrinking that
|
||
masquerades as selection. When an experiment has a material zeroed share,
|
||
a PASS on ``expR_delta_min`` additionally requires ``expR_delta_vs_null``
|
||
(delta minus that mechanical null) to clear the same margin. Non-gate
|
||
experiments (zeroed share ~0) are unaffected: vs_null == delta."""
|
||
n = sum(1 for r in runs if r["variant_r"] is not None)
|
||
referenced = {_metric_name(k) for k in list(bars) + list(futility or {})
|
||
if k != "min_n"}
|
||
metrics = {name: _metric(name, runs) for name in referenced}
|
||
|
||
if "expR_delta" in metrics and metrics["expR_delta"] is not None:
|
||
both = [r for r in runs if r["variant_r"] is not None and r["baseline_r"] is not None]
|
||
zeroed = sum(1 for r in both
|
||
if r["variant_r"] == 0.0 and r["baseline_r"] != 0.0)
|
||
zero_share = (zeroed / len(both)) if both else 0.0
|
||
mean_base = (sum(r["baseline_r"] for r in both) / len(both)) if both else 0.0
|
||
null_delta = zero_share * (0.0 - mean_base) # what a RANDOM gate earns
|
||
metrics["zeroed_share"] = round(zero_share, 4)
|
||
metrics["null_delta"] = null_delta
|
||
metrics["expR_delta_vs_null"] = (metrics["expR_delta"] - null_delta
|
||
if zero_share >= 0.05 else metrics["expR_delta"])
|
||
|
||
if futility and n >= int(futility.get("min_n", 1 << 60)):
|
||
conds = [c for c in futility if c != "min_n"]
|
||
# _meets is False on a None metric, so futility never fires on no data.
|
||
if conds and all(_meets(c, futility[c], metrics) for c in conds):
|
||
return "KILLED", metrics, n
|
||
|
||
if n == 0:
|
||
return "INCONCLUSIVE", metrics, n
|
||
if n >= int(bars.get("min_n", 0)):
|
||
bar_conds = [c for c in bars if c != "min_n"]
|
||
# a bar whose metric is uncomputable (None) is not a DECISIVE failure —
|
||
# can't conclude, so INCONCLUSIVE rather than a false FAIL.
|
||
if any(metrics.get(_metric_name(c)) is None for c in bar_conds):
|
||
return "INCONCLUSIVE", metrics, n
|
||
if ("expR_delta_min" in bars
|
||
and metrics.get("expR_delta_vs_null") is not None
|
||
and metrics["expR_delta_vs_null"] < float(bars["expR_delta_min"])
|
||
and metrics.get("zeroed_share", 0.0) >= 0.05):
|
||
# gate cleared the raw bar but not the random-gate null — the
|
||
# "gain" is exposure-shrinking, not selection. Never a PASS.
|
||
return "FAIL", metrics, n
|
||
if bar_conds and all(_meets(c, bars[c], metrics) for c in bar_conds):
|
||
return "PASS", metrics, n
|
||
return "FAIL", metrics, n
|
||
return "INCONCLUSIVE", metrics, n
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# pm1 bridge — pm1's data lives in the labeler tables, not in fires.
|
||
# --------------------------------------------------------------------------- #
|
||
def backfill_pm1(store: EventStore, exp_id: str = "pm1_v1") -> int:
|
||
"""Backfill experiment_runs for pm1_v1 from outcome_labels JOIN setup_events
|
||
(one row per labeled premarket setup): variant_r = reached-1R ? +1 : -1,
|
||
baseline_r NULL (pm1 is a detector track, not an intervention on fires),
|
||
max_upside/max_drawdown/gap into detail. Idempotent (PK + ON CONFLICT)."""
|
||
con = store.con
|
||
if registry.get(con, exp_id) is None:
|
||
return 0
|
||
from research import labeler
|
||
rows = con.execute(
|
||
"SELECT l.max_upside_next_60m, l.max_drawdown_next_60m, "
|
||
"CASE WHEN l.reached_1r_before_minus_1r THEN 1 ELSE 0 END, "
|
||
"se.gap_pct, se.setup_id, se.symbol, se.session_date "
|
||
"FROM outcome_labels l JOIN setup_events se ON se.setup_id = l.setup_id "
|
||
"WHERE se.setup_version = ?", [labeler.PREMARKET_SETUP_VERSION]).fetchall()
|
||
now = datetime.now(timezone.utc)
|
||
inserted = 0
|
||
for up, dd, r1, gap, sid, sym, sd in rows:
|
||
detail = {"kind": "pm1", "max_upside": _f(up), "max_drawdown": _f(dd),
|
||
"gap_pct": _f(gap), "reached_1r": bool(r1)}
|
||
con.execute(
|
||
"INSERT INTO experiment_runs "
|
||
"(experiment_id, fire_id, symbol, session_date, baseline_r, variant_r, "
|
||
" detail, computed_at) VALUES (?, ?, ?, ?, NULL, ?, ?::jsonb, ?) "
|
||
"ON CONFLICT DO NOTHING",
|
||
[exp_id, str(sid), sym, sd, 1.0 if r1 else -1.0, json.dumps(detail), now])
|
||
inserted += 1
|
||
return inserted
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Adjudicate
|
||
# --------------------------------------------------------------------------- #
|
||
def _load_runs(con, exp_id: str) -> list[dict]:
|
||
return [{"baseline_r": r[0], "variant_r": r[1], "detail": r[2] or {}}
|
||
for r in con.execute(
|
||
"SELECT baseline_r, variant_r, detail FROM experiment_runs "
|
||
"WHERE experiment_id = ?", [exp_id]).fetchall()]
|
||
|
||
|
||
def _emit_terminal(store: EventStore, exp_id: str, verdict: str, n: int, metrics: dict) -> None:
|
||
icon = {"PASS": "✅", "FAIL": "❌", "KILLED": "\U0001f480"}.get(verdict, "ℹ️")
|
||
msg = f"{icon} experiment {exp_id}: {verdict} (n={n})"
|
||
store.emit(BaseEvent(
|
||
timestamp=datetime.now(timezone.utc), mode=EventMode.PAPER,
|
||
event_type=EventType.EXPERIMENT_VERDICT, correlation_id=exp_id, message=msg,
|
||
payload={"experiment_id": exp_id, "verdict": verdict, "n": n, "metrics": metrics}))
|
||
# gated + fire-and-forget (no-op when Telegram unconfigured)
|
||
from runtime.notifier import send_telegram
|
||
send_telegram(msg)
|
||
|
||
|
||
def adjudicate(store: EventStore, bridge_pm1: bool = True) -> list[dict]:
|
||
"""Adjudicate every running experiment. Backfills pm1 first (if registered),
|
||
writes a verdict row per experiment, and on a terminal verdict flips status +
|
||
emits event/ping exactly once. Returns the verdicts computed this pass."""
|
||
con = store.con
|
||
registry.apply_schema(con)
|
||
if bridge_pm1:
|
||
try:
|
||
backfill_pm1(store)
|
||
except Exception: # noqa: BLE001 — the pm1 tables may not exist on a dev DB
|
||
pass
|
||
out = []
|
||
now = datetime.now(timezone.utc)
|
||
for exp in registry.list_experiments(con, running_only=True):
|
||
runs = _load_runs(con, exp["id"])
|
||
verdict, metrics, n = evaluate(runs, exp["bars"] or {}, exp["futility"])
|
||
con.execute(
|
||
"INSERT INTO experiment_verdicts (experiment_id, as_of, n, metrics, verdict) "
|
||
"VALUES (?, ?, ?, ?::jsonb, ?) ON CONFLICT DO NOTHING",
|
||
[exp["id"], now, n, json.dumps(metrics), verdict])
|
||
if verdict in _TERMINAL:
|
||
note = ", ".join(f"{k}={round(v, 4) if isinstance(v, float) else v}"
|
||
for k, v in metrics.items())
|
||
con.execute(
|
||
"UPDATE experiments SET status=?, verdict_at=?, verdict_note=? WHERE id=?",
|
||
[_STATUS[verdict], now, f"{verdict} (n={n}): {note}", exp["id"]])
|
||
_emit_terminal(store, exp["id"], verdict, n, metrics)
|
||
out.append({"experiment_id": exp["id"], "verdict": verdict, "n": n, "metrics": metrics})
|
||
return out
|