dilution(016): unbreak nightly EDGAR refresh + pre-register filing-veto experiment

premarket_eval's shadow enrichment call site queried session_date = current_date
(wrong on UTC-server Postgres at night, so it never matched any symbols) and
shared one try-block for float_cache + dilution filings (a float_cache failure
silently skipped the dilution refresh too). Fix: compute the ET session date
in Python, split into two independent try-blocks, lift into a testable
_refresh_shadow_enrichment(con, session_date). float_cache now applies its own
DDL unconditionally on use instead of only when an API key is present.

Adds iv_dilution_gate (additive, mirrors iv_regime_gate's keep-or-zero shape
keyed on symbol instead of session_date, fail-open on unfetched symbols) plus
its once-per-run preload, and registers the pre-registered dilution_veto_v1
experiment via the existing experiments/*.json self-registering cohort. Live
veto stays OFF; this only wires the shadow measurement.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

experiments/dilution_veto_v1.json

@@ -0,0 +1,8 @@
+{
+  "id": "dilution_veto_v1",
+  "hypothesis": "Zeroing ORB fires on names with an S-1/S-3/424B filing in the trailing 90 days raises whole-population expR — the dilution-trap gaps fade as new shares hit.",
+  "population": {"min_entry_gap_pct": 0.10, "min_day_volume": 300000, "price_min": 2.0, "price_max": 20.0},
+  "intervention": {"kind": "dilution_gate", "lookback_days": 90},
+  "bars": {"expR_delta_min": 0.05, "min_n": 300},
+  "fidelity": {"atr_frac": 0.25, "spread_mult": 3, "adverse_ticks": 1}
+}

research/experiments/replay.py

@@ -452,6 +452,58 @@ def _load_regime_favorable(con, model_version: str) -> dict:
     return {_to_date(r[0]): bool(r[1]) for r in rows}
 
 
+# --------------------------------------------------------------------------- #
+# Dilution gate (symbol-level; plan 016) — keep-or-zero a fire based on
+# whether an S-1/S-3/424B* filing was announced on the symbol in the trailing
+# ``lookback_days``. Same keep-or-zero shape as iv_regime_gate above, but keyed
+# on (symbol, session_date) instead of session_date alone, so history is a
+# {symbol: [announced_date, ...]} map rather than a {session_date: bool} map.
+# --------------------------------------------------------------------------- #
+def _dilution_flagged(history: dict, symbol: str, session_date: date, days: int) -> bool:
+    """True if any filing for ``symbol`` was announced within
+    ``0 <= (session_date - announced_date).days <= days``."""
+    for announced in history.get(symbol, ()):
+        delta = (session_date - announced).days
+        if 0 <= delta <= days:
+            return True
+    return False
+
+
+def iv_dilution_gate(fire: "Fire", baseline: float | None, history: dict,
+                     params: dict) -> tuple[float | None, dict]:
+    """variant_r = 0.0 if the symbol has a flagged filing in the lookback
+    window, else KEEP baseline. A symbol absent from ``dilution_events`` reads
+    unflagged — the no-data case is fail-open (an unfetched symbol must not
+    count as vetoed), same discipline as iv_regime_gate's no-score case.
+
+    Coverage note: dilution_events is seeded from ~now forward (the fetcher
+    looks back 180 days per symbol, but only once that symbol has been
+    fetched), so early adjudications run with PARTIAL flag coverage.
+    Unfetched symbols read unflagged (fail-open), which biases early
+    expR_delta TOWARD zero, never inflates it — an early INCONCLUSIVE here is
+    not a refutation.
+    """
+    lookback_days = params.get("lookback_days", 90)
+    flagged = _dilution_flagged(history, fire.symbol, fire.session_date, lookback_days)
+    detail = {"kind": "dilution_gate", "flagged": flagged, "lookback_days": lookback_days}
+    if flagged:
+        return 0.0, detail
+    return baseline, detail
+
+
+def _load_dilution_history(con) -> dict:
+    """{symbol: [announced_date, ...]} over all of ``dilution_events`` —
+    loaded once per ``run()`` call for a dilution_gate experiment (not per
+    fire), mirroring ``_load_regime_favorable``."""
+    rows = con.execute("SELECT symbol, announced_at FROM dilution_events").fetchall()
+    out: dict = {}
+    for sym, announced_at in rows:
+        if not sym or announced_at is None:
+            continue
+        out.setdefault(sym.upper(), []).append(_to_date(announced_at))
+    return out
+
+
 # --------------------------------------------------------------------------- #
 # Rank (portfolio-level; receipts-only, starves without plan 005)
 # --------------------------------------------------------------------------- #
@@ -484,7 +536,8 @@ def _insert_run(con, exp_id: str, fire: Fire, row: dict) -> None:
          datetime.now(timezone.utc)])
 
 
-def _compute_fire(con, exp: dict, fire: Fire, favorable: dict | None = None) -> dict | None:
+def _compute_fire(con, exp: dict, fire: Fire, favorable: dict | None = None,
+                  dilution_history: dict | None = None) -> dict | None:
     kind = (exp["intervention"] or {}).get("kind")
     bars = query_minute_bars(con, fire.symbol, fire.session_date)
     if bars is None or not len(bars):
@@ -510,6 +563,14 @@ def _compute_fire(con, exp: dict, fire: Fire, favorable: dict | None = None) ->
         if fire.sim_baseline:
             d["source"] = "setup_sim"
         return {"baseline_r": baseline, "variant_r": v, "detail": d}
+    if kind == "dilution_gate":
+        # same shape as regime_gate above: needs the baseline BEFORE computing
+        # variant.
+        baseline = _sim_baseline_r(fire, bars_after, fidelity) if fire.sim_baseline else fire.baseline_r
+        v, d = iv_dilution_gate(fire, baseline, dilution_history or {}, params)
+        if fire.sim_baseline:
+            d["source"] = "setup_sim"
+        return {"baseline_r": baseline, "variant_r": v, "detail": d}
     if kind == "time_stop":
         v, d = iv_time_stop(fire, bars_after, trigger_high, params, fidelity)
     elif kind == "probe_add":
@@ -728,8 +789,12 @@ def run(store, since: str | None = None) -> int:
         favorable = None
         if kind == "regime_gate":
             favorable = _load_regime_favorable(con, (exp["intervention"] or {}).get("model_version"))
+        dilution_history = None
+        if kind == "dilution_gate":
+            dilution_history = _load_dilution_history(con)
         for fire in new_fires:
-            row = _compute_fire(con, exp, fire, favorable=favorable)
+            row = _compute_fire(con, exp, fire, favorable=favorable,
+                                dilution_history=dilution_history)
             if row is None:
                 continue
             _insert_run(con, exp["id"], fire, row)

research/ingestion/float_cache.py

@@ -63,13 +63,13 @@ def fetch_float(symbol: str, api_key: str, timeout: float = 10.0) -> dict | None
 
 def refresh_float_cache(con, symbols: list[str], api_key: str | None = None) -> dict:
     """Fetch float for symbols missing/stale in the cache. Returns a summary."""
+    ensure_schema(con)  # apply-on-use: table must exist even on a no-key/no-op call
     api_key = api_key or os.environ.get("FMP_API_KEY") or os.environ.get(
         "Financial_Modeling_Prep_Api_Key")
     out = {"fetched": 0, "skipped_fresh": 0, "failed": 0, "no_key": False}
     if not api_key:
         out["no_key"] = True
         return out
-    ensure_schema(con)
     now = datetime.now(timezone.utc).replace(tzinfo=None)
     calls = 0
     for sym in dict.fromkeys(s.upper() for s in symbols if s):
@@ -101,6 +101,7 @@ def refresh_float_cache(con, symbols: list[str], api_key: str | None = None) ->
 def get_float(con, symbol: str) -> int | None:
     """Cached float shares for a symbol, or None."""
     try:
+        ensure_schema(con)  # apply-on-use: readable even if nothing has written yet
         r = con.execute("SELECT float_shares FROM float_cache WHERE symbol=?",
                         [symbol.upper()]).fetchall()
         return int(r[0][0]) if r and r[0][0] else None

research/premarket_eval.py

@@ -24,6 +24,31 @@ HIST = os.path.join(os.path.dirname(__file__), "..", "data", "premarket_eval_his
 _HEADER = "utc_ts\tn\tdays\trate_1r\twilson_lb\tmedian_up_60m\tmedian_dd_60m\tbig_gap_1r\tpassed\n"
 
 
+def _refresh_shadow_enrichment(con, session_date) -> None:
+    """Shadow enrichment: float cache (FMP) + SEC dilution filings for the
+    session's actively-traded names. SHADOW ONLY — neither gates entries
+    (float refuted as a gate 2026-06-29; filings veto needs a pre-registered
+    bar first, plan 016). Two INDEPENDENT try-blocks: float failing must
+    never skip dilution, and vice versa (previously they shared one
+    try-block, so an UndefinedTable on float_cache silently skipped the
+    dilution refresh too)."""
+    syms = [r[0] for r in con.execute(
+        "SELECT symbol FROM minute_bars WHERE session_date = ? "
+        "GROUP BY symbol ORDER BY sum(volume) DESC LIMIT 30", [session_date]).fetchall()]
+    if not syms:
+        return
+    try:
+        from research.ingestion.float_cache import refresh_float_cache
+        print("float cache:", refresh_float_cache(con, syms))
+    except Exception as exc:  # noqa: BLE001 — enrichment never breaks the eval
+        print(f"float cache enrichment skipped: {exc}")
+    try:
+        from research.ingestion.sec_dilution import refresh_dilution_events
+        print("dilution filings:", refresh_dilution_events(con, syms))
+    except Exception as exc:  # noqa: BLE001 — enrichment never breaks the eval
+        print(f"dilution enrichment skipped: {exc}")
+
+
 def _metrics(con) -> dict:
     rows = con.execute(
         "SELECT l.max_upside_next_60m, l.max_drawdown_next_60m, "
@@ -121,14 +146,9 @@ def main() -> None:
     # actively-traded names. SHADOW ONLY — neither gates entries (float refuted
     # as a gate 2026-06-29; filings veto needs a pre-registered bar first).
     try:
-        from research.ingestion.float_cache import refresh_float_cache
-        from research.ingestion.sec_dilution import refresh_dilution_events
-        syms = [r[0] for r in con.execute(
-            "SELECT symbol FROM minute_bars WHERE session_date = current_date "
-            "GROUP BY symbol ORDER BY sum(volume) DESC LIMIT 30").fetchall()]
-        if syms:
-            print("float cache:", refresh_float_cache(con, syms))
-            print("dilution filings:", refresh_dilution_events(con, syms))
+        from zoneinfo import ZoneInfo
+        session_date = datetime.now(ZoneInfo("America/New_York")).date()
+        _refresh_shadow_enrichment(con, session_date)
     except Exception as exc:  # noqa: BLE001 — enrichment never breaks the eval
         print(f"float/dilution enrichment skipped: {exc}")
     labeler.build_premarket(con, rebuild=True)          # rebuild from the grown DB

tests/unit/experiments/test_dilution_gate.py

@@ -0,0 +1,150 @@
+"""iv_dilution_gate (plan 016): keep-or-zero a fire on whether the symbol has
+an S-1/S-3/424B* filing in the trailing lookback window, plus the registered
+dilution_veto_v1 experiment running end-to-end on an isolated store. Mirrors
+test_regime_gate.py's keep-or-zero shape, keyed on (symbol, session_date)
+instead of session_date alone."""
+
+import os
+from datetime import date, datetime, timedelta, timezone
+
+from research.experiments import adjudicate, registry, replay
+from research.experiments.replay import Fire, iv_dilution_gate
+
+REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
+SESS = date(2026, 6, 11)
+
+
+def _fire(symbol="AAA", session_date=SESS, baseline=None):
+    return Fire(fire_id=f"f-{symbol}", symbol=symbol, session_date=session_date, fill_ts=None,
+               entry_price=10.0, stop_price=9.0, exit_price=None, baseline_r=baseline,
+               trigger_price=10.0)
+
+
+# --------------------------------------------------------------------------- #
+# iv_dilution_gate (pure)
+# --------------------------------------------------------------------------- #
+def test_dilution_gate_keeps_and_zeros():
+    # a filing 10 days before the session, well inside a 90-day lookback -> zeroed
+    history = {"AAA": [SESS - timedelta(days=10)]}
+    v, d = iv_dilution_gate(_fire("AAA"), 0.5, history, {"lookback_days": 90})
+    assert v == 0.0
+    assert d["kind"] == "dilution_gate" and d["flagged"] is True and d["lookback_days"] == 90
+
+    # symbol absent from dilution_events entirely (no-data case) -> KEPT, fail-open
+    v, d = iv_dilution_gate(_fire("OTHER"), 0.5, history, {"lookback_days": 90})
+    assert v == 0.5 and d["flagged"] is False
+
+    # filing OLDER than the lookback window -> kept
+    history_old = {"AAA": [SESS - timedelta(days=120)]}
+    v, d = iv_dilution_gate(_fire("AAA"), 0.5, history_old, {"lookback_days": 90})
+    assert v == 0.5 and d["flagged"] is False
+
+
+# --------------------------------------------------------------------------- #
+# expR_delta sign — proves adjudication rewards removing LOSERS, not just
+# removing trades (plan 011 discipline, copied for the dilution gate).
+# --------------------------------------------------------------------------- #
+def test_dilution_gate_expR_delta_sign():
+    symbols = ["AAA", "BBB", "CCC"]
+    # AAA/BBB flagged (10d-old filing), CCC clean
+    history = {"AAA": [SESS - timedelta(days=10)], "BBB": [SESS - timedelta(days=10)]}
+
+    # flagged names are net LOSERS -> removing them raises expR_delta (positive)
+    baselines = [-1.0, -0.5, 2.0]
+    runs = []
+    for sym, b in zip(symbols, baselines):
+        v, det = iv_dilution_gate(_fire(sym), b, history, {"lookback_days": 90})
+        runs.append({"baseline_r": b, "variant_r": v, "detail": det})
+    _, metrics, _ = adjudicate.evaluate(runs, {"expR_delta_min": 0.0, "min_n": 1}, None)
+    assert metrics["expR_delta"] > 0, metrics
+
+    # flagged names are net WINNERS -> expR_delta negative
+    baselines2 = [1.0, 0.5, -2.0]
+    runs2 = []
+    for sym, b in zip(symbols, baselines2):
+        v, det = iv_dilution_gate(_fire(sym), b, history, {"lookback_days": 90})
+        runs2.append({"baseline_r": b, "variant_r": v, "detail": det})
+    _, metrics2, _ = adjudicate.evaluate(runs2, {"expR_delta_min": 0.0, "min_n": 1}, None)
+    assert metrics2["expR_delta"] < 0, metrics2
+
+
+# --------------------------------------------------------------------------- #
+# registered experiment, end to end on an isolated store
+# --------------------------------------------------------------------------- #
+def _seed_daily(store, symbol, prev_close=8.0, sess_open=10.0, vol=1_000_000):
+    store.con.execute(
+        "INSERT INTO daily_bars (symbol, trade_date, open, close, volume) VALUES (?,?,?,?,?)",
+        [symbol, date(2026, 6, 10), prev_close - 0.5, prev_close, vol])
+    store.con.execute(
+        "INSERT INTO daily_bars (symbol, trade_date, open, close, volume) VALUES (?,?,?,?,?)",
+        [symbol, SESS, sess_open, sess_open, vol])
+
+
+def _seed_minutes(store, symbol, rows, start_min=30):
+    for i, (o, h, low, c) in enumerate(rows):
+        ts = datetime(2026, 6, 11, 13, start_min + i)
+        store.con.execute(
+            "INSERT INTO minute_bars (symbol, timestamp, session_date, is_premarket, "
+            "is_regular_hours, open, high, low, close, volume, vwap, quality_score) "
+            "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
+            [symbol, ts, SESS, False, True, o, h, low, c, 1000, c, 1.0])
+
+
+def _seed_setup(store, symbol, setup_id):
+    store.con.execute(
+        "INSERT INTO setup_events (setup_id, symbol, session_date, setup_time, setup_name, "
+        "setup_version, entry_reference_price, invalidation_price, gap_pct, relative_volume) "
+        "VALUES (?,?,?,?,?,?,?,?,?,?)",
+        [setup_id, symbol, SESS, datetime(2026, 6, 11, 13, 35), "orb", "v1",
+         10.0, 9.0, 0.12, 3.0])
+
+
+def test_dilution_experiment_registers_and_runs(store):
+    bars = [(10 + i * 0.1, 10.2 + i * 0.1, 9.99 + i * 0.1, 10.1 + i * 0.1) for i in range(10)]
+    for sym in ("AAA", "BBB"):
+        _seed_daily(store, sym)
+        _seed_minutes(store, sym, bars)
+        _seed_setup(store, sym, f"s-{sym}-1")
+
+    # AAA has a filing 10 days before the session -> flagged; BBB has none -> clean
+    store.con.execute(
+        "INSERT INTO dilution_events (id, symbol, event_type, announced_at) VALUES (?,?,?,?)",
+        ["acc-1", "AAA", "S-1", datetime.combine(SESS - timedelta(days=10), datetime.min.time())])
+
+    exp_id = registry.register(store.con, os.path.join(REPO, "experiments", "dilution_veto_v1.json"))
+    assert exp_id == "dilution_veto_v1"
+
+    n = replay.run(store)
+    assert n >= 2
+
+    aaa_row = store.con.execute(
+        "SELECT baseline_r, variant_r, detail FROM experiment_runs "
+        "WHERE experiment_id='dilution_veto_v1' AND fire_id='setup:s-AAA-1'").fetchone()
+    assert aaa_row is not None
+    baseline_r, variant_r, detail = aaa_row
+    assert variant_r == 0.0
+    assert detail["kind"] == "dilution_gate" and detail["flagged"] is True
+
+    bbb_row = store.con.execute(
+        "SELECT baseline_r, variant_r, detail FROM experiment_runs "
+        "WHERE experiment_id='dilution_veto_v1' AND fire_id='setup:s-BBB-1'").fetchone()
+    assert bbb_row is not None
+    bbb_baseline_r, bbb_variant_r, bbb_detail = bbb_row
+    assert bbb_variant_r == bbb_baseline_r    # unflagged -> kept
+    assert bbb_detail["flagged"] is False
+
+    out = adjudicate.adjudicate(store)
+    verdicts = {o["experiment_id"]: o["verdict"] for o in out}
+    assert "dilution_veto_v1" in verdicts
+    assert verdicts["dilution_veto_v1"] in {"PASS", "FAIL", "INCONCLUSIVE", "KILLED"}
+
+
+def test_dilution_veto_v1_is_in_the_self_registering_cohort(store):
+    """The declared cohort is every experiments/*.json (registry.register_cohort);
+    dropping the file there is the whole registration mechanism (no separate
+    id-list to edit)."""
+    path = os.path.join(REPO, "experiments", "dilution_veto_v1.json")
+    assert os.path.exists(path)
+    newly = registry.register_cohort(store.con)
+    assert "dilution_veto_v1" in newly
+    assert registry.get(store.con, "dilution_veto_v1") is not None

tests/unit/research/test_float_cache.py

@@ -0,0 +1,51 @@
+"""research/ingestion/float_cache.py (plan 016 step 2) — the module's own
+``CREATE TABLE IF NOT EXISTS float_cache`` DDL must apply on use so a fresh
+schema (like a first-boot production DB) never throws UndefinedTable, even
+when there's no FMP_API_KEY or no symbols to fetch. Runs against an ISOLATED
+throwaway Postgres schema (EventStore ":memory:"), never production, and
+makes no HTTP call (empty symbol list).
+
+Both FMP env var names are explicitly cleared via monkeypatch before the
+no-key test: an unrelated test elsewhere (test_admin_effective_config.py,
+via ``import admin_api``) triggers admin_api's own module-level bare
+``load_dotenv()`` during pytest COLLECTION, which walks up from cwd and can
+pick up the outer repo's real .env (this worktree is nested under it) —
+polluting os.environ for the rest of the session. Pre-existing, out of scope
+for this plan; this test must not depend on ambient environment either way."""
+
+from __future__ import annotations
+
+from storage.event_store import EventStore
+from research.ingestion.float_cache import get_float, refresh_float_cache
+
+
+def test_refresh_creates_table_and_is_a_noop_on_empty_symbols():
+    s = EventStore(":memory:")
+    try:
+        out = refresh_float_cache(s.con, [], api_key="x")
+        assert out == {"fetched": 0, "skipped_fresh": 0, "failed": 0, "no_key": False}
+        rows = s.con.execute("SELECT count(*) FROM float_cache").fetchall()
+        assert rows[0][0] == 0
+    finally:
+        s.close()
+
+
+def test_refresh_creates_table_even_without_an_api_key(monkeypatch):
+    monkeypatch.delenv("FMP_API_KEY", raising=False)
+    monkeypatch.delenv("Financial_Modeling_Prep_Api_Key", raising=False)
+    s = EventStore(":memory:")
+    try:
+        out = refresh_float_cache(s.con, ["AAA"], api_key=None)
+        assert out["no_key"] is True
+        rows = s.con.execute("SELECT count(*) FROM float_cache").fetchall()
+        assert rows[0][0] == 0
+    finally:
+        s.close()
+
+
+def test_get_float_is_a_noop_on_a_fresh_schema():
+    s = EventStore(":memory:")
+    try:
+        assert get_float(s.con, "AAA") is None
+    finally:
+        s.close()

tests/unit/research/test_premarket_eval_enrichment.py

@@ -0,0 +1,72 @@
+"""research/premarket_eval.py::_refresh_shadow_enrichment (plan 016) — the
+nightly float/dilution enrichment call site was broken two ways: a
+``session_date = current_date`` query (wrong on UTC-server Postgres at night)
+and a shared try-block where a float_cache failure silently skipped the
+dilution refresh too. This test runs against an ISOLATED throwaway Postgres
+schema (EventStore ":memory:"), never the production DB (mirrors
+tests/unit/experiments/conftest.py), and never makes a live HTTP call — both
+enrichment functions are monkeypatched."""
+
+from __future__ import annotations
+
+from datetime import date, datetime
+
+import pytest
+
+from research.premarket_eval import _refresh_shadow_enrichment
+from storage.event_store import EventStore
+
+SESS = date(2026, 6, 11)
+
+
+@pytest.fixture
+def store():
+    s = EventStore(":memory:")
+    yield s
+    s.close()
+
+
+def _seed_minute_bar(store, symbol, session_date, volume):
+    store.con.execute(
+        "INSERT INTO minute_bars (symbol, timestamp, session_date, is_premarket, "
+        "is_regular_hours, open, high, low, close, volume, vwap, quality_score) "
+        "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
+        [symbol, datetime(session_date.year, session_date.month, session_date.day, 13, 35),
+         session_date, False, True, 10.0, 10.1, 9.9, 10.0, volume, 10.0, 1.0])
+
+
+def test_dilution_still_called_when_float_raises(store, monkeypatch):
+    _seed_minute_bar(store, "AAA", SESS, 1_000_000)
+    _seed_minute_bar(store, "BBB", SESS, 500_000)
+    # a different session_date's bars must not leak into the query
+    _seed_minute_bar(store, "ZZZ", date(2026, 6, 10), 9_999_999)
+
+    def _raise_float(con, symbols):
+        raise RuntimeError("relation \"float_cache\" does not exist")
+
+    calls = []
+
+    def _record_dilution(con, symbols):
+        calls.append((con, list(symbols)))
+        return {"symbols": len(symbols), "filings": 0}
+
+    monkeypatch.setattr("research.ingestion.float_cache.refresh_float_cache", _raise_float)
+    monkeypatch.setattr("research.ingestion.sec_dilution.refresh_dilution_events", _record_dilution)
+
+    _refresh_shadow_enrichment(store.con, SESS)
+
+    assert len(calls) == 1
+    seen_con, seen_symbols = calls[0]
+    assert seen_con is store.con
+    # highest-volume first, only the seeded session's symbols
+    assert seen_symbols == ["AAA", "BBB"]
+
+
+def test_no_symbols_is_a_noop(store, monkeypatch):
+    called = {"float": False, "dilution": False}
+    monkeypatch.setattr("research.ingestion.float_cache.refresh_float_cache",
+                        lambda con, syms: called.__setitem__("float", True))
+    monkeypatch.setattr("research.ingestion.sec_dilution.refresh_dilution_events",
+                        lambda con, syms: called.__setitem__("dilution", True))
+    _refresh_shadow_enrichment(store.con, SESS)  # no minute_bars seeded
+    assert called == {"float": False, "dilution": False}