From 44ff36a669c80d12ea753f0e9fb9d57da2fecffb Mon Sep 17 00:00:00 2001 From: momentum dev Date: Wed, 8 Jul 2026 10:34:46 -0400 Subject: [PATCH] Population gate audit: static overnight gap everywhere + full band MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-07-08 audit (owner: "figure out if there is some kind of logic or audit issue") found BOTH entry paths feeding the population gate the WRONG gap. The fast path used the trigger book's dynamic gap (latest price vs prev close), so intraday rallies masqueraded as gappers: TVRD (6.0% true gap), SKIN (2.2%, filled at $0.98), JLHL (4.3%) all traded outside the validated population. The slow path used max(gap, intraday) with "open" taken from the first lookback-window bar, and had no upper band at all — dead-band names SEER (38.6%) and IOTR (40.7%) entered. Fixes: candidates carry gap_open_pct (session open vs prev close); trigger eligibility + submit_breakout_now use it exclusively once the session is open; _population_reject now implements the full validated band (10-35% OR volume-verified >=150% squeeze, $1-20 price band) for both paths; evaluator gap uses the session open; signals carry relative_volume for squeeze verification. Plus momentum-testpg.service so the test-isolation Postgres survives reboots. Co-Authored-By: Claude Opus 4.8 --- research/ingestion/signals.py | 4 ++- run_live_paper.py | 12 +++++++- runtime/triggers.py | 14 +++++++-- storage/projections.py | 1 + strategy/evaluation/setup_evaluator.py | 8 +++++ tests/unit/test_trading_execution.py | 21 +++++++++++++ tests/unit/test_triggers.py | 15 ++++++++++ trading_execution.py | 41 +++++++++++++++++++++----- 8 files changed, 103 insertions(+), 13 deletions(-) diff --git a/research/ingestion/signals.py b/research/ingestion/signals.py index 2da5f28..0220748 100644 --- a/research/ingestion/signals.py +++ b/research/ingestion/signals.py @@ -18,10 +18,11 @@ SCANNER_VERSION = "gapper-v2" class GapperCandidate: symbol: str price: float - gap_pct: float + gap_pct: float # DYNAMIC: latest close vs prev close (display/rank) cumulative_volume: int relative_volume: float vwap: float | None + prev_close: float | None = None # for the STATIC overnight gap (open vs prev close) rank: int = 0 @@ -88,6 +89,7 @@ def scan_gappers( GapperCandidate( symbol=symbol, price=float(close), + prev_close=float(prev_close), gap_pct=gap_pct, cumulative_volume=int(cum_volume), relative_volume=rvol, diff --git a/run_live_paper.py b/run_live_paper.py index 0f042e7..a81186f 100644 --- a/run_live_paper.py +++ b/run_live_paper.py @@ -758,9 +758,17 @@ def main(argv: list[str] | None = None) -> int: day_open = float(rth.iloc[0]["open"]) except Exception: # noqa: BLE001 dollar_vol = 0.0 + # STATIC overnight gap (session open vs prev close) — the population + # definition. The scanner's g.gap_pct is DYNAMIC (latest price vs prev + # close): intraday rallies masqueraded as gappers through it (TVRD 6% + # true gap entered 2026-07-07; audit 2026-07-08). None until the open. + gap_open_pct = None + if day_open and g.prev_close and g.prev_close > 0: + gap_open_pct = (day_open / g.prev_close - 1.0) * 100.0 candidates.append({ "symbol": g.symbol, "gap_pct": g.gap_pct, + "gap_open_pct": gap_open_pct, "rvol": g.relative_volume, "trigger": hi, "stop": stop, @@ -864,7 +872,9 @@ def main(argv: list[str] | None = None) -> int: t.symbol, t.trigger, t.stop, last_price=t.price, cum_volume=t.cum_volume, halted=_recently_halted(t.symbol), day_open=t.day_open, rank=t.rank, - gap_pct=(t.gap_pct / 100.0) if t.gap_pct else None, # book stores % units + # STATIC overnight gap only — never the dynamic book gap + gap_pct=(t.gap_open_pct / 100.0) if t.gap_open_pct is not None else None, + rvol=t.rvol, price=t.price, ) if res.get("ok"): book.mark_fired(t.symbol) diff --git a/runtime/triggers.py b/runtime/triggers.py index 2e5c99b..022c4eb 100644 --- a/runtime/triggers.py +++ b/runtime/triggers.py @@ -53,7 +53,8 @@ def _synced(method): @dataclass class ArmedTrigger: symbol: str - gap_pct: float = 0.0 + gap_pct: float = 0.0 # DYNAMIC gap (latest vs prev close) — display/rank only + gap_open_pct: float | None = None # STATIC overnight gap (session open vs prev close) rvol: float = 0.0 trigger: float | None = None # opening-range high (entry breakout level) stop: float | None = None # opening-range low, cushioned (protective stop) @@ -122,10 +123,15 @@ class ArmedTriggerBook: _lock: object = field(default_factory=threading.RLock, repr=False, compare=False) def _gap_ok(self, t: ArmedTrigger) -> bool: - if self.gap_min <= t.gap_pct <= self.gap_max: + # STATIC overnight gap (open vs prev close) defines the validated population. + # The dynamic gap let intraday rallies masquerade as gappers (TVRD 6% true + # gap traded 2026-07-07 as a "13%+ gapper"; audit 2026-07-08). Before the + # open (no session open yet) the dynamic gap is the only available proxy. + g = t.gap_open_pct if t.gap_open_pct is not None else t.gap_pct + if self.gap_min <= g <= self.gap_max: return True return (self.gap_squeeze_min > 0 - and t.gap_pct >= self.gap_squeeze_min + and g >= self.gap_squeeze_min and t.rvol >= self.gap_squeeze_rvol) def _eligible(self, t: ArmedTrigger) -> bool: @@ -163,6 +169,8 @@ class ArmedTriggerBook: rank += 1 t = self.triggers.get(sym) or ArmedTrigger(symbol=sym) t.gap_pct = float(c.get("gap_pct") or 0.0) + _go = c.get("gap_open_pct") + t.gap_open_pct = float(_go) if _go is not None else None t.rvol = float(c.get("rvol") or 0.0) t.trigger = c.get("trigger") t.stop = c.get("stop") diff --git a/storage/projections.py b/storage/projections.py index 50d6090..b0bff03 100644 --- a/storage/projections.py +++ b/storage/projections.py @@ -232,6 +232,7 @@ def query_ready_signals_snapshot(store, session_id: str | None = None) -> list[d "day_open": signal_data.get("day_open"), "cum_volume": signal_data.get("cum_volume"), "gap_pct": signal_data.get("gap_pct"), + "relative_volume": signal_data.get("relative_volume"), "timestamp": event.get("timestamp"), } return sorted( diff --git a/strategy/evaluation/setup_evaluator.py b/strategy/evaluation/setup_evaluator.py index d6535d3..13731ac 100644 --- a/strategy/evaluation/setup_evaluator.py +++ b/strategy/evaluation/setup_evaluator.py @@ -58,7 +58,14 @@ def evaluate_setup( bars = bars.reset_index(drop=True) price = float(bars["close"].iloc[-1]) + # SESSION open, not the first bar of the lookback window: with a 420-min + # premarket-inclusive window the first bar is an early-premarket print, which + # skewed the "overnight gap" fed to the population gate (2026-07-08 audit). open_price = float(bars["open"].iloc[0]) + if "is_regular_hours" in bars.columns: + _rth = bars[bars["is_regular_hours"].astype(bool)] + if len(_rth): + open_price = float(_rth["open"].iloc[0]) ref_close = previous_close if previous_close and previous_close > 0 else open_price gap_pct = (open_price - ref_close) / ref_close if ref_close > 0 else 0.0 intraday_change = (price - open_price) / open_price if open_price > 0 else 0.0 @@ -163,6 +170,7 @@ def evaluate_setup( "above_vwap": bool(above_vwap), "day_open": round(float(open_price), 4) if open_price > 0 else None, "cum_volume": float(bars["volume"].sum()), + "relative_volume": round(float(relative_volume), 3), # true overnight gap (open vs prev close) — input to the live POPULATION # gate: the year-long gapper replay showed the +EV boundary is the # population (gap>=10% + real volume), not entry timing diff --git a/tests/unit/test_trading_execution.py b/tests/unit/test_trading_execution.py index 0886e8c..c8ba964 100644 --- a/tests/unit/test_trading_execution.py +++ b/tests/unit/test_trading_execution.py @@ -834,3 +834,24 @@ def test_breakout_now_respects_dilution_veto(store): service = _service_with_catalyst(store, advisory, veto_enabled=True) res = service.submit_breakout_now("DILUT", trigger=14.0, stop=13.45, last_price=14.0) assert res["ok"] is False and res["skipped"] == "dilution_veto" + + +def test_population_gate_full_band_semantics(): + """2026-07-08 audit: the gate must implement the VALIDATED population — static + overnight gap in [10%, 35%] OR volume-verified squeeze >=150%; $1-20 price band. + TVRD (6%), SKIN (2.2%, $0.98), JLHL (4.3%) traded outside it; SEER (38.6%) and + IOTR (40.7%) came from the dead band.""" + from trading_execution import ExecutionSettings, TradingExecutionService + svc = TradingExecutionService.__new__(TradingExecutionService) + svc.settings = ExecutionSettings(min_entry_gap_pct=0.10, min_day_volume=300_000) + ok = lambda **kw: svc._population_reject(**{"gap_pct": None, "cum_volume": 500_000, **kw}) + assert ok(gap_pct=0.06) is not None # TVRD: below floor + assert ok(gap_pct=0.20) is None # core band trades + assert ok(gap_pct=0.40) is not None # SEER/IOTR: dead band + assert ok(gap_pct=2.78, rvol=40.0) is None # LHSW: verified squeeze + assert ok(gap_pct=2.78, rvol=0.4) is not None # split fake: rejected + assert ok(gap_pct=2.78) is not None # unverified squeeze: rejected + assert ok(gap_pct=0.20, price=0.98) is not None # SKIN: below $1 + assert ok(gap_pct=0.20, price=21.5) is not None # above $20 + assert ok(gap_pct=0.20, price=5.0) is None + assert ok(gap_pct=None) is None # fail-open on missing gap diff --git a/tests/unit/test_triggers.py b/tests/unit/test_triggers.py index 9c2571b..d6e5316 100644 --- a/tests/unit/test_triggers.py +++ b/tests/unit/test_triggers.py @@ -386,3 +386,18 @@ def test_squeeze_exception_off_by_default(): b = ArmedTriggerBook(gap_min=10, gap_max=35, rvol_min=2) b.arm([_cand("LHSW", gap=278.0, rvol=40.0)]) assert b.triggers["LHSW"].state == WEAK + + +def test_trigger_eligibility_uses_static_open_gap_not_dynamic(): + """2026-07-08 audit: TVRD's true overnight gap was 6% but its dynamic gap crossed + 10% on an intraday rally and it traded as a 'gapper'. Once the session open is + known, eligibility must use the STATIC gap.""" + b = ArmedTriggerBook(gap_min=10, gap_max=35, rvol_min=2) + c = _cand("TVRD", gap=13.2, rvol=6.0) # dynamic says 13.2%... + c["gap_open_pct"] = 6.0 # ...but the true overnight gap is 6% + b.arm([c]) + assert b.triggers["TVRD"].state == WEAK + c2 = _cand("BABU", gap=25.0, rvol=6.0) + c2["gap_open_pct"] = 19.0 # true gapper stays armed + b.arm([c2]) + assert b.triggers["BABU"].state == ARMED diff --git a/trading_execution.py b/trading_execution.py index 1e924bd..ed9105d 100644 --- a/trading_execution.py +++ b/trading_execution.py @@ -680,13 +680,33 @@ class TradingExecutionService: m = self._confirm_minute_of(ts) return m is not None and m > limit - def _population_reject(self, gap_pct, cum_volume) -> str | None: - """Reason string when the signal is OUTSIDE the +EV population (gap/volume floors); - None when tradeable. Fail-open: missing values never block.""" + def _population_reject(self, gap_pct, cum_volume, price=None, rvol=None) -> str | None: + """Reason string when the signal is OUTSIDE the +EV population (gap band, + volume floor, price band); None when tradeable. gap_pct MUST be the STATIC + overnight gap (session open vs prev close) — the 2026-07-08 audit found both + paths passing dynamic/intraday gaps, letting non-gappers (TVRD 6%, SKIN 2.2%, + JLHL 4.3%) and dead-band names (SEER 38.6%, IOTR 40.7%) into the book. + Fail-open: missing values never block (except the dead band, which requires + a VERIFIED squeeze to re-admit).""" if self.settings.min_entry_gap_pct > 0 and gap_pct is not None: try: - if float(gap_pct) < self.settings.min_entry_gap_pct: - return f"gap {float(gap_pct):.1%} < {self.settings.min_entry_gap_pct:.0%}" + g = float(gap_pct) + if g < self.settings.min_entry_gap_pct: + return f"gap {g:.1%} < {self.settings.min_entry_gap_pct:.0%}" + # dead band (35-150%): +0.02R at year scale — excluded. Above 150% + # only VOLUME-VERIFIED squeezes re-enter (real ones ran +0.63R; + # reverse-split fakes show rvol ~0.01). + if 0.35 < g < 1.50: + return f"gap {g:.0%} in dead band 35-150%" + if g >= 1.50 and (rvol is None or float(rvol) < 5.0): + return f"gap {g:.0%} squeeze unverified (rvol {rvol})" + except (TypeError, ValueError): + pass + if price is not None: + try: + px = float(price) + if px > 0 and not (1.0 <= px <= 20.0): + return f"price {px:.2f} outside $1-20 band" except (TypeError, ValueError): pass if self.settings.min_day_volume > 0 and cum_volume is not None: @@ -769,8 +789,10 @@ class TradingExecutionService: # POPULATION gate — the year-long +EV boundary: real gappers with real # volume. Fail-open on missing data; skips are logged and visible. - pop_reason = self._population_reject(signal.get("gap_pct"), - signal.get("cum_volume")) + pop_reason = self._population_reject( + signal.get("gap_pct"), signal.get("cum_volume"), + price=signal.get("entry_price"), + rvol=signal.get("relative_volume")) if pop_reason: self.store.emit( RiskRuleTriggeredEvent( @@ -1052,6 +1074,8 @@ class TradingExecutionService: day_open: float | None = None, rank: int = 0, gap_pct: float | None = None, + rvol: float | None = None, + price: float | None = None, ) -> dict: """Fire a breakout entry immediately on a LIVE price cross. @@ -1074,7 +1098,8 @@ class TradingExecutionService: if self._late_confirmation(self._now()): return {"ok": False, "skipped": "late_confirmation"} # POPULATION gate (same as the auto path): the year-long +EV boundary - if self._population_reject(gap_pct, cum_volume or None): + if self._population_reject(gap_pct, cum_volume or None, + price=price if price else last_price, rvol=rvol): return {"ok": False, "skipped": "population_below"} # CONCENTRATE-BY-RANK: only the top-N armed names may enter, and risk is