From 11b0fcd7b7f4933c568792b182323a3676738ad1 Mon Sep 17 00:00:00 2001 From: momentum dev Date: Mon, 14 Sep 2026 15:27:01 -0500 Subject: [PATCH] fix: read the session open from the first regular-hours bar, not bars[0] _population_reject computed the static open gap from bars["open"].iloc[0]. Premarket that is a PREMARKET bar, so the gap is fabricated. Live on 2026-09-14: FTFT opened 3.30 against a 2.88 previous close (+14.6%, well inside the tradeable 10-35% band) but its premarket bars started near 2.52, so the filter read -12.5% and benched it at 09:12 -- before the session had even opened. It never self-corrected. The frame keeps its premarket bars all session, so every later pass recomputed the same wrong number, and _transition only emits on change, so the symbol stayed out_of_population for the day. Several of the day's rejections show negative gaps (-12.5%, -17.4%, -1.9%) that look like the same artifact. Trading was unaffected -- the trigger path does not consult watcher state, and FTFT was still evaluated and rejected on its own merits (over_extended_day, it was already +59% off the open). The damage is diagnostic: valid candidates lose their criteria trail, which is the blank-reasoning-panel problem a previous commit set out to fix. Read the open off the first is_regular_hours bar; return None (fail open) when the session has not opened, since the static gap is not yet knowable. A frame with no session flag is treated as already regular-hours-only, which is what tests and replay hand it. Also drop the pm-range=N counter from step_arm's return string. The arm step never emits a module_tick -- verified against both 2026-09-11 and 2026-09-14, where only discovery, ingestion, enrichment and triggers do -- so it reached container stdout and nothing else. Adding a tick purely to carry it would put ~4k events/day back after the population filter had just removed 88% of them. pm_range_pct is already on the armed candidate rows for the board; persist it on the approval payload if it needs to be measurable per trade. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01A2LepdJfPCfK5Q8uznjFBM --- run_live_paper.py | 12 ++- runtime/watcher.py | 15 +++- .../runtime/test_watcher_population_gate.py | 73 ++++++++++++++++--- 3 files changed, 85 insertions(+), 15 deletions(-) diff --git a/run_live_paper.py b/run_live_paper.py index 14d488a..6d2c63e 100644 --- a/run_live_paper.py +++ b/run_live_paper.py @@ -1053,11 +1053,15 @@ def main(argv: list[str] | None = None) -> int: pass armed = sum(1 for t in book.triggers.values() if t.state == "armed") n_news = sum(1 for c in candidates if c.get("catalyst")) - n_pm = sum(1 for c in candidates - if (c.get("pm_range_pct") or 0.0) >= pm_cfg.min_range_pct) + # NOTE: no pm-range counter here. The arm step never emits a module_tick + # (verified against 2026-09-11 and 2026-09-14: only discovery, ingestion, + # enrichment and triggers do), so anything added to this string reaches + # container stdout and nothing else. Adding a tick just for it would put + # ~4k events/day back after the watcher filter removed 88% of them. The + # per-candidate pm_range_pct above is on the armed rows for the board; + # persist it on the approval payload if it needs to be measurable per trade. return (f"tracking={len(book.triggers)} armed={armed}" - + (f" news={n_news}" if n_news else "") - + (f" pm-range={n_pm}" if n_pm else "")) + + (f" news={n_news}" if n_news else "")) def step_quotes(): """Record NBBO bid/ask SIZES for tracked + held names (order-book-imbalance diff --git a/runtime/watcher.py b/runtime/watcher.py index c039826..d608e46 100644 --- a/runtime/watcher.py +++ b/runtime/watcher.py @@ -143,7 +143,20 @@ class Watcher: never silently bench a symbol. """ try: - open_px = float(bars["open"].iloc[0]) + # The SESSION open is the first REGULAR-HOURS bar's open. bars[0] is a + # PREMARKET bar before 09:30, which produces a fabricated gap: FTFT was + # benched 2026-09-14 as "gap -12.5%" when its true static open gap was + # +14.6%, and it never recovered because the frame keeps its premarket + # bars all session and _transition only fires on change. + if "is_regular_hours" in bars: + rth = bars[bars["is_regular_hours"] == True] # noqa: E712 — pandas mask + if len(rth) == 0: + return None # session has not opened — gap unknowable, fail open + open_px = float(rth["open"].iloc[0]) + else: + # no session flag: the frame is already regular-hours only (tests, + # replay). Using bars[0] is correct there. + open_px = float(bars["open"].iloc[0]) except Exception: # noqa: BLE001 return None prev_close = candidate.previous_close diff --git a/tests/unit/runtime/test_watcher_population_gate.py b/tests/unit/runtime/test_watcher_population_gate.py index 62e7d2f..2177aa4 100644 --- a/tests/unit/runtime/test_watcher_population_gate.py +++ b/tests/unit/runtime/test_watcher_population_gate.py @@ -49,16 +49,33 @@ class FakeProvider: return self._bars.get(symbol, pd.DataFrame()) -def _bars(open_px: float, n: int = 30, volume: int = 50_000) -> pd.DataFrame: - """n flat-ish minute bars starting at open_px.""" - return pd.DataFrame({ - "timestamp": pd.date_range("2026-09-10 13:30", periods=n, freq="1min"), - "open": [open_px] * n, - "high": [open_px * 1.01] * n, - "low": [open_px * 0.99] * n, - "close": [open_px] * n, - "volume": [volume] * n, - }) +def _bars(open_px: float, n: int = 30, volume: int = 50_000, + premarket_open: float | None = None) -> pd.DataFrame: + """n regular-hours minute bars from open_px, optionally preceded by premarket. + + ``premarket_open`` prepends 5 PREMARKET bars at that price. The session open + must still be read off the first REGULAR-HOURS bar — reading bars[0] instead + fabricates a gap (FTFT, 2026-09-14). + """ + rows = [] + if premarket_open is not None: + rows += [{"timestamp": pd.Timestamp("2026-09-10 09:00") + pd.Timedelta(minutes=i), + "is_regular_hours": False, "open": premarket_open, + "high": premarket_open * 1.01, "low": premarket_open * 0.99, + "close": premarket_open, "volume": volume} for i in range(5)] + rows += [{"timestamp": pd.Timestamp("2026-09-10 13:30") + pd.Timedelta(minutes=i), + "is_regular_hours": True, "open": open_px, + "high": open_px * 1.01, "low": open_px * 0.99, + "close": open_px, "volume": volume} for i in range(n)] + return pd.DataFrame(rows) + + +def _premarket_only(open_px: float, n: int = 10) -> pd.DataFrame: + """Bars before the session opens — the gap is not yet knowable.""" + return pd.DataFrame([{ + "timestamp": pd.Timestamp("2026-09-10 09:00") + pd.Timedelta(minutes=i), + "is_regular_hours": False, "open": open_px, "high": open_px * 1.01, + "low": open_px * 0.99, "close": open_px, "volume": 1000} for i in range(n)]) def _watcher(store, candidates, bars, population=None): @@ -161,3 +178,39 @@ def test_dead_band_gap_is_skipped(): avg_daily_volume=500_000)], {"MID": _bars(3.0)}, population=GAPPER) assert w.tick(SESSION).skipped == 1 + + +# --- the FTFT bug (2026-09-14) ------------------------------------------------ +# The gap was read off bars["open"].iloc[0], which premarket is a PREMARKET bar. +# FTFT opened 3.30 against a 2.88 previous close (+14.6%, comfortably tradeable) +# but its premarket bars started near 2.52, so the filter computed -12.5% and +# benched it. It never self-corrected: the frame keeps its premarket bars all +# session, and _transition only emits on change. + +def test_gap_is_measured_from_the_session_open_not_a_premarket_bar(): + """FTFT: premarket 2.52, session open 3.30, previous close 2.88 -> +14.6%.""" + store = FakeStore() + w = _watcher(store, [WatchCandidate("FTFT", previous_close=2.88, + avg_daily_volume=500_000)], + {"FTFT": _bars(3.30, premarket_open=2.52)}, population=GAPPER) + r = w.tick(SESSION) + assert r.skipped == 0 # +14.6% is inside the 10-35% band + assert r.evaluated == 1 + + +def test_premarket_bars_alone_fail_open(): + """Before the session opens the static gap is unknowable — never bench on it.""" + store = FakeStore() + w = _watcher(store, [WatchCandidate("FTFT", previous_close=2.88)], + {"FTFT": _premarket_only(2.52)}, population=GAPPER) + r = w.tick(SESSION) + assert r.skipped == 0 + assert store.count("SymbolStateChangedEvent") >= 0 # never out_of_population + + +def test_a_genuinely_flat_name_is_still_skipped_with_premarket_present(): + """The filter must still work when premarket bars exist — NOK opens flat.""" + store = FakeStore() + w = _watcher(store, [WatchCandidate("NOK", previous_close=4.0)], + {"NOK": _bars(4.0, premarket_open=3.6)}, population=GAPPER) + assert w.tick(SESSION).skipped == 1