Regime gate (plan 011): deploy-wire the shadow pipeline + on-demand admin trigger + prod backtest

The session-level metalabel P(our ORB profits today) — the one lever that
survives out-of-sample. Backtested on 266 prod sessions (walk-forward):

  trade-always (trained subset): -0.111R/session, 36% hit
  meta_v1 GATED (trades best ~32% of days): -0.004R/session, 48% hit
  delta +0.107R in-sample, +0.128R OOS (recent 80 sessions), improves across folds

It does NOT make the bot profitable — it takes a bleeding strategy to roughly
flat by sitting out the worst ~2/3 of sessions. threshold_v1 rule gates too
little to matter; meta_v1 (walk-forward GBM) is the signal. Leaky full-fit
(+0.61R) >> walk-forward confirms the discipline was load-bearing; the OOS
holding is the evidence the +0.078R is real, not look-ahead.

Changes:
- research/regime.py: batch minute-bar load per session (64k per-fire round-trips
  -> ~266 queries; fixed a >5min build to ~3min) + incremental label pass (skip
  already-labeled sessions; --rebuild forces full). Nightly cost now ~one new day.
- docker-compose.yml + truenas-compose.yml: scheduler runs build-features + score
  (both models) before `experiments run`, nightly. ALL-SHADOW — does NOT gate live
  trading (that stays owner-gated Step 5).
- admin_api.py + admin.html: POST /api/regime/run (detached) + GET /api/regime/status
  + a Run/Status button — the panel had no way to trigger this.
- plans/011 + docs/research/regime_notes.md (6-book distillation) + plans 008/009 docs.

Prod state (shadow, rollback = DROP regime_features/regime_scores + DELETE
experiments WHERE id LIKE 'regime%'): 266 feature rows, 532 scores, 2 experiments
registered.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.gitignore

@@ -59,3 +59,6 @@ momentum-rebuilt.zip
 archive/
 uv.lock
 .firecrawl/
+
+# large research PDFs (local only)
+books/

admin_api.py

@@ -21,6 +21,8 @@ import http.client
 import json
 import os
 import socket
+import subprocess
+import sys
 import urllib.request
 from datetime import datetime
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -33,6 +35,45 @@ _load_overrides()
 from runtime import env_overrides  # noqa: E402
 
 DOCKER_SOCK = os.environ.get("DOCKER_SOCK", "/var/run/docker.sock")
+# Regime backtest (plan 011) on-demand trigger — logs to the app-data volume so
+# GET /api/regime/status can tail it (the panel had no way to fire this).
+REGIME_LOG = os.environ.get(
+    "REGIME_RUN_LOG", os.path.join(os.path.dirname(__file__), "data", "regime_run.log"))
+_REGIME_DONE = "REGIME PIPELINE DONE"
+
+
+def run_regime_pipeline() -> int:
+    """Fire the full regime shadow pipeline DETACHED (it takes a few min): rebuild
+    session labels -> walk-forward score both models -> replay + adjudicate the
+    regime_gate experiments. Returns the child pid. Shadow only — never touches a
+    live order. stdout+stderr -> REGIME_LOG (truncated each run so status shows
+    the CURRENT run); a trailing marker lets status report completion."""
+    py = sys.executable
+    steps = " && ".join([
+        f"{py} -m research.regime build-features",
+        f"{py} -m research.regime score --model threshold_v1",
+        f"{py} -m research.regime score --model meta_v1",
+        f"{py} -m research.experiments run",
+        f"{py} -m research.experiments adjudicate",
+        f"echo {_REGIME_DONE}",
+    ])
+    os.makedirs(os.path.dirname(REGIME_LOG), exist_ok=True)
+    log = open(REGIME_LOG, "wb")  # noqa: SIM115 — handed to the detached child
+    proc = subprocess.Popen(["/bin/sh", "-c", steps], stdout=log,
+                            stderr=subprocess.STDOUT, start_new_session=True)
+    return proc.pid
+
+
+def regime_status() -> dict:
+    """Tail REGIME_LOG + whether the last run finished (the marker) — no DB query
+    needed; the adjudicate step prints each experiment's verdict into the log."""
+    try:
+        with open(REGIME_LOG, encoding="utf-8", errors="replace") as fh:
+            lines = fh.read().splitlines()
+    except FileNotFoundError:
+        return {"log_tail": [], "running": False, "ran": False}
+    return {"log_tail": lines[-40:], "ran": True,
+            "running": _REGIME_DONE not in lines[-3:] if lines else False}
 PROFILES_DIR = os.environ.get(
     "PROFILES_DIR", os.path.join(os.path.dirname(__file__), "data", "profiles"))
 ADMIN_TOKEN = os.environ.get("ADMIN_TOKEN", "")
@@ -282,6 +323,8 @@ class Handler(BaseHTTPRequestHandler):
             return self._send(200, profiles())
         if path == "/api/config/effective":
             return self._send(200, {"config": effective_config()})
+        if path == "/api/regime/status":
+            return self._send(200, regime_status())
         return self._send(404, {"error": "unknown route"})
 
     def do_POST(self):  # noqa: N802
@@ -345,6 +388,15 @@ class Handler(BaseHTTPRequestHandler):
                 return self._send(400, {"error": "mode must be paper|live"})
             return self._send(200, {"ok": True, "mode": mode, "overrides": merged,
                                     "note": "restart the loop to apply"})
+
+        if path == "/api/regime/run":
+            try:
+                pid = run_regime_pipeline()
+            except Exception as e:  # noqa: BLE001
+                return self._send(500, {"ok": False, "error": str(e)[:200]})
+            return self._send(200, {"ok": True, "pid": pid, "log": REGIME_LOG,
+                                    "note": "regime shadow pipeline started (~a few min); "
+                                            "poll GET /api/regime/status"})
         return self._send(404, {"error": "unknown route"})
 
     def log_message(self, *a):  # quiet

api/static/admin.html

@@ -75,6 +75,15 @@ color:var(--bg);padding:10px 20px;border-radius:10px;font-size:14px;opacity:0;tr
   <button class="danger" onclick="if(confirm('Switch the loop to LIVE trading with real money on next restart. Are you sure?'))setMode('live')">Go Live</button>
 </div>
 
+<h2>Research jobs</h2>
+<div class="card row">
+  <div style="flex:1"><b>Regime backtest <span class="mut" style="text-transform:none;letter-spacing:0">— plan 011, SHADOW</span></b>
+    <div class="mut">Rebuild session labels → walk-forward score both models → replay + adjudicate the regime_gate experiments. Runs a few min; does NOT touch live trading.</div></div>
+  <button class="sec" onclick="runRegime()">Run</button>
+  <button class="sec" onclick="regimeStatus()">Status</button>
+</div>
+<div class="card" id="regimeout" style="display:none"><pre id="regimelog" class="mut" style="white-space:pre-wrap;max-height:240px;overflow:auto;margin:0;font-size:12px"></pre></div>
+
 <h2>Secrets present</h2>
 <div class="card" id="secrets"></div>
 <div id="toast"></div>
@@ -87,6 +96,15 @@ function saveToken(){localStorage.setItem('admintok',$('#token').value);load()}
 async function api(p,opt={}){opt.headers={...(opt.headers||{}),'X-Admin-Token':tok(),'Content-Type':'application/json'};
 const r=await fetch(p,opt);if(r.status===401){toast('bad token');throw 0}return r.json()}
 function toast(m){const t=$('#toast');t.textContent=m;t.style.opacity=1;setTimeout(()=>t.style.opacity=0,2600)}
+async function runRegime(){toast('starting regime pipeline…');
+  try{const r=await api('/api/regime/run',{method:'POST',body:'{}'});
+    toast(r.ok?('started (pid '+r.pid+') — poll Status'):('failed: '+(r.error||'')));
+    setTimeout(regimeStatus,2000)}catch(e){toast('failed')}}
+async function regimeStatus(){const r=await api('/api/regime/status');
+  $('#regimeout').style.display='block';
+  $('#regimelog').textContent=(r.log_tail&&r.log_tail.length?r.log_tail.join('\n'):'(no run yet — click Run)')
+    +(r.running?'\n\n… running …':(r.ran?'\n\n[finished]':''));
+  if(r.running)setTimeout(regimeStatus,4000)}
 function dot(ok){return `<span class="dot ${ok===true?'ok':ok===false?'bad':'warn'}"></span>`}
 async function load(){
   const s=await api('/api/status');

deploy/docker/truenas-compose.yml

@@ -205,6 +205,14 @@ services:
             # NOT look-ahead (replay is a labeling job, not a live gate).
             python eod_replay.py || echo "[scheduler] eod_replay FAILED"
             python nightly_tune.py || echo "[scheduler] nightly_tune FAILED"
+            # Regime gate (plan 011, ALL-SHADOW): a session-level metalabel
+            # P(our ORB profits today). build-features labels each session from its
+            # OWN realized expR; score writes the WALK-FORWARD favorable flag the
+            # experiments' regime_gate intervention reads. Must run BEFORE
+            # experiments run. Does NOT gate live trading (that is owner-gated).
+            python -m research.regime build-features || echo "[scheduler] regime build-features FAILED"
+            python -m research.regime score --model threshold_v1 || echo "[scheduler] regime score threshold FAILED"
+            python -m research.regime score --model meta_v1 || echo "[scheduler] regime score meta FAILED"
             # Experiment OS (plan 006, ALL-SHADOW): replay running experiments over
             # newly-logged fires, then adjudicate against pre-registered bars.
             python -m research.experiments run || echo "[scheduler] experiments run FAILED"

docker-compose.yml

@@ -164,6 +164,14 @@ services:
             # NOT look-ahead (replay is a labeling job, not a live gate).
             python eod_replay.py || echo "[scheduler] eod_replay FAILED"
             python nightly_tune.py || echo "[scheduler] nightly_tune FAILED"
+            # Regime gate (plan 011, ALL-SHADOW): a session-level metalabel
+            # P(our ORB profits today). build-features labels each session from its
+            # OWN realized expR; score writes the WALK-FORWARD favorable flag the
+            # experiments' regime_gate intervention reads. Must run BEFORE
+            # experiments run. Does NOT gate live trading (that is owner-gated).
+            python -m research.regime build-features || echo "[scheduler] regime build-features FAILED"
+            python -m research.regime score --model threshold_v1 || echo "[scheduler] regime score threshold FAILED"
+            python -m research.regime score --model meta_v1 || echo "[scheduler] regime score meta FAILED"
             # Experiment OS (plan 006, ALL-SHADOW): replay running experiments over
             # newly-logged fires, then adjudicate against pre-registered bars.
             python -m research.experiments run || echo "[scheduler] experiments run FAILED"

docs/research/regime_notes.md

@@ -0,0 +1,55 @@
+# Regime-detection research notes — distilled for the momentum bot
+
+**Date:** 2026-07-15
+**Method:** 6 books in `books/market_regime_state_detection/` distilled by parallel readers (Ang & Timmermann NBER survey; Kritzman *Regime Shifts*; Zucchini/MacDonald/Langrock *HMM for Time Series*; Chan *Hands-On AI Trading* & *Quantitative Trading*; Kakkar et al *Trading Decoded*). Hamilton 1989 is a scanned image (no extractable text); its Markov-switching method is covered by the others.
+**Problem this feeds:** the bot is ~breakeven with no entry edge after honest fills; the one thing untried is **regime awareness** — it did well 07-08/09 (a runner regime) and sits out / bleeds otherwise. Goal: a FREE-data daily regime signal that gates/sizes ORB entries, harness-backtestable over ~260 sessions.
+
+---
+
+## The one insight that reframes everything (Chan's metalabeling)
+
+> "The key to successfully apply AI/ML to finance is to focus on **metalabeling** — finding the probability of profit of *your own* simple trading strategy, and **not** to use it to predict the market directly. Your own strategy's past track record is private; no one else is trying to predict it." — Chan, *Quantitative Trading*
+
+Don't detect abstract "market regime." **Predict `P(our ORB makes money today)`** from free pre-open features, using our OWN per-session expR (which the plan-006 harness already computes) as the label. This targets exactly what matters (does *our* edge work today), sidesteps the "high-vol lumps runner-days with crash-days" problem, reuses the harness, is free, and is a *private* signal nobody arbs away. Every practical book (Chan CPO, Trading Decoded RF) converges here; the academic survey (Ang-Timmermann) calls the same thing "regime-conditioned position sizing."
+
+## The method stack (what to build, in escalation order — ponytail)
+
+1. **Threshold baseline first** (Kritzman, Trading Decoded both insist): a trivial rule — e.g. `SPY 5d momentum > 0 AND SPY 20d vol < median`, or Kritzman **Financial Turbulence** (`sqrt((r−μ)Σ⁻¹(r−μ)')` on ~11 sector-ETF returns) below its 75th percentile. If the fancy model can't beat this net-of-costs out-of-sample, ship the rule.
+2. **Metalabel classifier** (the primary): `GradientBoostingClassifier` / `RandomForest` (free sklearn — the books' `predictnow.ai`/`gbdt` replaced) predicting `P(profitable ORB session)` from ~5–8 free features. Gate/size on the probability.
+3. **HMM regime as a FEATURE** (not the whole model): a 2-state Markov-switching state on SPY (`statsmodels MarkovRegression(spy_returns, k_regimes=2, switching_variance=True)`) or on the turbulence series — one column feeding the classifier, or a standalone gate to compare.
+
+## Free features (all pre-open, all we have or can ingest free)
+
+| Feature | Source | Note |
+|---|---|---|
+| SPY 5d/20d momentum + 20d realized vol (VIX proxy) | already ingesting SPY (plan 009) | the level-1 input every book uses |
+| IWM (small-cap index) + sector-ETF momentum | Alpaca/yfinance, **free** | small-caps specifically; sector rotation |
+| Kritzman turbulence (Mahalanobis on ~11 sector ETFs) | sector-ETF daily returns, **free** | "prices behaving uncharacteristically" |
+| Breadth = % of the gapper universe green / held-VWAP yesterday | our own stored data | the SIGNED feature that breaks vol-symmetry |
+| **Bot's own trailing-N-day realized expR + hit-rate** | our harness | the most on-target feature — a hot/cold-streak detector on our OWN edge |
+| VIX level + 1d change | CBOE/yfinance, **free** | the vol axis directly |
+
+Keep to **≤5–8 features** — only ~260 sessions, ~a third fewer "runner days." More features = curve-fit.
+
+## The load-bearing discipline (or the whole thing is a lie)
+
+- **Filtered / walk-forward / purged only.** Only a regime label computed from data **≤ today** is tradeable. Smoothed HMM probs and full-sample fits **peek at the future**. Sanity check every book demanded: *if a smoothed-gated backtest crushes the filtered-gated one, your edge is look-ahead.* Use purged/embargoed walk-forward CV (drop the boundary day) — the same leakage-safe split used in `plans/008`.
+- **Label is same-day-outcome, features are strictly pre-open.** The session's expR (label) uses open-knowable exits; every feature is lagged to the prior close.
+- **Out-of-sample split is mandatory.** Fit on the older half, apply to the recent half; confirm the favorable-regime expR advantage survives. "In-sample regimes need not persist out-of-sample" (Ang-Timmermann) — if the edge evaporates OOS, the regimes were in-sample noise.
+- **Acceptance = NET expR improves across EVERY walk-forward fold**, not AUC. A 52.8% classifier can still be net-negative (Chan Hands-On's own OOS accuracy was 52.77%).
+- **Let the DATA pick the sign.** Ang-Timmermann finds low-vol = high-Sharpe for a *long-index* investor — but a long-*dispersion* small-cap momentum strategy likely inverts this (runners need volatility). Measure conditional expR per regime; never assume high-vol = bad.
+- **Small-sample humility.** 2 states (don't LR-test the count), shallow trees (`max_depth ≤ 3`), report bootstrap/posterior bands on the regime probs, deflate any OOS Sharpe by the number of tweaks tried (Deflated Sharpe Ratio).
+- **Regimes lag.** A daily gate enters the runner regime a day or two late and exits late — structural. It sits out *some* good days and takes *some* bad ones. The bar is net improvement, not perfection.
+
+## Honest prior
+
+This is the most promising lever of the session — it targets the bot's own edge, uses free data, reuses the harness, and directly explains the 07-08/09-vs-today swing. **But 260 sessions is a small sample**, so the real risk is curve-fitting a regime that doesn't persist. The threshold-baseline-first rule, purged walk-forward, and OOS split are the guards. If, after all that, the gated expR doesn't clearly beat trade-always out-of-sample → the honest finding is "no exploitable regime signal in free data at this sample size," and we've learned it cheaply.
+
+## Sources (in `books/market_regime_state_detection/`)
+
+- Ang & Timmermann (2011), *Regime Changes and Financial Markets*, NBER WP 17182 — the survey + the sizing template + every caveat.
+- Kritzman et al, *Regime Shifts: Implications for Dynamic Strategies* — Financial Turbulence + absorption ratio.
+- Zucchini, MacDonald, Langrock, *Hidden Markov Models for Time Series* — the estimation recipe (forward-filter, BIC state selection, why filtered ≠ smoothed).
+- Chan et al, *Hands-On AI Trading* — the exact `statsmodels MarkovRegression` daily-gate recipe (Example 4) + the 4-year-lookback calibration.
+- Chan, *Quantitative Trading* — Conditional Parameter Optimization + the metalabeling principle.
+- Kakkar, Sahni, Shanmugamani, *Trading Decoded* — the RandomForest regime-as-feature recipe + purged walk-forward CV + the "beat the trivial rule or ship the rule" discipline.

plans/008-conversion-signal-model.md

@@ -0,0 +1,143 @@
+# Plan 008: Measure, then model — does the labeled data contain a separable conversion signal, and can a model (+ an Ollama catalyst feature) find it?
+
+> **Executor instructions**: This plan is GATED. Step 1 is a measurement whose
+> result decides whether Steps 2–4 run at all. Do not build the model before
+> Step 1's go/no-go passes. Run every verification command; if a STOP condition
+> fires, stop and report. Update the status row in `plans/README.md` when done.
+>
+> **Drift check**: `git diff --stat d28fe25..HEAD -- research/ storage/` — the
+> experiment harness (`research/experiments/`) and labeler tables are the
+> substrate; if they moved, re-confirm the schema in Step 1 before proceeding.
+
+## Status
+
+- **Priority**: P1 (this is the one entry-conversion lever never tested)
+- **Effort**: L (but Step 1 alone is S and may end the plan)
+- **Risk**: LOW to the live system — this is all SHADOW (predict + log, never gate) until an experiment verdict promotes it
+- **Depends on**: 006 (experiment harness — the promotion path), 004 (population module)
+- **Category**: direction / research-infrastructure
+- **Planned at**: commit `d28fe25`, 2026-07-14
+
+## MEASURED RESULT (Step 1 ran 2026-07-14) — verdict: NO-GO on the current data
+
+Leakage-safe expanding-window CV (5 date-blocks, whole-day splits) over 63,803 labeled setups, HistGradientBoostingClassifier on the trigger-time features that are actually populated:
+
+- **Mean CV AUC = 0.514** (folds: 0.489 / 0.538 / 0.528 / 0.526 / 0.487) — noise. The largest fold (62k rows) is 0.487, below random.
+- **Top-decile +1R = 0.456 vs base 0.428** — a ~+2.8pp lift, inconsistent across folds (some below base). Not tradeable.
+- **Top feature by far: `session_minute_number` (+0.010 AUC)**, then `relative_volume` (+0.003); everything else ≈ 0. The only faint signal is *enter early* + *higher RVOL* — both already known and already in the live gate.
+
+**Two data-quality findings that gate any future attempt:**
+1. **The rich features are mostly NULL** — `gap_pct` 85% null, `impulse_pct` 94%, `pullback_depth_pct` 98%, `catalyst_freshness_at_trigger` 76%, `float_rotation`/`tape_regime`/`spy_return` ~all null. The labeler populates different fields per track, so the "24-feature vector" is really ~8 dense features for most rows. A model can't find signal in columns that aren't filled.
+2. **The dataset is 85% `liq1`** (53,992/63,803 — the *refuted* liquid-breakout track); the live ORB `v1` is only 6,217 rows and converts WORSE (+1R rate 0.241 vs liq1's 0.437). So this isn't even a clean test of the live strategy.
+
+**Conclusion:** on the data as it exists, there is no separable joint signal — the model confirms at the multi-feature level what the 7 single-lever tests found. This is NOT a clean rejection of "could a model ever work," because the potentially-predictive features (gap, structure, catalyst, tape regime) are unpopulated. The honest next move is a **prerequisite data-plumbing fix** (populate those features for live ORB setups + turn on catalyst enrichment so `catalyst_freshness` fills), then re-run this probe — with a strong negative prior. Absent that, the lever is a *different entry* (premarket axis), not a better filter on ORB.
+
+## Why this matters (read before building anything)
+
+The project has refuted **7+ single-indicator selection levers** (float, RVOL beyond a floor, liquidity, VWAP-reclaim, leading-gainer, premarket-volume, runner-propensity) and, this week, **every exit variant** (volume_fade wash, time_stop worse, asym_exit killed). The consistent finding: **no lone indicator separates the ~50% of ORB setups that convert from the ~50% that stop out.** The bottleneck is entry-conversion.
+
+But every lever was tested **in isolation**. The move never tried: a **model over the full feature vector**. There are now **18,164 labeled setups** (`setup_events` JOIN `outcome_labels`) with ~24 features (gap, rvol, VWAP position, float rotation, impulse/pullback structure, session minute, tape regime, catalyst freshness, SPY return) and a clean binary label (`reached_1r_before_minus_1r`). A model asks *"does the combination separate winners?"* and its feature importances are literally *"which indicators matter, jointly"* — the disciplined version of the operator's "surely some indicator works." The ML approach was correctly deferred before because **n was tiny**; it is no longer tiny.
+
+The "humanistic / LLM / Alpaca-data" idea has a disciplined home too: the catalyst engine (Ollama + Alpaca news) is already built and gated off. The wrong use is a live gate (that is the refuted-folklore class). The right use is a **feature**: classify each setup's catalyst via Ollama over Alpaca news, add it to the vector, and let the model say whether the qualitative "story" separates winners.
+
+**The hard discipline** (this project has been burned by look-ahead 3+ times): purged/embargoed time-series splits, strict out-of-sample validation, and **shadow-first promotion through the experiment harness** — never a live gate on an unvalidated score.
+
+## Current state
+
+- Labeled dataset: `setup_events` (features) JOIN `outcome_labels` (labels) on `setup_id`, ~18,164 rows.
+  - **Feature columns** (`setup_events`): `entry_reference_price, invalidation_price, target_r_multiple, impulse_pct, pullback_depth_pct, pullback_bars, pullback_volume_ratio, above_vwap_flag, vwap_at_trigger, gap_pct, relative_volume, float_rotation_at_trigger, catalyst_freshness_at_trigger, session_minute_number, tape_regime_at_trigger, spy_intraday_return_pct_at_trigger` (+ `symbol, session_date, setup_time, setup_name, setup_version` as keys/metadata — NOT features).
+  - **Label columns** (`outcome_labels`): `reached_1r_before_minus_1r` (primary target), `reached_2r_before_minus_1r`, `max_upside_next_{5,15,60}m`, `max_drawdown_next_{5,15,60}m`, `held_vwap_{5,15}m`, `trend_day_flag`, `failed_breakout_flag`, `time_to_max_upside_minutes`.
+- **LEAKAGE RULE (non-negotiable)**: the model may use ONLY features knowable at `setup_time`. Every `*_at_trigger` field and gap/rvol/vwap/impulse/pullback/session-minute qualify. **No `outcome_labels` column may be a feature** (they are the future). `target_r_multiple` is a config, fine. Confirm each feature is trigger-time-knowable before including it.
+- Experiment harness (`research/experiments/`) is the promotion path: a conversion-score gate becomes a registered experiment replayed over logged fires before it ever gates live.
+- Ollama catalyst engine: `config.py` `OllamaConfig`; enrichment gated by `NEWS_ENRICH_ENABLED` (currently off). Alpaca news via the existing news ingestion. `catalyst_freshness_at_trigger` already exists as a feature (mostly null until enrichment runs).
+- Tooling: `scikit-learn` is available in the analysis environment (1.9.0). No sklearn dependency exists in `requirements.txt` yet — Step 3 adds it only if the model ships.
+- DB: `DATABASE_URL` → the production Postgres. Analysis is READ-ONLY. Extract with `python3 -c "…"` reading `DATABASE_URL`; never source `.env`.
+
+## Commands you will need
+
+| Purpose | Command | Expected |
+|---|---|---|
+| DB reachable | `python3 -c "import os,psycopg2;psycopg2.connect(os.environ['DATABASE_URL']).close();print('ok')"` | `ok` |
+| Suite | `DATABASE_URL=<url> python3 -m pytest -q` | green (baseline: 15 WSL-loopback dashboard failures only) |
+| ruff | `ruff check --select F research/` | clean |
+
+## Scope
+
+**In scope**:
+- `research/conversion/` (new package: `dataset.py`, `signal_probe.py`, `model.py`, `cli.py`)
+- `research/experiments/` — a new intervention/experiment that gates on the model's score (shadow), reusing the harness
+- `config.py` / catalyst wiring — ONLY to expose the catalyst classification as a feature (Step 4), no live-gate change
+- `requirements.txt` — add `scikit-learn` ONLY if Step 3 ships a model
+- tests under `tests/unit/conversion/`
+- `plans/README.md` (status row)
+
+**Out of scope**:
+- ANY live gate/sizing/exit change. The model produces a SHADOW score; promotion to live is a separate, owner-approved change through the harness (like every other lever).
+- Turning `NEWS_ENRICH_ENABLED` on as a live gate. Catalyst is a feature only.
+- Deep-learning / new heavy deps. Gradient-boosted trees (sklearn `HistGradientBoostingClassifier`) is the right tool for imbalanced tabular data and is already available.
+- Predicting anything other than trigger-time-knowable → `reached_1r_before_minus_1r` (and optionally `reached_2r`). No regression on P&L.
+
+## Steps
+
+### Step 1 — GO/NO-GO: does separable signal exist at all? (leakage-safe CV AUC)
+
+Build `research/conversion/dataset.py` (load the JOINed dataset, drop leakage columns, coerce types, one-hot `tape_regime`/`setup_name`) and `research/conversion/signal_probe.py`:
+
+- Split by TIME, not randomly: sort by `session_date`; use `sklearn.model_selection.TimeSeriesSplit` (5 folds) with a **1-day embargo** between train and test (drop the boundary day) so no same-day leakage.
+- Train `HistGradientBoostingClassifier` (handles NaNs natively — important, many features are sparse) on each fold; score test-fold **ROC-AUC** and **precision@top-decile** (if you only took the model's top 10% most-confident setups, what's the +1R rate vs the base rate?).
+- Report per-fold and mean AUC, the top-decile lift, and permutation feature importances on the held-out folds.
+
+**Verify**: `DATABASE_URL=<url> python3 -m research.conversion.signal_probe` prints mean CV AUC, top-decile +1R rate vs base rate, and ranked feature importances.
+
+**GO/NO-GO gate (this decides the plan):**
+- **Mean CV AUC ≥ ~0.55 AND top-decile +1R rate materially above base rate** → signal exists. Proceed to Step 2.
+- **AUC ≈ 0.50 (0.47–0.53) and no top-decile lift** → **STOP and report**: the data contains no trigger-time-knowable signal that separates conversion. That is the definitive answer to "is there an indicator" — there isn't one in this feature set, and the lever is a *different setup* (the premarket axis), not more modeling of this one. Do NOT proceed to build a model that fits noise.
+
+### Step 2 — Honest out-of-sample evaluation + calibration
+
+Only if Step 1 passed. `research/conversion/model.py`: train on the first ~80% of sessions, hold out the most recent ~20% as a true forward test (never touched in tuning). Report forward AUC, a calibration curve (`CalibratedClassifierCV`), and the top-decile forward +1R rate. Persist the trained model + the exact feature list + the training date range to `data/conversion_model.pkl` (atomic write, like `learned_params.json`).
+
+**Verify**: forward-test AUC within ~0.03 of CV AUC (no overfit blowup); calibration curve roughly diagonal. If forward AUC collapses to ~0.50 while CV was high → **STOP**, that is leakage or overfit; report which feature leaked.
+
+### Step 3 — Shadow score through the experiment harness (NOT a live gate)
+
+Only if Step 2 held. Register a harness experiment `conversion_gate_v1`: intervention = "take only setups whose model score ≥ threshold τ"; replay over logged fires/setups (the harness already does this); pre-register the bar (expR_delta_min vs taking all). Add `scikit-learn` to `requirements.txt`. The loop may COMPUTE and LOG a conversion score per candidate (a new shadow field on the decision receipt) but MUST NOT gate on it.
+
+**Verify**: `python3 -m research.experiments run && … adjudicate` shows `conversion_gate_v1` accruing; score logged on receipts; zero change to live gate code (`git diff` touches no live-path gate).
+
+### Step 4 — The catalyst/LLM feature (the "humanistic" layer), as a feature only
+
+Only if Steps 1–3 showed promise AND `catalyst_freshness`/tape features ranked meaningfully (i.e. qualitative context looks predictive). Wire the Ollama catalyst classification (news type, freshness, dilution risk from Alpaca news) into the labeler so it lands as `setup_events` features at build time; rebuild the dataset; re-run Step 1's probe WITH the catalyst features and compare AUC. If catalyst features lift AUC out-of-sample, that is the evidence to enrich; if not, leave enrichment off.
+
+**Verify**: probe AUC with-vs-without catalyst features, reported side by side. Promotion of any of this to a live gate remains a separate owner decision.
+
+## Test plan
+
+- `tests/unit/conversion/test_dataset_no_leakage.py` — assert the feature list contains ZERO `outcome_labels` columns and only trigger-time-knowable fields (the single most important test; a leaked label is how you get a fake AUC of 0.9).
+- `test_signal_probe_time_split` — the CV uses time-ordered folds with an embargo (no shuffle); a synthetic dataset with a known signal recovers AUC > 0.5, pure noise recovers ~0.5.
+- `test_model_persist_roundtrip` — save/load preserves the feature list + date range.
+- Verification: `DATABASE_URL=<url> python3 -m pytest -q` green; non-loopback failures = 0.
+
+## Done criteria
+
+- [ ] Step 1 probe runs and prints mean CV AUC + top-decile lift + feature importances
+- [ ] The go/no-go was recorded (AUC value + decision) in `plans/README.md`
+- [ ] If NO-GO: plan stops here with the honest "no separable signal" finding logged — this is a valid, complete outcome
+- [ ] If GO: forward-test AUC + calibration reported; model persisted; harness experiment registered; live gate code unchanged (`git diff` proves it)
+- [ ] Leakage test green; suite green
+- [ ] `scikit-learn` in requirements.txt only if a model shipped
+
+## STOP conditions
+
+- DB unreachable (the current blocker) — wait; do not fabricate results.
+- Step 1 AUC ≈ 0.50 — STOP, report "no signal", do not model.
+- Step 2 forward AUC collapses vs CV — STOP, report leakage/overfit and which feature.
+- Any step requires touching a live gate — STOP, that is promotion (owner-gated).
+- A feature turns out to be trigger-time-UNknowable (leak) — remove it, re-run, report.
+
+## Maintenance notes
+
+- The model is a SHADOW predictor until the harness proves its score separates forward and the owner promotes it. Same discipline as every refuted lever — the graveyard is full of things that looked good in-sample.
+- Feature importances are the real deliverable even if the model never gates: they tell the owner which of the 24 measured things actually co-vary with conversion, ending the one-indicator-at-a-time guessing.
+- If Step 1 says no-signal, that is genuinely useful: it redirects effort from re-modeling this setup to the premarket axis (the one unexplored setup), which is a *new entry*, not a *better filter* on this one.
+- Retrain cadence + drift: if promoted, the model retrains on the same nightly scheduler as `nightly_tune`, with the same staleness alarm.

plans/009-feature-population-prerequisite.md

@@ -0,0 +1,133 @@
+# Plan 009: Instrument the data so a conversion model gets ONE fair test — fill the three feature gaps, then re-run the go/no-go
+
+> **Executor instructions**: This is the prerequisite to plan 008. It fixes the
+> data-quality reasons the 2026-07-14 signal probe was inconclusive, then re-runs
+> that probe scoped to the live strategy. It is GATED: the final step's AUC
+> decides whether modeling is ever worth pursuing. Run every verification; honor
+> STOP conditions; update `plans/README.md` when done.
+>
+> **Drift check**: `git diff --stat d28fe25..HEAD -- research/labeler.py research/ingestion/ config.py` — the labeler is the substrate; re-confirm the excerpts below if it moved.
+
+## Status
+
+- **Priority**: P2 (real payoff, but a multi-week accrual play against a negative prior — read "Honest framing")
+- **Effort**: L (mostly forward data accrual; the code is M)
+- **Risk**: LOW to live trading — all of this is labeler/feature plumbing + a config flag; no live gate/sizing/exit changes
+- **Depends on**: 006 (harness), 008 (the probe this re-runs)
+- **Category**: research-infrastructure / data-quality
+- **Planned at**: commit `d28fe25`, 2026-07-14
+
+## Honest framing (read before starting — this is not a quick win)
+
+The 2026-07-14 probe (plan 008) found **CV AUC 0.514 ≈ noise** on 63,803 labeled setups. Investigation showed the test was crippled by data quality, not necessarily by absence of signal:
+
+- 85% of rows are `liq1` (a *refuted* track); the live ORB `v1` is only 6,217 rows and converts at 24%.
+- Per-version null audit:
+  - `v1` (live ORB): has `gap_pct`, `above_vwap_flag`, `vwap_at_trigger`, `session_minute_number`; **missing `relative_volume` (100% null), catalyst, market-context.** (Pullback/impulse fields are legitimately N/A for a breakout setup — do not "fix" those.)
+  - `pm1` (premarket): the **best-instrumented** track — `gap_pct`, `relative_volume`, `impulse_pct` all populated; only catalyst missing. Also the one unexplored strategy axis.
+  - `tape_regime_at_trigger` and `spy_intraday_return_pct_at_trigger` are **100% null on every track** — hardcoded `None` in the labeler.
+
+This plan fills the three *fillable* gaps so a model gets one fair, fully-instrumented test. **Set expectations honestly:** even fully instrumented, the prior is strongly negative (7 refuted single levers + this joint-model probe), `v1` is small (~6k) and low-converting, and the market-context + catalyst features can only accrue **forward** (thin history), so a fair re-test needs a few weeks of data. If the re-run still returns ~0.51, that is the **definitive** answer — stop filtering ORB and build the premarket entry (which this plan also leaves better-instrumented). The alternative to running this plan is to skip straight to the premarket-setup work; this plan is worth it only if the owner wants modeling to get its one clean shot first.
+
+## Current state
+
+- Labeler: `research/labeler.py`. The v1/ORB `SetupEvent` build (~lines 288–316) computes `rvol = (vol_upto / avg_vol) if avg_vol else None` — so `relative_volume` is None whenever `avg_vol` (average daily volume) is 0/null, which is the norm for thin IPO-class small caps (documented blind spot; see the `ignition.py` comment "scan_gappers RVOL zeros out on thin IPOs, avg-daily-vol = 0").
+- Market context: `research/labeler.py:~322` literally sets `tape_regime=None, spy_intraday_return=None`. These features are declared in `setup_events` but never populated.
+- SPY/QQQ data: `SELECT count(*), count(DISTINCT session_date) FROM minute_bars WHERE symbol IN ('SPY','QQQ')` → 53,452 rows across **only 30 sessions**. So market-context backfill is limited to ~30 days; forward-fill requires ingesting SPY/QQQ daily going forward.
+- Catalyst: `config.py` `OllamaConfig`; enrichment gated by `NEWS_ENRICH_ENABLED` (off in the deployed compose — verified). `catalyst_freshness_at_trigger` / `catalyst_type` columns exist but are 73–97% null because enrichment isn't running. `_catalyst_at(...)` in the labeler already reads catalyst events IF they exist.
+- The probe: plan 008's `research/conversion/signal_probe.py` (to be created per plan 008) — this plan re-runs it scoped to a single `setup_version`.
+- READ-ONLY analysis + labeler rebuilds only. No live-path changes. `DATABASE_URL` = production; never source `.env` (extract `DATABASE_URL` only).
+
+## Commands you will need
+
+| Purpose | Command | Expected |
+|---|---|---|
+| Suite | `DATABASE_URL=<url> python3 -m pytest -q` | green (baseline: 15 WSL-loopback dashboard failures only) |
+| ruff | `ruff check --select F research/` | clean |
+| Rebuild a track | `DATABASE_URL=<url> python3 -c "from research.labeler import build_<track>; from research.query import open_research_db; build_<track>(open_research_db('market'), rebuild=True)"` | prints rows built |
+
+## Scope
+
+**In scope**:
+- `research/labeler.py` — compute `spy_intraday_return` + `tape_regime` (replace the hardcoded `None`); add a thin-name RVOL fallback for the v1 build path
+- `research/ingestion/` — a daily SPY/QQQ ingest so market context has a source going forward (small addition; follow the existing daily-bar ingest pattern)
+- Both compose scheduler blocks / `.env.example` — turn on `NEWS_ENRICH_ENABLED` and add SPY/QQQ ingest to the nightly job
+- `tests/unit/` — labeler feature-computation tests
+- `plans/README.md`
+
+**Out of scope**:
+- ANY live gate/sizing/exit change; turning catalyst into a live gate (it is a FEATURE only — same rule as plan 008)
+- "Fixing" impulse/pullback nulls on v1 (legitimately N/A for a breakout setup)
+- Backfilling market context beyond the ~30 days of SPY data that exist (forward-fill only)
+- Building the model (that is plan 008, gated on this plan's Step 5 result)
+
+## Steps
+
+### Step 1 — Compute market context (the "context outside the single-stock calculation")
+
+In `research/labeler.py`, replace `tape_regime=None, spy_intraday_return=None` with computed values, for every track's build path that has a `setup_time`:
+- `spy_intraday_return` = SPY's return from the session open to the setup's minute (from `minute_bars` where `symbol='SPY'`, that session). Null when SPY bars are absent for that day (pre-30-day-window history).
+- `tape_regime` = a small discrete classification from SPY's intraday path at trigger: e.g. `trend_up` / `trend_down` / `chop` based on SPY's return sign + whether it's making higher-highs vs oscillating (a simple rule; do not over-engineer — this is a feature to test, not a signal to trust).
+
+This is the closest thing to the operator's "humanistic market context" — is the whole tape working with or against the trade. Add a daily SPY/QQQ ingest (`research/ingestion/`) wired into the nightly scheduler so this fills going forward.
+
+**Verify**: rebuild one recent session's setups; `SELECT count(*) FILTER (WHERE spy_intraday_return_pct_at_trigger IS NOT NULL), count(*) FROM setup_events WHERE session_date = '<a day with SPY data>'` → non-null count > 0. Unit test: a synthetic SPY-up session yields `spy_intraday_return > 0` and `tape_regime='trend_up'`.
+
+### Step 2 — RVOL fallback for thin names (v1)
+
+In the v1 build path, when `avg_vol` is 0/null (thin IPO-class name), fall back to an alternative volume baseline rather than emitting `relative_volume=None`: e.g. the symbol's own prior-session total volume, or the median of available prior days, or a premarket-volume proxy — whichever the stored data supports. Record which baseline was used in a new nullable column or in an existing diagnostic field so the model can down-weight fallback RVOL. Never fabricate a baseline that isn't grounded in real volume.
+
+**Verify**: after rebuild, `SELECT avg((relative_volume IS NULL)::int) FROM setup_events WHERE setup_version='v1'` drops well below 1.0. Unit test: a thin name with `avg_vol=0` but real prior-day volume yields a non-null `relative_volume`.
+
+### Step 3 — Turn catalyst enrichment on (as a feature source)
+
+- Set `NEWS_ENRICH_ENABLED=true` (loop + wherever the labeler builds) in both compose files + `.env.example`. Confirm `OLLAMA_ENABLED` + a reachable `OLLAMA_HOST`.
+- Verify the enrichment actually writes catalyst events the labeler's `_catalyst_at(...)` reads, so `catalyst_freshness_at_trigger` / `catalyst_type` populate on newly-built setups.
+
+**Verify**: after a night of enrichment (or a manual enrichment run), rebuild a recent session; `SELECT avg((catalyst_freshness_at_trigger IS NULL)::int) FROM setup_events WHERE setup_version IN ('v1','pm1') AND session_date='<recent>'` is materially below the current 0.97/0.96. If it stays ~1.0, STOP — enrichment isn't wired to the labeler; report the gap (that is a separate wiring fix, not a config flip).
+
+### Step 4 — Rebuild the instrumented tracks
+
+Rebuild `v1` and `pm1` setups so the new features land (market context where SPY data exists; RVOL fallback; catalyst where enrichment has run). This is a labeler rebuild, not new trading.
+
+**Verify**: per-version null audit (the query in Current state) shows `relative_volume` and `spy_intraday_return` null-rates dropped for v1 on the covered window; suite green.
+
+### Step 5 — GO/NO-GO: re-run the probe SCOPED to the live strategy
+
+Re-run plan 008's `signal_probe.py` with a `--setup-version v1` filter (and separately `pm1`), on the enriched, non-`liq1` data:
+- Same leakage-safe date-block CV, HistGradientBoostingClassifier, ROC-AUC + top-decile lift + permutation importance.
+
+**GO/NO-GO:**
+- **v1 (or pm1) mean CV AUC ≥ ~0.55 with a consistent top-decile lift** → signal exists in the instrumented live data → proceed to plan 008 Step 2 (build the model, shadow-first).
+- **Still ~0.50–0.52 after full instrumentation** → **STOP. This is the definitive answer:** there is no trigger-time-knowable signal that separates ORB conversion, even fully instrumented. Redirect all effort to the premarket entry (a new setup), not a filter on ORB. Record the AUC + this conclusion in `plans/README.md`.
+
+**Verify**: `DATABASE_URL=<url> python3 -m research.conversion.signal_probe --setup-version v1` prints the scoped AUC; the go/no-go decision is recorded.
+
+## Test plan
+
+- `test_market_context_computed` — synthetic SPY-up/down/chop sessions → correct `spy_intraday_return` sign + `tape_regime`.
+- `test_rvol_fallback_thin_name` — `avg_vol=0` + real prior-day volume → non-null `relative_volume`, fallback flagged.
+- `test_labeler_no_live_path_change` — grep guard: the diff touches no gate/sizing/exit code (`trading_execution.py`, `runtime/`, `run_live_paper.py` gate paths unchanged).
+- Verification: `DATABASE_URL=<url> python3 -m pytest -q` green; non-loopback failures = 0.
+
+## Done criteria
+
+- [ ] `tape_regime` + `spy_intraday_return` computed (not `None`); SPY/QQQ daily ingest scheduled
+- [ ] v1 `relative_volume` null-rate materially below 1.0 after rebuild
+- [ ] `NEWS_ENRICH_ENABLED` on in both compose files; catalyst null-rate dropping on newly-built v1/pm1 (or STOP-reported if wiring is missing)
+- [ ] Step 5 probe re-run scoped to v1 (and pm1); AUC + go/no-go decision recorded in `plans/README.md`
+- [ ] `git diff` shows zero live gate/sizing/exit changes
+- [ ] Suite green
+
+## STOP conditions
+
+- Step 3 catalyst stays ~100% null after enabling enrichment → enrichment isn't wired to the labeler; report, don't force.
+- Step 5 AUC still ~0.51 → STOP, record the definitive "no ORB signal" finding, recommend the premarket-setup pivot.
+- Any step needs a live gate change → STOP (promotion is owner-gated).
+- Market-context backfill tempts you to fetch >30 days of SPY history to "fix" coverage → don't; forward-fill is the honest scope, note the limited backfill window.
+
+## Maintenance notes
+
+- Market context + catalyst accrue FORWARD; a fair Step-5 re-test wants a few weeks of post-enablement data. Re-running the probe monthly and watching AUC is the right cadence — one shot won't be decisive on the enriched features until n accrues.
+- `pm1` comes out of this the best-instrumented track — if the owner pivots to the premarket axis, the feature groundwork is already laid.
+- The whole plan is falsifiable by design: if AUC stays flat, it ends the "surely some indicator works" thread with data, which is worth the effort even as a negative result.

plans/011-regime-gate.md

@@ -0,0 +1,184 @@
+# Plan 011: Regime gate — a FREE-data daily signal that decides whether the ORB trades today, proven in the harness before it ever gates a live order
+
+> **Executor instructions**: This plan is GATED and staged. Build the rungs in
+> order; each rung has its own STOP. The whole thing is SHADOW/analysis — it adds
+> a harness intervention and two offline jobs, and changes NO live trading path
+> until a final owner-gated promotion (Step 5, which this plan does NOT perform).
+> Run every verification. Update `plans/README.md` when done.
+>
+> **Drift check**: `git diff --stat 1da168d..HEAD -- research/ strategy/population.py experiments/` — the experiment harness + the shared population gate are the substrate. Line numbers may have shifted; verify the quoted code exists *semantically*.
+
+## Status
+
+- **Priority**: P1 (the one genuinely untried lever this milestone — it targets the bot's OWN edge, not the market, and it directly explains why the bot printed on 2026-07-08/09 and bleeds now)
+- **Effort**: L
+- **Risk**: LOW to live — all offline + one new harness intervention; no live gate/sizing/exit change until Step 5 (owner-gated, NOT in this plan)
+- **Depends on**: 006 (experiment harness), 004 (`strategy/population.py`), 009 (SPY/QQQ ingest + `setup_events` market-context columns already landed)
+- **Category**: selection / regime (session-level, not per-name)
+- **Planned at**: commit `1da168d`, 2026-07-15
+- **Companion research**: `docs/research/regime_notes.md` (6-book distillation — read it first; it is the *why* behind every discipline rule below)
+
+## Why this matters (and the one insight that makes it different from the 7 refuted levers)
+
+Every prior lever tried to separate *which name* will run (float, liquidity, RVOL, VWAP-reclaim, leading-gainer, gap-band…). All refuted at year scale — memory calls the bottleneck "entry-conversion, not selection." This lever is orthogonal: it does not pick names, it decides **whether the ORB should trade at all today**.
+
+The reframe (Chan's *metalabeling*, `regime_notes.md`): **do not detect abstract "market regime." Predict `P(our ORB makes money this session)` from free pre-open features, using our OWN per-session realized expR as the label.** That targets exactly what matters (does *our* edge work today), sidesteps the "high-vol lumps runner-days with crash-days" trap, reuses the harness, is free, and is a *private* signal nobody arbitrages away.
+
+Concretely: the bot did well 07-08/09 (a runner regime) and is ~breakeven-to-bleeding otherwise. If a free pre-open signal can tell a runner-friendly session from a chop session even a little, gating the ORB to the friendly ones lifts whole-population expR. If it can't — that is a real, cheap finding, and it closes the lever.
+
+## The load-bearing discipline (or the whole thing is a lie — from `regime_notes.md`)
+
+1. **Threshold baseline FIRST.** A trivial no-fit rule (Step 1) must be beaten by any ML (Step 2) *net, out-of-sample*, or you ship the rule. A trivial rule cannot leak; it is the honest floor.
+2. **Walk-forward / filtered only.** The `favorable` flag for session D must be computed from data **strictly before D** (an embargo of ≥1 session dropped at the boundary). A full-sample fit, or any percentile computed over the whole set including D, **peeks at the future**. Sanity check: if a full-sample-fit gate crushes the walk-forward gate, your edge is look-ahead — report it, don't ship it.
+3. **Label is same-day outcome; every feature is strictly pre-open (lagged to the prior close).** The session's realized ORB expR is the label; a feature that reads any intraday/at-close value of session D is leakage.
+4. **Acceptance = NET expR improves, not AUC.** A 53%-accurate classifier can still be net-negative. The bar is `expR_delta > 0` on the harness AND on a held-out recent split.
+5. **Let the DATA pick the sign.** Do NOT assume high-vol = bad. A long-dispersion small-cap momentum book likely *needs* volatility. Measure conditional expR per regime; the gate keeps whichever regime the data says is +EV.
+6. **Small sample humility.** ~150–260 sessions, fewer runner-days. Keep features ≤8, trees shallow (`max_depth ≤ 3`), and expect the gate to lag (it enters/exits the runner regime a day late). The bar is *net* improvement, not perfection.
+
+## Current state (what already exists — reuse it, don't rebuild)
+
+- **Harness keep-or-zero pattern**: `research/experiments/replay.py` `_run_rank` / the `rank_one_bullet` intervention already does exactly "keep this fire's `baseline_r`, or zero it" per a session-level decision. The regime gate is the same shape, keyed on a session flag instead of a per-fire rank.
+- **Setup-sourced fires** (`reconstruct_fires_from_setups`) give ~18k backtestable ORB fires with `entry_reference_price`/`invalidation_price`; their baseline is the LIVE exit simulated on the bar path (`_sim_baseline_r`). **This is both the fire source AND the label source** — the per-session label is the mean of `_sim_baseline_r` over that session's setup fires.
+- **Per-session pre-open context already stored** in `setup_events` (plan 009): `session_date`, `gap_pct`, `relative_volume`, `above_vwap_flag`, `tape_regime_at_trigger`, `spy_intraday_return_pct_at_trigger`.
+- **SPY/QQQ bars ingested** by `research/ingestion/market_context.py` (minute bars 04:00–20:00 ET + recent daily). SPY daily → 5d/20d momentum + 20d realized vol.
+- **Shared population gate**: `strategy/population.py` `population_reject` — the regime gate composes ON TOP of it (population screens the name; regime screens the session).
+- **Adjudication**: `research/experiments/adjudicate.py` computes `expR_delta = mean(variant_r - baseline_r)` and gates on `bars.expR_delta_min` + `bars.min_n`. For a keep-or-zero gate, `expR_delta = -mean(baseline_r over ZEROED fires) × frac_zeroed` — **positive iff the sessions the gate removes were net losers.** That is precisely the right verdict.
+- **Deps**: `sklearn 1.9.0`, `numpy`, `pandas` present. **`statsmodels` is ABSENT** — so the HMM rung (Step 3) is deferred and needs an owner `pip install`; Steps 1–2 need no new dependency.
+
+## Commands you will need
+
+| Purpose | Command | Expected |
+|---|---|---|
+| Suite | `DATABASE_URL=<url> python3 -m pytest -q` | green (the ~15 WSL-loopback dashboard failures are pre-existing; non-loopback = 0) |
+| Build regime features | `python3 -m research.regime build-features` | writes `regime_features` rows, prints count |
+| Score walk-forward | `python3 -m research.regime score --model threshold_v1` | writes `regime_scores`, prints favorable/unfavorable session counts |
+| Register/run/adjudicate | `python3 -m research.experiments {register <json>\|run\|adjudicate}` | as documented |
+| ruff | `ruff check --select F research/ strategy/` | clean |
+
+**Never run against production from a test.** Use `MOMENTUM_PG_SCHEMA` or `:memory:` (see the [[pg-test-isolation-landmine]] memory) — a file/default `db_path` silently hits the live DB.
+
+## Scope
+
+**In scope**:
+- `research/regime.py` — NEW. Two offline subcommands: `build-features` (per-session pre-open feature table) and `score` (walk-forward `favorable` flag → `regime_scores`). Pure offline analysis; imports nothing from the live loop.
+- `storage/regime_schema.sql` (or an idempotent `CREATE TABLE IF NOT EXISTS` in `research/regime.py`) — `regime_features` + `regime_scores` tables.
+- `research/experiments/replay.py` — ONE new intervention `iv_regime_gate` + wiring in `run()`/`_compute_fire` (mirror `rank_one_bullet`: keep `baseline_r` on favorable sessions, 0 otherwise). Additive; existing interventions untouched.
+- `experiments/regime_threshold_v1.json`, `experiments/regime_meta_v1.json` — the two go/no-go experiments.
+- `tests/unit/` + `tests/unit/experiments/` — coverage per the test plan.
+- `plans/README.md`.
+
+**Out of scope** (HARD — do not touch):
+- ANY live trading path: `trading_execution.py`, the live gate, sizing, exits, `run_live_paper.py`. The regime gate does NOT gate a live order in this plan. Live promotion is Step 5, owner-gated, its own change.
+- `strategy/population.py` — reuse it, do not edit it.
+- Adding `statsmodels` or any new dependency for Steps 1–2. The HMM rung (Step 3) is deferred and owner-gated on the install.
+- Smoothed/full-sample regime fits used as the tradeable flag — leakage; they may appear ONLY as the labelled "look-ahead sanity check" in Step 4, never as the shipped `favorable` flag.
+
+## Steps
+
+### Step 0 — Data-availability probe (STOP gate)
+
+Before building anything, confirm the data supports it:
+
+1. Count distinct `session_date` in `setup_events` with ≥1 setup carrying `entry_reference_price` and `invalidation_price` (the labelable sessions).
+2. Count SPY daily bars available (`daily_bars WHERE symbol='SPY'`) and their date span vs the setup sessions.
+3. Count sessions where the SPY 20-day lookback is satisfiable (≥20 prior SPY daily bars).
+
+**STOP conditions:**
+- Fewer than ~150 labelable sessions with a satisfiable SPY lookback → **STOP and report.** The sample is too thin for a walk-forward fit that means anything; note the count and recommend accruing more sessions (the scheduler ingests SPY nightly) before revisiting. The threshold rung (Step 1) may still be reported as descriptive-only, but do NOT fit the ML.
+- SPY daily coverage does not overlap the setup sessions (market_context ingest only has ~30 sessions of SPY, per its docstring) → **the SPY-momentum features are unavailable for most of history.** Report this; fall back the Step-1 rule to features that DO cover the range (`tape_regime_at_trigger`, session-level `gap_pct`/`relative_volume` breadth aggregated from `setup_events`), and note SPY-based features await backfill. Do NOT fabricate SPY history.
+
+**Verify**: the probe prints the three counts; you have written down which feature set is actually populated over the setup date range.
+
+### Step 1 — Threshold baseline (no ML, no possible leak) — the shippable floor
+
+1. **`research/regime.py build-features`** — for each labelable `session_date`, compute a per-session feature row from **pre-open / prior-close-lagged** inputs only. Start with what is populated over the full range (per Step 0), a subset of:
+   - `breadth_gap` = fraction of that session's `setup_events` names with `gap_pct > 0` (session-level breadth; knowable pre-open from the gapper scan).
+   - `breadth_rvol` = median `relative_volume` across the session's names.
+   - `tape_regime` = the session's `tape_regime_at_trigger` (already an ingested regime tag) — encode to an ordinal.
+   - `spy_mom5`, `spy_mom20`, `spy_vol20` = SPY 5d/20d return + 20d realized vol from `daily_bars` **as of the prior close** (only where Step 0 says SPY coverage exists).
+   - `bot_trailing_expR_10d` = the mean session-label expR over the **prior** 10 labelable sessions (a hot/cold-streak detector on our own edge — strictly backward-looking; the single most on-target feature per `regime_notes.md`).
+
+   Store in `regime_features(session_date PRIMARY KEY, <feature cols>, label_expR, label_pos)` where `label_expR` = mean `_sim_baseline_r` over the session's setup fires and `label_pos` = `label_expR > 0`. **`label_*` are outcome columns — they are NEVER read as features; they exist so `score` can fit/evaluate without recomputing the harness.**
+
+2. **`research/regime.py score --model threshold_v1`** — the trivial rule, computed **walk-forward**: for each session D, `favorable(D)` = the rule evaluated on D's features using ONLY thresholds derivable from sessions `< D` (with a 1-session embargo). Rule v1 (data picks the winning side — measure both directions, keep the +EV one):
+   - Primary: `bot_trailing_expR_10d > 0` (trade when our own recent edge is positive) **AND** `tape_regime` in the runner-favorable set.
+   - If SPY features exist: optionally `spy_vol20` on the side the data prefers (do NOT hardcode low-vol=good; pick the side whose *prior-sessions* conditional label_expR is higher).
+   - Write `regime_scores(model_version, session_date, p_profit, favorable)` — for a rule, `p_profit ∈ {0,1}` = `favorable`.
+
+3. **Add `iv_regime_gate` to `replay.py`** — mirror `_run_rank`. Load `{session_date: favorable}` for the experiment's `intervention.model_version` once per run; for each fire: `variant_r = baseline_r if favorable.get(session_date) else 0.0`; `detail = {"kind":"regime_gate","favorable":bool,"model_version":...}`. Fire sessions with no score row → keep (fail-open, like the population gate), and count them in the detail so a stale scores table is visible.
+
+4. **Register + run + adjudicate `experiments/regime_threshold_v1.json`**: population = the live v1 population (`{"min_entry_gap_pct":0.10,"min_day_volume":300000,"price_min":2.0,"price_max":20.0}`), intervention = `{"kind":"regime_gate","model_version":"threshold_v1"}`, bars = `{"expR_delta_min":0.05,"min_n":300}`, fidelity = `{"atr_frac":0.25,"spread_mult":3,"adverse_ticks":1}`.
+
+**GO/NO-GO (Step 1):**
+- `regime_threshold_v1` `expR_delta > 0` with `n ≥ 300` → the trivial rule removes net-losing sessions. A real, shippable floor. Proceed to Step 2 to see if ML beats it.
+- `expR_delta ≤ 0` → the free features carry no session signal even for a trivial rule. **This is a strong NO-GO signal.** Proceed to Step 2 ONCE (a classifier can find a rule the hand-rule missed), but with a low prior. If Step 2 also fails, STOP and report "no exploitable regime signal in free data at this sample size" — a complete, valuable outcome.
+
+**Verify**: `python3 -m research.experiments run && python3 -m research.experiments adjudicate` prints `regime_threshold_v1: <verdict> (n=…)`; you record its `expR_delta` and the favorable/unfavorable session split.
+
+### Step 2 — Metalabel classifier (sklearn, walk-forward) — the primary
+
+1. **`research/regime.py score --model meta_v1`** — a `sklearn.ensemble.GradientBoostingClassifier(max_depth=3)` (or `RandomForestClassifier`, ≤8 features, shallow). Walk-forward: for each session D, fit on sessions `< D` (embargo 1) with `X` = the feature columns, `y` = `label_pos`; predict `p_profit(D)`. `favorable(D) = p_profit(D) ≥ p_min` (default 0.5, but pick `p_min` from a walk-forward-internal split, never from the test tail). Write `regime_scores(model_version='meta_v1', …)`.
+   - Require a minimum train size (e.g. ≥60 sessions) before scoring; below it, `favorable = True` (fail-open, don't gate on an unfit model).
+2. **Register + run + adjudicate `experiments/regime_meta_v1.json`** — same population/bars/fidelity as threshold, `intervention.model_version = "meta_v1"`.
+
+**GO/NO-GO (Step 2):**
+- `regime_meta_v1` `expR_delta > 0`, `n ≥ 300`, **AND** its `expR_delta` exceeds `regime_threshold_v1`'s by a real margin → the classifier adds signal over the trivial rule. Proceed to Step 4 (the OOS + leakage audit that decides promotion-readiness).
+- `regime_meta_v1` `expR_delta ≤ threshold's` → the ML adds nothing; **the rule is the product.** If the threshold passed, carry `threshold_v1` into Step 4; if neither passed, STOP with the honest NO-GO.
+
+**Verify**: adjudicate prints both experiments' verdicts side by side; you record `expR_delta(meta) − expR_delta(threshold)`.
+
+### Step 3 — HMM regime as a FEATURE (DEFERRED, owner-gated on `pip install statsmodels`)
+
+Only if Step 2 shows signal AND you want a volatility-state feature the tree lacks. Add a 2-state `statsmodels MarkovRegression(spy_returns, k_regimes=2, switching_variance=True)` **filtered** (not smoothed) state probability as one extra column into `build-features`, re-fit `meta_v1`, re-adjudicate. **Requires the owner to install statsmodels first** — do NOT add the dep yourself. If it doesn't lift `expR_delta`, drop it. This rung is optional by design; the plan is complete without it.
+
+### Step 4 — Out-of-sample split + leakage audit (the promotion-readiness gate)
+
+The harness `expR_delta` is walk-forward already, but confirm it survives a held-out recent split and is not a look-ahead artifact:
+
+1. **OOS split**: recompute `expR_delta` on the most-recent ~30% of sessions only (fit/score used only their own priors — already true by walk-forward). The favorable-regime expR advantage must survive on this recent slice. "In-sample regimes need not persist out-of-sample" — if it evaporates here, the regime was in-sample noise.
+2. **Leakage sanity check**: build a *deliberately leaky* variant — `favorable` computed from a FULL-SAMPLE fit (or an in-D percentile) — and compare its `expR_delta` to the walk-forward one. If the leaky version is *dramatically* better, that gap is exactly the look-ahead you correctly excluded; note it. If walk-forward ≈ leaky, the signal is robust.
+3. **Every-fold check**: `expR_delta` should be positive across each of ~3 sequential time folds, not driven by one lucky month (e.g. 07-08/09 alone).
+
+**GO/NO-GO (Step 4):**
+- Survives OOS + every fold, and the leaky-vs-walkforward gap is not the whole edge → **promotion-ready.** Present the owner the Step-5 decision (a live change), do NOT flip anything.
+- Fails OOS or is one-month-driven → **do NOT promote.** Report it as "regime signal is in-sample only at this sample size." Keep the offline scorer running as a shadow so the verdict can be re-checked as sessions accrue.
+
+### Step 5 — Live promotion (OWNER-GATED — this plan does NOT perform it)
+
+If Step 4 is promotion-ready, the owner decides whether the live loop consults `regime_scores` for *today* to gate/size fresh entries. That is a live-path change (a new session-level gate in `trading_execution`, computed from strictly-pre-open features on the live clock, fail-open when the model is unfit/stale) and gets its **own** owner-gated plan — never an env flip inside this one. Document the interface (`regime_scores` keyed by today's date, written by a pre-open scorer job) and stop.
+
+## Test plan
+
+- `test_regime_gate_keeps_and_zeros` — `iv_regime_gate` returns `baseline_r` for a favorable session and `0.0` for an unfavorable one; a session with no score row is kept (fail-open) and flagged in `detail`.
+- `test_regime_score_is_walk_forward` — for a synthetic feature/label series, `favorable(D)` uses NO information from session `≥ D`: shuffling the labels of sessions `≥ D` must not change `favorable(D)`. This is the leakage guard — it must be a real assert, not a smoke test.
+- `test_regime_features_no_label_leak` — the feature matrix `X` passed to the classifier excludes `label_expR`/`label_pos` (assert the column set).
+- `test_threshold_experiment_registers_and_runs` — `regime_threshold_v1` registers, runs over synthetic setup fires with a scored session map, adjudicates to a verdict.
+- `test_regime_gate_expR_delta_sign` — construct fires where the zeroed sessions are net-negative → `expR_delta > 0`; where zeroed sessions are net-positive → `expR_delta < 0` (proves adjudication rewards removing losers, not just removing trades).
+- Verification: `DATABASE_URL=<url> python3 -m pytest -q` green; non-loopback failures = 0; `ruff check --select F research/ strategy/` clean.
+
+## Done criteria
+
+- [ ] Step 0 probe run; the labelable-session count + SPY coverage recorded in `plans/README.md`; STOP honored if under threshold
+- [ ] `regime_features` + `regime_scores` tables created idempotently; `build-features` + `score` runnable
+- [ ] `iv_regime_gate` added; `test_regime_gate_keeps_and_zeros` + `test_regime_gate_expR_delta_sign` green
+- [ ] `test_regime_score_is_walk_forward` green (the leakage guard)
+- [ ] `regime_threshold_v1` registered + adjudicated; `expR_delta` + session split recorded in `plans/README.md`
+- [ ] `regime_meta_v1` registered + adjudicated; `expR_delta(meta) − expR_delta(threshold)` recorded
+- [ ] Step 4 OOS + every-fold + leakage-gap results recorded; a clear GO (promotion-ready, owner-gated) or NO-GO ("in-sample only / no free-data regime signal at this sample size") written down
+- [ ] ZERO live-path change: `git diff 1da168d..HEAD -- trading_execution.py run_live_paper.py strategy/exits.py` is empty
+- [ ] Suite green; ruff clean
+
+## STOP conditions
+
+- Step 0: under ~150 labelable sessions, or SPY coverage absent over the range → STOP/report; do not fit ML on a thin sample.
+- Step 1 AND Step 2 both `expR_delta ≤ 0` → STOP: "no exploitable regime signal in free data at this sample size." A complete outcome — it closes the regime lever with data, same value as a yes.
+- Any step tempts a live gate/sizing/exit change or an env flip → HARD STOP (Step 5 is owner-gated, separate plan).
+- The walk-forward test (`test_regime_score_is_walk_forward`) cannot be made to pass → your scorer leaks; do NOT trust any `expR_delta` until it does. A leaking scorer makes every downstream verdict a lie (`regime_notes.md`, discipline rule 2).
+- Adding `statsmodels` is required to proceed → that is Step 3, deferred; do NOT install a dependency. Ship Steps 1–2 without it.
+
+## Maintenance notes
+
+- **The walk-forward flag is the load-bearing assumption.** If `score` ever computes a threshold/percentile/fit over the whole set (including the session being scored), the gate silently becomes look-ahead and every backtest lies. The leakage test is the tripwire — keep it.
+- **Let the data pick the sign, always.** Do not hardcode "low vol = good." The bot is long-dispersion small-cap momentum; it may *need* volatility. `score` chooses the favorable side from prior-session conditional expR, not from a prior belief.
+- **The gate lags by construction.** A daily regime flag enters the runner regime a session or two late and exits late. It will sit out some good days and take some bad ones. The bar is *net* whole-population expR improvement across folds, never a perfect call.
+- **This is the most promising lever of the milestone AND the most likely to curve-fit.** ~150–260 sessions with fewer runner-days is a small sample. The threshold-first rule, walk-forward, OOS split, and every-fold check are the four guards. If, after all four, the gated expR does not clearly beat trade-always out-of-sample → the honest finding is "no exploitable regime signal in free data at this sample size," and we learned it cheaply, without a single live dollar at risk.

plans/README.md

@@ -26,6 +26,7 @@ your row when done.
 | 008 | Conversion-signal model — measure if the labeled data separates winners | P1 | L | 006, 004 | **MEASURED 2026-07-14 (corrected): the 0.514 was the liq1-dominated MIX. Scoped to live v1: CV AUC 0.641 — REAL signal, but it is ~entirely the VWAP gate the bot ALREADY runs (above-VWAP +1R 35.4% vs below 12.7%). Residual within above-VWAP = 0.526 ≈ noise. RVOL-fallback added (read-only test 2026-07-14): 0.526→0.535, within noise — RVOL adds nothing beyond VWAP. Verdict: VWAP is the whole story for current features; no second indicator. Only market-context + catalyst remain untested (forward-accrual only, negative prior). The lever is a different setup (premarket), not more ORB filters.** |
 | 010 | Premarket shadow entry — prove a FILL-ABLE edge before any live premarket order | P2 | L | 006, 009 | **NO-GO (Step 1 measured 2026-07-15): premarket is a paper-fill mirage. pm1+fat-tail-exit expR: −0.50R @0.5% fill, −1.27R @1% (realistic thin book), −2.74R @2%, vs the ORB's −0.11R. The 22-min fat tail can't outrun the 1-2% premarket spread. STOP — do not build the live shadow track. The ORB's 5-min wait exists to trade in penny-spread regular-session liquidity.** |
 | 009 | Feature-population prerequisite — instrument v1/pm1 (market context, RVOL fallback, catalyst-on), then re-run 008's probe | P2 | L | 006, 008 | code DONE, awaiting supervised rebuild+probe (market context + thin-name RVOL fallback + SPY/QQQ ingest + enrichment-on + committed probe all landed & tested; Steps 4-5 rebuild/AUC run supervised) |
+| 011 | Regime gate — a FREE-data daily signal deciding whether the ORB trades today; metalabel `P(our ORB profits this session)`, proven in the harness (walk-forward) before any live gate | P1 | L | 006, 004, 009 | **CODE DONE + REVIEW-APPROVED (worktree `worktree-agent-a2709a9c0f0bb7d65`, commit `5cef4aa`), awaiting SUPERVISED real-data adjudication. Built: `research/regime.py` (probe + walk-forward `build-features`/`score`), `iv_regime_gate` harness intervention (keep-or-zero, fail-open), `regime_threshold_v1`+`regime_meta_v1` JSONs, 5 tests incl. the mutate-the-future leakage guard. Reviewer re-ran: 9 new + 35 existing experiments tests green, ruff clean, live paths untouched. HMM rung skipped (statsmodels absent). Research: `docs/research/regime_notes.md`. SUPERVISED next step (hits production Postgres — NOT done by the executor): `python3 -m research.regime probe` (Step-0 STOP gate) → `build-features` → `score --model threshold_v1`/`meta_v1` → register+run+adjudicate the two experiments → record expR_delta + OOS/fold results here. Review findings to fold in at that point: (1) `tape_regime` = earliest setup's at-trigger (~9:35) reading, not strictly pre-open — fine for the shadow harness, must be lagged before Step-5 live promotion; (2) minor: `_load_regime_favorable` model_version has no None-guard (both JSONs set it, fail-open if absent). Honest prior unchanged: most promising AND most curve-fit-prone (~150-260 sessions); walk-forward + threshold-floor + OOS decide GO vs "no free-data regime signal at this sample size".** |
 
 Status values: TODO | IN PROGRESS | DONE | BLOCKED (with one-line reason) | REJECTED (with one-line rationale)
 
@@ -55,5 +56,5 @@ plan in this series (check the status table above).
 - **Reviving the NiceGUI controlroom** — owner preference: :8765 is the dashboard.
 - **LISTEN/NOTIFY for dashboard change detection** — deferred, not rejected: the shared cached signal (007 step 2) is the sufficient lazy fix; NOTIFY is the named upgrade path.
 - **TimescaleDB / pg extensions** — not required at current scale (README's own assessment); revisit only if event/bar volume grows past what the hot-path index handles.
-- **`strategy/regime.py` deletion** — already deleted at HEAD; recon reference was stale.
+- **`strategy/regime.py` deletion** — already deleted at HEAD; recon reference was stale. NOTE: plan 011 introduces a NEW, unrelated `research/regime.py` — an *offline* metalabel scorer (session-level `P(ORB profits today)`), not the deleted live regime module. Different file, different purpose; do not conflate.
 - **Schwab account sync revival for human-vs-bot comparison** — the only non-delete rationale for schwab/; owner may veto 002's schwab deletion if that comparison is still wanted (plan 002 grep-gates it either way).

research/regime.py

@@ -128,26 +128,55 @@ class _ConShim:
         self.con = con
 
 
-def _session_labels(con, sessions: list[date]) -> dict[date, float]:
+def _session_bars(con, session_date: date, symbols: list[str]) -> dict:
+    """All minute bars for `symbols` on one session, grouped by symbol — ONE
+    query per session instead of one per fire. The per-fire round-trip to the
+    LAN Postgres was the build-features bottleneck (64k fires -> 64k queries ->
+    5min timeout); this cuts it to one query per session (~266 total). Each
+    per-symbol frame matches query_minute_bars' shape (same cols, timestamp-
+    sorted, fresh RangeIndex) so _entry_index/_sim_baseline_r work unchanged."""
+    if not symbols:
+        return {}
+    ph = ",".join(["?"] * len(symbols))
+    df = con.execute(
+        "SELECT symbol, timestamp, session_date, is_premarket, is_regular_hours, "
+        "open, high, low, close, volume, vwap, quality_score FROM minute_bars "
+        f"WHERE session_date = ? AND symbol IN ({ph}) ORDER BY symbol, timestamp",
+        [session_date, *symbols]).df()
+    if df is None or not len(df):
+        return {}
+    return {sym: g.reset_index(drop=True) for sym, g in df.groupby("symbol")}
+
+
+def _session_labels(con, sessions: list[date], skip: set | None = None) -> dict[date, float]:
     """{session_date: label_expr} = mean(_sim_baseline_r) over that session's
     setup fires — the SAME simulation the exit-experiment harness uses for a
     setup-sourced fire's baseline. Sessions with no computable fire (no bars,
-    no risk) are omitted, not zero-filled."""
+    no risk) are omitted, not zero-filled. `skip` sessions (already labeled in
+    regime_features — a past session's label is deterministic and never changes)
+    are not re-simulated: this is what makes the nightly run incremental,
+    matching the harness's idempotent 'only new fires' model."""
     from research.experiments.replay import _entry_index, _sim_baseline_r, reconstruct_fires_from_setups
-    from research.query import query_minute_bars
 
     fires = reconstruct_fires_from_setups(_ConShim(con))
     by_session: dict[date, list] = defaultdict(list)
     for f in fires:
         by_session[f.session_date].append(f)
 
+    skip = skip or set()
     out: dict[date, float] = {}
     for d in sessions:
+        if d in skip:
+            continue
+        day_fires = [f for f in by_session.get(d, [])
+                     if f.entry_price is not None and f.stop_price is not None
+                     and f.entry_price - f.stop_price > 0]
+        if not day_fires:
+            continue
+        bars_by_sym = _session_bars(con, d, sorted({f.symbol for f in day_fires}))
         rs = []
-        for f in by_session.get(d, []):
+        for f in day_fires:
-            if f.entry_price is None or f.stop_price is None or f.entry_price - f.stop_price <= 0:
+            bars = bars_by_sym.get(f.symbol)
-                continue
-            bars = query_minute_bars(con, f.symbol, f.session_date)
             if bars is None or not len(bars):
                 continue
             idx = _entry_index(bars, f.fill_ts)
@@ -160,6 +189,15 @@ def _session_labels(con, sessions: list[date]) -> dict[date, float]:
     return out
 
 
+def _existing_labels(con) -> dict[date, float]:
+    """{session_date: label_expr} already in regime_features — the incremental
+    skip set so the nightly build re-simulates only NEW sessions."""
+    rows = con.execute(
+        "SELECT session_date, label_expr FROM regime_features WHERE label_expr IS NOT NULL"
+    ).fetchall()
+    return {_as_date(r[0]): float(r[1]) for r in rows}
+
+
 def _session_context(con, sessions: list[date]) -> dict[date, dict]:
     """{session_date: {breadth_gap, breadth_rvol, tape_regime}} from that
     session's OWN labelable setup_events rows (gap_pct/relative_volume/
@@ -228,13 +266,21 @@ def _trailing_expr(labels: dict[date, float], sessions: list[date]) -> dict[date
     return out
 
 
-def build_features(con, since: str | None = None) -> int:
+def build_features(con, since: str | None = None, rebuild: bool = False) -> int:
     """Step 1: one regime_features row per labelable session with a computable
     label. Idempotent (PK session_date + ON CONFLICT DO UPDATE — features are
-    safe to rebuild as more setup/SPY data lands)."""
+    safe to rebuild as more setup/SPY data lands).
+
+    INCREMENTAL by default: a past session's label is deterministic, so only
+    sessions not yet in regime_features get the (expensive) _sim_baseline_r
+    pass — the nightly run then simulates just the new day(s). ``rebuild=True``
+    forces a full re-simulation (use after an exit-config change, which would
+    change historical labels)."""
     apply_schema(con)
     all_sessions = _labelable_sessions(con)
-    labels = _session_labels(con, all_sessions)       # full history: label per session
+    existing = {} if rebuild else _existing_labels(con)
+    new_labels = _session_labels(con, all_sessions, skip=set(existing))  # simulate ONLY new sessions
+    labels = {**existing, **new_labels}               # full history: label per session
     trailing = _trailing_expr(labels, all_sessions)   # full history: correct rolling window
 
     sessions = all_sessions
@@ -459,7 +505,9 @@ def main(argv=None) -> int:
     sub = ap.add_subparsers(dest="cmd", required=True)
     sub.add_parser("probe")
     bf = sub.add_parser("build-features")
-    bf.add_argument("--since", help="only rebuild sessions >= this date (YYYY-MM-DD)")
+    bf.add_argument("--since", help="only WRITE sessions >= this date (YYYY-MM-DD)")
+    bf.add_argument("--rebuild", action="store_true",
+                    help="force full re-simulation of ALL session labels (default: incremental)")
     sc = sub.add_parser("score")
     sc.add_argument("--model", required=True, choices=["threshold_v1", "meta_v1"])
     sc.add_argument("--p-min", type=float, default=P_MIN_DEFAULT)
@@ -477,7 +525,7 @@ def main(argv=None) -> int:
                 return 1
             return 0
         if args.cmd == "build-features":
-            n = build_features(store.con, since=args.since)
+            n = build_features(store.con, since=args.since, rebuild=args.rebuild)
             print(f"regime_features: {n} rows")
             return 0
         if args.cmd == "score":