diff --git a/.gitignore b/.gitignore index 08d5425..787e2fc 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,6 @@ momentum-rebuilt.zip archive/ uv.lock .firecrawl/ + +# large research PDFs (local only) +books/ diff --git a/admin_api.py b/admin_api.py index 87a07c7..d2edec5 100644 --- a/admin_api.py +++ b/admin_api.py @@ -21,6 +21,8 @@ import http.client import json import os import socket +import subprocess +import sys import urllib.request from datetime import datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -33,6 +35,45 @@ _load_overrides() from runtime import env_overrides # noqa: E402 DOCKER_SOCK = os.environ.get("DOCKER_SOCK", "/var/run/docker.sock") +# Regime backtest (plan 011) on-demand trigger — logs to the app-data volume so +# GET /api/regime/status can tail it (the panel had no way to fire this). +REGIME_LOG = os.environ.get( + "REGIME_RUN_LOG", os.path.join(os.path.dirname(__file__), "data", "regime_run.log")) +_REGIME_DONE = "REGIME PIPELINE DONE" + + +def run_regime_pipeline() -> int: + """Fire the full regime shadow pipeline DETACHED (it takes a few min): rebuild + session labels -> walk-forward score both models -> replay + adjudicate the + regime_gate experiments. Returns the child pid. Shadow only — never touches a + live order. stdout+stderr -> REGIME_LOG (truncated each run so status shows + the CURRENT run); a trailing marker lets status report completion.""" + py = sys.executable + steps = " && ".join([ + f"{py} -m research.regime build-features", + f"{py} -m research.regime score --model threshold_v1", + f"{py} -m research.regime score --model meta_v1", + f"{py} -m research.experiments run", + f"{py} -m research.experiments adjudicate", + f"echo {_REGIME_DONE}", + ]) + os.makedirs(os.path.dirname(REGIME_LOG), exist_ok=True) + log = open(REGIME_LOG, "wb") # noqa: SIM115 — handed to the detached child + proc = subprocess.Popen(["/bin/sh", "-c", steps], stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) + return proc.pid + + +def regime_status() -> dict: + """Tail REGIME_LOG + whether the last run finished (the marker) — no DB query + needed; the adjudicate step prints each experiment's verdict into the log.""" + try: + with open(REGIME_LOG, encoding="utf-8", errors="replace") as fh: + lines = fh.read().splitlines() + except FileNotFoundError: + return {"log_tail": [], "running": False, "ran": False} + return {"log_tail": lines[-40:], "ran": True, + "running": _REGIME_DONE not in lines[-3:] if lines else False} PROFILES_DIR = os.environ.get( "PROFILES_DIR", os.path.join(os.path.dirname(__file__), "data", "profiles")) ADMIN_TOKEN = os.environ.get("ADMIN_TOKEN", "") @@ -282,6 +323,8 @@ class Handler(BaseHTTPRequestHandler): return self._send(200, profiles()) if path == "/api/config/effective": return self._send(200, {"config": effective_config()}) + if path == "/api/regime/status": + return self._send(200, regime_status()) return self._send(404, {"error": "unknown route"}) def do_POST(self): # noqa: N802 @@ -345,6 +388,15 @@ class Handler(BaseHTTPRequestHandler): return self._send(400, {"error": "mode must be paper|live"}) return self._send(200, {"ok": True, "mode": mode, "overrides": merged, "note": "restart the loop to apply"}) + + if path == "/api/regime/run": + try: + pid = run_regime_pipeline() + except Exception as e: # noqa: BLE001 + return self._send(500, {"ok": False, "error": str(e)[:200]}) + return self._send(200, {"ok": True, "pid": pid, "log": REGIME_LOG, + "note": "regime shadow pipeline started (~a few min); " + "poll GET /api/regime/status"}) return self._send(404, {"error": "unknown route"}) def log_message(self, *a): # quiet diff --git a/api/static/admin.html b/api/static/admin.html index 14528ec..054f54c 100644 --- a/api/static/admin.html +++ b/api/static/admin.html @@ -75,6 +75,15 @@ color:var(--bg);padding:10px 20px;border-radius:10px;font-size:14px;opacity:0;tr +

Research jobs

+
+
Regime backtest — plan 011, SHADOW +
Rebuild session labels → walk-forward score both models → replay + adjudicate the regime_gate experiments. Runs a few min; does NOT touch live trading.
+ + +
+ +

Secrets present

@@ -87,6 +96,15 @@ function saveToken(){localStorage.setItem('admintok',$('#token').value);load()} async function api(p,opt={}){opt.headers={...(opt.headers||{}),'X-Admin-Token':tok(),'Content-Type':'application/json'}; const r=await fetch(p,opt);if(r.status===401){toast('bad token');throw 0}return r.json()} function toast(m){const t=$('#toast');t.textContent=m;t.style.opacity=1;setTimeout(()=>t.style.opacity=0,2600)} +async function runRegime(){toast('starting regime pipeline…'); + try{const r=await api('/api/regime/run',{method:'POST',body:'{}'}); + toast(r.ok?('started (pid '+r.pid+') — poll Status'):('failed: '+(r.error||''))); + setTimeout(regimeStatus,2000)}catch(e){toast('failed')}} +async function regimeStatus(){const r=await api('/api/regime/status'); + $('#regimeout').style.display='block'; + $('#regimelog').textContent=(r.log_tail&&r.log_tail.length?r.log_tail.join('\n'):'(no run yet — click Run)') + +(r.running?'\n\n… running …':(r.ran?'\n\n[finished]':'')); + if(r.running)setTimeout(regimeStatus,4000)} function dot(ok){return ``} async function load(){ const s=await api('/api/status'); diff --git a/deploy/docker/truenas-compose.yml b/deploy/docker/truenas-compose.yml index c3a7b86..5bfe607 100644 --- a/deploy/docker/truenas-compose.yml +++ b/deploy/docker/truenas-compose.yml @@ -205,6 +205,14 @@ services: # NOT look-ahead (replay is a labeling job, not a live gate). python eod_replay.py || echo "[scheduler] eod_replay FAILED" python nightly_tune.py || echo "[scheduler] nightly_tune FAILED" + # Regime gate (plan 011, ALL-SHADOW): a session-level metalabel + # P(our ORB profits today). build-features labels each session from its + # OWN realized expR; score writes the WALK-FORWARD favorable flag the + # experiments' regime_gate intervention reads. Must run BEFORE + # experiments run. Does NOT gate live trading (that is owner-gated). + python -m research.regime build-features || echo "[scheduler] regime build-features FAILED" + python -m research.regime score --model threshold_v1 || echo "[scheduler] regime score threshold FAILED" + python -m research.regime score --model meta_v1 || echo "[scheduler] regime score meta FAILED" # Experiment OS (plan 006, ALL-SHADOW): replay running experiments over # newly-logged fires, then adjudicate against pre-registered bars. python -m research.experiments run || echo "[scheduler] experiments run FAILED" diff --git a/docker-compose.yml b/docker-compose.yml index c112416..b753851 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -164,6 +164,14 @@ services: # NOT look-ahead (replay is a labeling job, not a live gate). python eod_replay.py || echo "[scheduler] eod_replay FAILED" python nightly_tune.py || echo "[scheduler] nightly_tune FAILED" + # Regime gate (plan 011, ALL-SHADOW): a session-level metalabel + # P(our ORB profits today). build-features labels each session from its + # OWN realized expR; score writes the WALK-FORWARD favorable flag the + # experiments' regime_gate intervention reads. Must run BEFORE + # experiments run. Does NOT gate live trading (that is owner-gated). + python -m research.regime build-features || echo "[scheduler] regime build-features FAILED" + python -m research.regime score --model threshold_v1 || echo "[scheduler] regime score threshold FAILED" + python -m research.regime score --model meta_v1 || echo "[scheduler] regime score meta FAILED" # Experiment OS (plan 006, ALL-SHADOW): replay running experiments over # newly-logged fires, then adjudicate against pre-registered bars. python -m research.experiments run || echo "[scheduler] experiments run FAILED" diff --git a/docs/research/regime_notes.md b/docs/research/regime_notes.md new file mode 100644 index 0000000..e84bc4f --- /dev/null +++ b/docs/research/regime_notes.md @@ -0,0 +1,55 @@ +# Regime-detection research notes — distilled for the momentum bot + +**Date:** 2026-07-15 +**Method:** 6 books in `books/market_regime_state_detection/` distilled by parallel readers (Ang & Timmermann NBER survey; Kritzman *Regime Shifts*; Zucchini/MacDonald/Langrock *HMM for Time Series*; Chan *Hands-On AI Trading* & *Quantitative Trading*; Kakkar et al *Trading Decoded*). Hamilton 1989 is a scanned image (no extractable text); its Markov-switching method is covered by the others. +**Problem this feeds:** the bot is ~breakeven with no entry edge after honest fills; the one thing untried is **regime awareness** — it did well 07-08/09 (a runner regime) and sits out / bleeds otherwise. Goal: a FREE-data daily regime signal that gates/sizes ORB entries, harness-backtestable over ~260 sessions. + +--- + +## The one insight that reframes everything (Chan's metalabeling) + +> "The key to successfully apply AI/ML to finance is to focus on **metalabeling** — finding the probability of profit of *your own* simple trading strategy, and **not** to use it to predict the market directly. Your own strategy's past track record is private; no one else is trying to predict it." — Chan, *Quantitative Trading* + +Don't detect abstract "market regime." **Predict `P(our ORB makes money today)`** from free pre-open features, using our OWN per-session expR (which the plan-006 harness already computes) as the label. This targets exactly what matters (does *our* edge work today), sidesteps the "high-vol lumps runner-days with crash-days" problem, reuses the harness, is free, and is a *private* signal nobody arbs away. Every practical book (Chan CPO, Trading Decoded RF) converges here; the academic survey (Ang-Timmermann) calls the same thing "regime-conditioned position sizing." + +## The method stack (what to build, in escalation order — ponytail) + +1. **Threshold baseline first** (Kritzman, Trading Decoded both insist): a trivial rule — e.g. `SPY 5d momentum > 0 AND SPY 20d vol < median`, or Kritzman **Financial Turbulence** (`sqrt((r−μ)Σ⁻¹(r−μ)')` on ~11 sector-ETF returns) below its 75th percentile. If the fancy model can't beat this net-of-costs out-of-sample, ship the rule. +2. **Metalabel classifier** (the primary): `GradientBoostingClassifier` / `RandomForest` (free sklearn — the books' `predictnow.ai`/`gbdt` replaced) predicting `P(profitable ORB session)` from ~5–8 free features. Gate/size on the probability. +3. **HMM regime as a FEATURE** (not the whole model): a 2-state Markov-switching state on SPY (`statsmodels MarkovRegression(spy_returns, k_regimes=2, switching_variance=True)`) or on the turbulence series — one column feeding the classifier, or a standalone gate to compare. + +## Free features (all pre-open, all we have or can ingest free) + +| Feature | Source | Note | +|---|---|---| +| SPY 5d/20d momentum + 20d realized vol (VIX proxy) | already ingesting SPY (plan 009) | the level-1 input every book uses | +| IWM (small-cap index) + sector-ETF momentum | Alpaca/yfinance, **free** | small-caps specifically; sector rotation | +| Kritzman turbulence (Mahalanobis on ~11 sector ETFs) | sector-ETF daily returns, **free** | "prices behaving uncharacteristically" | +| Breadth = % of the gapper universe green / held-VWAP yesterday | our own stored data | the SIGNED feature that breaks vol-symmetry | +| **Bot's own trailing-N-day realized expR + hit-rate** | our harness | the most on-target feature — a hot/cold-streak detector on our OWN edge | +| VIX level + 1d change | CBOE/yfinance, **free** | the vol axis directly | + +Keep to **≤5–8 features** — only ~260 sessions, ~a third fewer "runner days." More features = curve-fit. + +## The load-bearing discipline (or the whole thing is a lie) + +- **Filtered / walk-forward / purged only.** Only a regime label computed from data **≤ today** is tradeable. Smoothed HMM probs and full-sample fits **peek at the future**. Sanity check every book demanded: *if a smoothed-gated backtest crushes the filtered-gated one, your edge is look-ahead.* Use purged/embargoed walk-forward CV (drop the boundary day) — the same leakage-safe split used in `plans/008`. +- **Label is same-day-outcome, features are strictly pre-open.** The session's expR (label) uses open-knowable exits; every feature is lagged to the prior close. +- **Out-of-sample split is mandatory.** Fit on the older half, apply to the recent half; confirm the favorable-regime expR advantage survives. "In-sample regimes need not persist out-of-sample" (Ang-Timmermann) — if the edge evaporates OOS, the regimes were in-sample noise. +- **Acceptance = NET expR improves across EVERY walk-forward fold**, not AUC. A 52.8% classifier can still be net-negative (Chan Hands-On's own OOS accuracy was 52.77%). +- **Let the DATA pick the sign.** Ang-Timmermann finds low-vol = high-Sharpe for a *long-index* investor — but a long-*dispersion* small-cap momentum strategy likely inverts this (runners need volatility). Measure conditional expR per regime; never assume high-vol = bad. +- **Small-sample humility.** 2 states (don't LR-test the count), shallow trees (`max_depth ≤ 3`), report bootstrap/posterior bands on the regime probs, deflate any OOS Sharpe by the number of tweaks tried (Deflated Sharpe Ratio). +- **Regimes lag.** A daily gate enters the runner regime a day or two late and exits late — structural. It sits out *some* good days and takes *some* bad ones. The bar is net improvement, not perfection. + +## Honest prior + +This is the most promising lever of the session — it targets the bot's own edge, uses free data, reuses the harness, and directly explains the 07-08/09-vs-today swing. **But 260 sessions is a small sample**, so the real risk is curve-fitting a regime that doesn't persist. The threshold-baseline-first rule, purged walk-forward, and OOS split are the guards. If, after all that, the gated expR doesn't clearly beat trade-always out-of-sample → the honest finding is "no exploitable regime signal in free data at this sample size," and we've learned it cheaply. + +## Sources (in `books/market_regime_state_detection/`) + +- Ang & Timmermann (2011), *Regime Changes and Financial Markets*, NBER WP 17182 — the survey + the sizing template + every caveat. +- Kritzman et al, *Regime Shifts: Implications for Dynamic Strategies* — Financial Turbulence + absorption ratio. +- Zucchini, MacDonald, Langrock, *Hidden Markov Models for Time Series* — the estimation recipe (forward-filter, BIC state selection, why filtered ≠ smoothed). +- Chan et al, *Hands-On AI Trading* — the exact `statsmodels MarkovRegression` daily-gate recipe (Example 4) + the 4-year-lookback calibration. +- Chan, *Quantitative Trading* — Conditional Parameter Optimization + the metalabeling principle. +- Kakkar, Sahni, Shanmugamani, *Trading Decoded* — the RandomForest regime-as-feature recipe + purged walk-forward CV + the "beat the trivial rule or ship the rule" discipline. diff --git a/plans/008-conversion-signal-model.md b/plans/008-conversion-signal-model.md new file mode 100644 index 0000000..ac8676d --- /dev/null +++ b/plans/008-conversion-signal-model.md @@ -0,0 +1,143 @@ +# Plan 008: Measure, then model — does the labeled data contain a separable conversion signal, and can a model (+ an Ollama catalyst feature) find it? + +> **Executor instructions**: This plan is GATED. Step 1 is a measurement whose +> result decides whether Steps 2–4 run at all. Do not build the model before +> Step 1's go/no-go passes. Run every verification command; if a STOP condition +> fires, stop and report. Update the status row in `plans/README.md` when done. +> +> **Drift check**: `git diff --stat d28fe25..HEAD -- research/ storage/` — the +> experiment harness (`research/experiments/`) and labeler tables are the +> substrate; if they moved, re-confirm the schema in Step 1 before proceeding. + +## Status + +- **Priority**: P1 (this is the one entry-conversion lever never tested) +- **Effort**: L (but Step 1 alone is S and may end the plan) +- **Risk**: LOW to the live system — this is all SHADOW (predict + log, never gate) until an experiment verdict promotes it +- **Depends on**: 006 (experiment harness — the promotion path), 004 (population module) +- **Category**: direction / research-infrastructure +- **Planned at**: commit `d28fe25`, 2026-07-14 + +## MEASURED RESULT (Step 1 ran 2026-07-14) — verdict: NO-GO on the current data + +Leakage-safe expanding-window CV (5 date-blocks, whole-day splits) over 63,803 labeled setups, HistGradientBoostingClassifier on the trigger-time features that are actually populated: + +- **Mean CV AUC = 0.514** (folds: 0.489 / 0.538 / 0.528 / 0.526 / 0.487) — noise. The largest fold (62k rows) is 0.487, below random. +- **Top-decile +1R = 0.456 vs base 0.428** — a ~+2.8pp lift, inconsistent across folds (some below base). Not tradeable. +- **Top feature by far: `session_minute_number` (+0.010 AUC)**, then `relative_volume` (+0.003); everything else ≈ 0. The only faint signal is *enter early* + *higher RVOL* — both already known and already in the live gate. + +**Two data-quality findings that gate any future attempt:** +1. **The rich features are mostly NULL** — `gap_pct` 85% null, `impulse_pct` 94%, `pullback_depth_pct` 98%, `catalyst_freshness_at_trigger` 76%, `float_rotation`/`tape_regime`/`spy_return` ~all null. The labeler populates different fields per track, so the "24-feature vector" is really ~8 dense features for most rows. A model can't find signal in columns that aren't filled. +2. **The dataset is 85% `liq1`** (53,992/63,803 — the *refuted* liquid-breakout track); the live ORB `v1` is only 6,217 rows and converts WORSE (+1R rate 0.241 vs liq1's 0.437). So this isn't even a clean test of the live strategy. + +**Conclusion:** on the data as it exists, there is no separable joint signal — the model confirms at the multi-feature level what the 7 single-lever tests found. This is NOT a clean rejection of "could a model ever work," because the potentially-predictive features (gap, structure, catalyst, tape regime) are unpopulated. The honest next move is a **prerequisite data-plumbing fix** (populate those features for live ORB setups + turn on catalyst enrichment so `catalyst_freshness` fills), then re-run this probe — with a strong negative prior. Absent that, the lever is a *different entry* (premarket axis), not a better filter on ORB. + +## Why this matters (read before building anything) + +The project has refuted **7+ single-indicator selection levers** (float, RVOL beyond a floor, liquidity, VWAP-reclaim, leading-gainer, premarket-volume, runner-propensity) and, this week, **every exit variant** (volume_fade wash, time_stop worse, asym_exit killed). The consistent finding: **no lone indicator separates the ~50% of ORB setups that convert from the ~50% that stop out.** The bottleneck is entry-conversion. + +But every lever was tested **in isolation**. The move never tried: a **model over the full feature vector**. There are now **18,164 labeled setups** (`setup_events` JOIN `outcome_labels`) with ~24 features (gap, rvol, VWAP position, float rotation, impulse/pullback structure, session minute, tape regime, catalyst freshness, SPY return) and a clean binary label (`reached_1r_before_minus_1r`). A model asks *"does the combination separate winners?"* and its feature importances are literally *"which indicators matter, jointly"* — the disciplined version of the operator's "surely some indicator works." The ML approach was correctly deferred before because **n was tiny**; it is no longer tiny. + +The "humanistic / LLM / Alpaca-data" idea has a disciplined home too: the catalyst engine (Ollama + Alpaca news) is already built and gated off. The wrong use is a live gate (that is the refuted-folklore class). The right use is a **feature**: classify each setup's catalyst via Ollama over Alpaca news, add it to the vector, and let the model say whether the qualitative "story" separates winners. + +**The hard discipline** (this project has been burned by look-ahead 3+ times): purged/embargoed time-series splits, strict out-of-sample validation, and **shadow-first promotion through the experiment harness** — never a live gate on an unvalidated score. + +## Current state + +- Labeled dataset: `setup_events` (features) JOIN `outcome_labels` (labels) on `setup_id`, ~18,164 rows. + - **Feature columns** (`setup_events`): `entry_reference_price, invalidation_price, target_r_multiple, impulse_pct, pullback_depth_pct, pullback_bars, pullback_volume_ratio, above_vwap_flag, vwap_at_trigger, gap_pct, relative_volume, float_rotation_at_trigger, catalyst_freshness_at_trigger, session_minute_number, tape_regime_at_trigger, spy_intraday_return_pct_at_trigger` (+ `symbol, session_date, setup_time, setup_name, setup_version` as keys/metadata — NOT features). + - **Label columns** (`outcome_labels`): `reached_1r_before_minus_1r` (primary target), `reached_2r_before_minus_1r`, `max_upside_next_{5,15,60}m`, `max_drawdown_next_{5,15,60}m`, `held_vwap_{5,15}m`, `trend_day_flag`, `failed_breakout_flag`, `time_to_max_upside_minutes`. +- **LEAKAGE RULE (non-negotiable)**: the model may use ONLY features knowable at `setup_time`. Every `*_at_trigger` field and gap/rvol/vwap/impulse/pullback/session-minute qualify. **No `outcome_labels` column may be a feature** (they are the future). `target_r_multiple` is a config, fine. Confirm each feature is trigger-time-knowable before including it. +- Experiment harness (`research/experiments/`) is the promotion path: a conversion-score gate becomes a registered experiment replayed over logged fires before it ever gates live. +- Ollama catalyst engine: `config.py` `OllamaConfig`; enrichment gated by `NEWS_ENRICH_ENABLED` (currently off). Alpaca news via the existing news ingestion. `catalyst_freshness_at_trigger` already exists as a feature (mostly null until enrichment runs). +- Tooling: `scikit-learn` is available in the analysis environment (1.9.0). No sklearn dependency exists in `requirements.txt` yet — Step 3 adds it only if the model ships. +- DB: `DATABASE_URL` → the production Postgres. Analysis is READ-ONLY. Extract with `python3 -c "…"` reading `DATABASE_URL`; never source `.env`. + +## Commands you will need + +| Purpose | Command | Expected | +|---|---|---| +| DB reachable | `python3 -c "import os,psycopg2;psycopg2.connect(os.environ['DATABASE_URL']).close();print('ok')"` | `ok` | +| Suite | `DATABASE_URL= python3 -m pytest -q` | green (baseline: 15 WSL-loopback dashboard failures only) | +| ruff | `ruff check --select F research/` | clean | + +## Scope + +**In scope**: +- `research/conversion/` (new package: `dataset.py`, `signal_probe.py`, `model.py`, `cli.py`) +- `research/experiments/` — a new intervention/experiment that gates on the model's score (shadow), reusing the harness +- `config.py` / catalyst wiring — ONLY to expose the catalyst classification as a feature (Step 4), no live-gate change +- `requirements.txt` — add `scikit-learn` ONLY if Step 3 ships a model +- tests under `tests/unit/conversion/` +- `plans/README.md` (status row) + +**Out of scope**: +- ANY live gate/sizing/exit change. The model produces a SHADOW score; promotion to live is a separate, owner-approved change through the harness (like every other lever). +- Turning `NEWS_ENRICH_ENABLED` on as a live gate. Catalyst is a feature only. +- Deep-learning / new heavy deps. Gradient-boosted trees (sklearn `HistGradientBoostingClassifier`) is the right tool for imbalanced tabular data and is already available. +- Predicting anything other than trigger-time-knowable → `reached_1r_before_minus_1r` (and optionally `reached_2r`). No regression on P&L. + +## Steps + +### Step 1 — GO/NO-GO: does separable signal exist at all? (leakage-safe CV AUC) + +Build `research/conversion/dataset.py` (load the JOINed dataset, drop leakage columns, coerce types, one-hot `tape_regime`/`setup_name`) and `research/conversion/signal_probe.py`: + +- Split by TIME, not randomly: sort by `session_date`; use `sklearn.model_selection.TimeSeriesSplit` (5 folds) with a **1-day embargo** between train and test (drop the boundary day) so no same-day leakage. +- Train `HistGradientBoostingClassifier` (handles NaNs natively — important, many features are sparse) on each fold; score test-fold **ROC-AUC** and **precision@top-decile** (if you only took the model's top 10% most-confident setups, what's the +1R rate vs the base rate?). +- Report per-fold and mean AUC, the top-decile lift, and permutation feature importances on the held-out folds. + +**Verify**: `DATABASE_URL= python3 -m research.conversion.signal_probe` prints mean CV AUC, top-decile +1R rate vs base rate, and ranked feature importances. + +**GO/NO-GO gate (this decides the plan):** +- **Mean CV AUC ≥ ~0.55 AND top-decile +1R rate materially above base rate** → signal exists. Proceed to Step 2. +- **AUC ≈ 0.50 (0.47–0.53) and no top-decile lift** → **STOP and report**: the data contains no trigger-time-knowable signal that separates conversion. That is the definitive answer to "is there an indicator" — there isn't one in this feature set, and the lever is a *different setup* (the premarket axis), not more modeling of this one. Do NOT proceed to build a model that fits noise. + +### Step 2 — Honest out-of-sample evaluation + calibration + +Only if Step 1 passed. `research/conversion/model.py`: train on the first ~80% of sessions, hold out the most recent ~20% as a true forward test (never touched in tuning). Report forward AUC, a calibration curve (`CalibratedClassifierCV`), and the top-decile forward +1R rate. Persist the trained model + the exact feature list + the training date range to `data/conversion_model.pkl` (atomic write, like `learned_params.json`). + +**Verify**: forward-test AUC within ~0.03 of CV AUC (no overfit blowup); calibration curve roughly diagonal. If forward AUC collapses to ~0.50 while CV was high → **STOP**, that is leakage or overfit; report which feature leaked. + +### Step 3 — Shadow score through the experiment harness (NOT a live gate) + +Only if Step 2 held. Register a harness experiment `conversion_gate_v1`: intervention = "take only setups whose model score ≥ threshold τ"; replay over logged fires/setups (the harness already does this); pre-register the bar (expR_delta_min vs taking all). Add `scikit-learn` to `requirements.txt`. The loop may COMPUTE and LOG a conversion score per candidate (a new shadow field on the decision receipt) but MUST NOT gate on it. + +**Verify**: `python3 -m research.experiments run && … adjudicate` shows `conversion_gate_v1` accruing; score logged on receipts; zero change to live gate code (`git diff` touches no live-path gate). + +### Step 4 — The catalyst/LLM feature (the "humanistic" layer), as a feature only + +Only if Steps 1–3 showed promise AND `catalyst_freshness`/tape features ranked meaningfully (i.e. qualitative context looks predictive). Wire the Ollama catalyst classification (news type, freshness, dilution risk from Alpaca news) into the labeler so it lands as `setup_events` features at build time; rebuild the dataset; re-run Step 1's probe WITH the catalyst features and compare AUC. If catalyst features lift AUC out-of-sample, that is the evidence to enrich; if not, leave enrichment off. + +**Verify**: probe AUC with-vs-without catalyst features, reported side by side. Promotion of any of this to a live gate remains a separate owner decision. + +## Test plan + +- `tests/unit/conversion/test_dataset_no_leakage.py` — assert the feature list contains ZERO `outcome_labels` columns and only trigger-time-knowable fields (the single most important test; a leaked label is how you get a fake AUC of 0.9). +- `test_signal_probe_time_split` — the CV uses time-ordered folds with an embargo (no shuffle); a synthetic dataset with a known signal recovers AUC > 0.5, pure noise recovers ~0.5. +- `test_model_persist_roundtrip` — save/load preserves the feature list + date range. +- Verification: `DATABASE_URL= python3 -m pytest -q` green; non-loopback failures = 0. + +## Done criteria + +- [ ] Step 1 probe runs and prints mean CV AUC + top-decile lift + feature importances +- [ ] The go/no-go was recorded (AUC value + decision) in `plans/README.md` +- [ ] If NO-GO: plan stops here with the honest "no separable signal" finding logged — this is a valid, complete outcome +- [ ] If GO: forward-test AUC + calibration reported; model persisted; harness experiment registered; live gate code unchanged (`git diff` proves it) +- [ ] Leakage test green; suite green +- [ ] `scikit-learn` in requirements.txt only if a model shipped + +## STOP conditions + +- DB unreachable (the current blocker) — wait; do not fabricate results. +- Step 1 AUC ≈ 0.50 — STOP, report "no signal", do not model. +- Step 2 forward AUC collapses vs CV — STOP, report leakage/overfit and which feature. +- Any step requires touching a live gate — STOP, that is promotion (owner-gated). +- A feature turns out to be trigger-time-UNknowable (leak) — remove it, re-run, report. + +## Maintenance notes + +- The model is a SHADOW predictor until the harness proves its score separates forward and the owner promotes it. Same discipline as every refuted lever — the graveyard is full of things that looked good in-sample. +- Feature importances are the real deliverable even if the model never gates: they tell the owner which of the 24 measured things actually co-vary with conversion, ending the one-indicator-at-a-time guessing. +- If Step 1 says no-signal, that is genuinely useful: it redirects effort from re-modeling this setup to the premarket axis (the one unexplored setup), which is a *new entry*, not a *better filter* on this one. +- Retrain cadence + drift: if promoted, the model retrains on the same nightly scheduler as `nightly_tune`, with the same staleness alarm. diff --git a/plans/009-feature-population-prerequisite.md b/plans/009-feature-population-prerequisite.md new file mode 100644 index 0000000..7c69bac --- /dev/null +++ b/plans/009-feature-population-prerequisite.md @@ -0,0 +1,133 @@ +# Plan 009: Instrument the data so a conversion model gets ONE fair test — fill the three feature gaps, then re-run the go/no-go + +> **Executor instructions**: This is the prerequisite to plan 008. It fixes the +> data-quality reasons the 2026-07-14 signal probe was inconclusive, then re-runs +> that probe scoped to the live strategy. It is GATED: the final step's AUC +> decides whether modeling is ever worth pursuing. Run every verification; honor +> STOP conditions; update `plans/README.md` when done. +> +> **Drift check**: `git diff --stat d28fe25..HEAD -- research/labeler.py research/ingestion/ config.py` — the labeler is the substrate; re-confirm the excerpts below if it moved. + +## Status + +- **Priority**: P2 (real payoff, but a multi-week accrual play against a negative prior — read "Honest framing") +- **Effort**: L (mostly forward data accrual; the code is M) +- **Risk**: LOW to live trading — all of this is labeler/feature plumbing + a config flag; no live gate/sizing/exit changes +- **Depends on**: 006 (harness), 008 (the probe this re-runs) +- **Category**: research-infrastructure / data-quality +- **Planned at**: commit `d28fe25`, 2026-07-14 + +## Honest framing (read before starting — this is not a quick win) + +The 2026-07-14 probe (plan 008) found **CV AUC 0.514 ≈ noise** on 63,803 labeled setups. Investigation showed the test was crippled by data quality, not necessarily by absence of signal: + +- 85% of rows are `liq1` (a *refuted* track); the live ORB `v1` is only 6,217 rows and converts at 24%. +- Per-version null audit: + - `v1` (live ORB): has `gap_pct`, `above_vwap_flag`, `vwap_at_trigger`, `session_minute_number`; **missing `relative_volume` (100% null), catalyst, market-context.** (Pullback/impulse fields are legitimately N/A for a breakout setup — do not "fix" those.) + - `pm1` (premarket): the **best-instrumented** track — `gap_pct`, `relative_volume`, `impulse_pct` all populated; only catalyst missing. Also the one unexplored strategy axis. + - `tape_regime_at_trigger` and `spy_intraday_return_pct_at_trigger` are **100% null on every track** — hardcoded `None` in the labeler. + +This plan fills the three *fillable* gaps so a model gets one fair, fully-instrumented test. **Set expectations honestly:** even fully instrumented, the prior is strongly negative (7 refuted single levers + this joint-model probe), `v1` is small (~6k) and low-converting, and the market-context + catalyst features can only accrue **forward** (thin history), so a fair re-test needs a few weeks of data. If the re-run still returns ~0.51, that is the **definitive** answer — stop filtering ORB and build the premarket entry (which this plan also leaves better-instrumented). The alternative to running this plan is to skip straight to the premarket-setup work; this plan is worth it only if the owner wants modeling to get its one clean shot first. + +## Current state + +- Labeler: `research/labeler.py`. The v1/ORB `SetupEvent` build (~lines 288–316) computes `rvol = (vol_upto / avg_vol) if avg_vol else None` — so `relative_volume` is None whenever `avg_vol` (average daily volume) is 0/null, which is the norm for thin IPO-class small caps (documented blind spot; see the `ignition.py` comment "scan_gappers RVOL zeros out on thin IPOs, avg-daily-vol = 0"). +- Market context: `research/labeler.py:~322` literally sets `tape_regime=None, spy_intraday_return=None`. These features are declared in `setup_events` but never populated. +- SPY/QQQ data: `SELECT count(*), count(DISTINCT session_date) FROM minute_bars WHERE symbol IN ('SPY','QQQ')` → 53,452 rows across **only 30 sessions**. So market-context backfill is limited to ~30 days; forward-fill requires ingesting SPY/QQQ daily going forward. +- Catalyst: `config.py` `OllamaConfig`; enrichment gated by `NEWS_ENRICH_ENABLED` (off in the deployed compose — verified). `catalyst_freshness_at_trigger` / `catalyst_type` columns exist but are 73–97% null because enrichment isn't running. `_catalyst_at(...)` in the labeler already reads catalyst events IF they exist. +- The probe: plan 008's `research/conversion/signal_probe.py` (to be created per plan 008) — this plan re-runs it scoped to a single `setup_version`. +- READ-ONLY analysis + labeler rebuilds only. No live-path changes. `DATABASE_URL` = production; never source `.env` (extract `DATABASE_URL` only). + +## Commands you will need + +| Purpose | Command | Expected | +|---|---|---| +| Suite | `DATABASE_URL= python3 -m pytest -q` | green (baseline: 15 WSL-loopback dashboard failures only) | +| ruff | `ruff check --select F research/` | clean | +| Rebuild a track | `DATABASE_URL= python3 -c "from research.labeler import build_; from research.query import open_research_db; build_(open_research_db('market'), rebuild=True)"` | prints rows built | + +## Scope + +**In scope**: +- `research/labeler.py` — compute `spy_intraday_return` + `tape_regime` (replace the hardcoded `None`); add a thin-name RVOL fallback for the v1 build path +- `research/ingestion/` — a daily SPY/QQQ ingest so market context has a source going forward (small addition; follow the existing daily-bar ingest pattern) +- Both compose scheduler blocks / `.env.example` — turn on `NEWS_ENRICH_ENABLED` and add SPY/QQQ ingest to the nightly job +- `tests/unit/` — labeler feature-computation tests +- `plans/README.md` + +**Out of scope**: +- ANY live gate/sizing/exit change; turning catalyst into a live gate (it is a FEATURE only — same rule as plan 008) +- "Fixing" impulse/pullback nulls on v1 (legitimately N/A for a breakout setup) +- Backfilling market context beyond the ~30 days of SPY data that exist (forward-fill only) +- Building the model (that is plan 008, gated on this plan's Step 5 result) + +## Steps + +### Step 1 — Compute market context (the "context outside the single-stock calculation") + +In `research/labeler.py`, replace `tape_regime=None, spy_intraday_return=None` with computed values, for every track's build path that has a `setup_time`: +- `spy_intraday_return` = SPY's return from the session open to the setup's minute (from `minute_bars` where `symbol='SPY'`, that session). Null when SPY bars are absent for that day (pre-30-day-window history). +- `tape_regime` = a small discrete classification from SPY's intraday path at trigger: e.g. `trend_up` / `trend_down` / `chop` based on SPY's return sign + whether it's making higher-highs vs oscillating (a simple rule; do not over-engineer — this is a feature to test, not a signal to trust). + +This is the closest thing to the operator's "humanistic market context" — is the whole tape working with or against the trade. Add a daily SPY/QQQ ingest (`research/ingestion/`) wired into the nightly scheduler so this fills going forward. + +**Verify**: rebuild one recent session's setups; `SELECT count(*) FILTER (WHERE spy_intraday_return_pct_at_trigger IS NOT NULL), count(*) FROM setup_events WHERE session_date = ''` → non-null count > 0. Unit test: a synthetic SPY-up session yields `spy_intraday_return > 0` and `tape_regime='trend_up'`. + +### Step 2 — RVOL fallback for thin names (v1) + +In the v1 build path, when `avg_vol` is 0/null (thin IPO-class name), fall back to an alternative volume baseline rather than emitting `relative_volume=None`: e.g. the symbol's own prior-session total volume, or the median of available prior days, or a premarket-volume proxy — whichever the stored data supports. Record which baseline was used in a new nullable column or in an existing diagnostic field so the model can down-weight fallback RVOL. Never fabricate a baseline that isn't grounded in real volume. + +**Verify**: after rebuild, `SELECT avg((relative_volume IS NULL)::int) FROM setup_events WHERE setup_version='v1'` drops well below 1.0. Unit test: a thin name with `avg_vol=0` but real prior-day volume yields a non-null `relative_volume`. + +### Step 3 — Turn catalyst enrichment on (as a feature source) + +- Set `NEWS_ENRICH_ENABLED=true` (loop + wherever the labeler builds) in both compose files + `.env.example`. Confirm `OLLAMA_ENABLED` + a reachable `OLLAMA_HOST`. +- Verify the enrichment actually writes catalyst events the labeler's `_catalyst_at(...)` reads, so `catalyst_freshness_at_trigger` / `catalyst_type` populate on newly-built setups. + +**Verify**: after a night of enrichment (or a manual enrichment run), rebuild a recent session; `SELECT avg((catalyst_freshness_at_trigger IS NULL)::int) FROM setup_events WHERE setup_version IN ('v1','pm1') AND session_date=''` is materially below the current 0.97/0.96. If it stays ~1.0, STOP — enrichment isn't wired to the labeler; report the gap (that is a separate wiring fix, not a config flip). + +### Step 4 — Rebuild the instrumented tracks + +Rebuild `v1` and `pm1` setups so the new features land (market context where SPY data exists; RVOL fallback; catalyst where enrichment has run). This is a labeler rebuild, not new trading. + +**Verify**: per-version null audit (the query in Current state) shows `relative_volume` and `spy_intraday_return` null-rates dropped for v1 on the covered window; suite green. + +### Step 5 — GO/NO-GO: re-run the probe SCOPED to the live strategy + +Re-run plan 008's `signal_probe.py` with a `--setup-version v1` filter (and separately `pm1`), on the enriched, non-`liq1` data: +- Same leakage-safe date-block CV, HistGradientBoostingClassifier, ROC-AUC + top-decile lift + permutation importance. + +**GO/NO-GO:** +- **v1 (or pm1) mean CV AUC ≥ ~0.55 with a consistent top-decile lift** → signal exists in the instrumented live data → proceed to plan 008 Step 2 (build the model, shadow-first). +- **Still ~0.50–0.52 after full instrumentation** → **STOP. This is the definitive answer:** there is no trigger-time-knowable signal that separates ORB conversion, even fully instrumented. Redirect all effort to the premarket entry (a new setup), not a filter on ORB. Record the AUC + this conclusion in `plans/README.md`. + +**Verify**: `DATABASE_URL= python3 -m research.conversion.signal_probe --setup-version v1` prints the scoped AUC; the go/no-go decision is recorded. + +## Test plan + +- `test_market_context_computed` — synthetic SPY-up/down/chop sessions → correct `spy_intraday_return` sign + `tape_regime`. +- `test_rvol_fallback_thin_name` — `avg_vol=0` + real prior-day volume → non-null `relative_volume`, fallback flagged. +- `test_labeler_no_live_path_change` — grep guard: the diff touches no gate/sizing/exit code (`trading_execution.py`, `runtime/`, `run_live_paper.py` gate paths unchanged). +- Verification: `DATABASE_URL= python3 -m pytest -q` green; non-loopback failures = 0. + +## Done criteria + +- [ ] `tape_regime` + `spy_intraday_return` computed (not `None`); SPY/QQQ daily ingest scheduled +- [ ] v1 `relative_volume` null-rate materially below 1.0 after rebuild +- [ ] `NEWS_ENRICH_ENABLED` on in both compose files; catalyst null-rate dropping on newly-built v1/pm1 (or STOP-reported if wiring is missing) +- [ ] Step 5 probe re-run scoped to v1 (and pm1); AUC + go/no-go decision recorded in `plans/README.md` +- [ ] `git diff` shows zero live gate/sizing/exit changes +- [ ] Suite green + +## STOP conditions + +- Step 3 catalyst stays ~100% null after enabling enrichment → enrichment isn't wired to the labeler; report, don't force. +- Step 5 AUC still ~0.51 → STOP, record the definitive "no ORB signal" finding, recommend the premarket-setup pivot. +- Any step needs a live gate change → STOP (promotion is owner-gated). +- Market-context backfill tempts you to fetch >30 days of SPY history to "fix" coverage → don't; forward-fill is the honest scope, note the limited backfill window. + +## Maintenance notes + +- Market context + catalyst accrue FORWARD; a fair Step-5 re-test wants a few weeks of post-enablement data. Re-running the probe monthly and watching AUC is the right cadence — one shot won't be decisive on the enriched features until n accrues. +- `pm1` comes out of this the best-instrumented track — if the owner pivots to the premarket axis, the feature groundwork is already laid. +- The whole plan is falsifiable by design: if AUC stays flat, it ends the "surely some indicator works" thread with data, which is worth the effort even as a negative result. diff --git a/plans/011-regime-gate.md b/plans/011-regime-gate.md new file mode 100644 index 0000000..1a298f7 --- /dev/null +++ b/plans/011-regime-gate.md @@ -0,0 +1,184 @@ +# Plan 011: Regime gate — a FREE-data daily signal that decides whether the ORB trades today, proven in the harness before it ever gates a live order + +> **Executor instructions**: This plan is GATED and staged. Build the rungs in +> order; each rung has its own STOP. The whole thing is SHADOW/analysis — it adds +> a harness intervention and two offline jobs, and changes NO live trading path +> until a final owner-gated promotion (Step 5, which this plan does NOT perform). +> Run every verification. Update `plans/README.md` when done. +> +> **Drift check**: `git diff --stat 1da168d..HEAD -- research/ strategy/population.py experiments/` — the experiment harness + the shared population gate are the substrate. Line numbers may have shifted; verify the quoted code exists *semantically*. + +## Status + +- **Priority**: P1 (the one genuinely untried lever this milestone — it targets the bot's OWN edge, not the market, and it directly explains why the bot printed on 2026-07-08/09 and bleeds now) +- **Effort**: L +- **Risk**: LOW to live — all offline + one new harness intervention; no live gate/sizing/exit change until Step 5 (owner-gated, NOT in this plan) +- **Depends on**: 006 (experiment harness), 004 (`strategy/population.py`), 009 (SPY/QQQ ingest + `setup_events` market-context columns already landed) +- **Category**: selection / regime (session-level, not per-name) +- **Planned at**: commit `1da168d`, 2026-07-15 +- **Companion research**: `docs/research/regime_notes.md` (6-book distillation — read it first; it is the *why* behind every discipline rule below) + +## Why this matters (and the one insight that makes it different from the 7 refuted levers) + +Every prior lever tried to separate *which name* will run (float, liquidity, RVOL, VWAP-reclaim, leading-gainer, gap-band…). All refuted at year scale — memory calls the bottleneck "entry-conversion, not selection." This lever is orthogonal: it does not pick names, it decides **whether the ORB should trade at all today**. + +The reframe (Chan's *metalabeling*, `regime_notes.md`): **do not detect abstract "market regime." Predict `P(our ORB makes money this session)` from free pre-open features, using our OWN per-session realized expR as the label.** That targets exactly what matters (does *our* edge work today), sidesteps the "high-vol lumps runner-days with crash-days" trap, reuses the harness, is free, and is a *private* signal nobody arbitrages away. + +Concretely: the bot did well 07-08/09 (a runner regime) and is ~breakeven-to-bleeding otherwise. If a free pre-open signal can tell a runner-friendly session from a chop session even a little, gating the ORB to the friendly ones lifts whole-population expR. If it can't — that is a real, cheap finding, and it closes the lever. + +## The load-bearing discipline (or the whole thing is a lie — from `regime_notes.md`) + +1. **Threshold baseline FIRST.** A trivial no-fit rule (Step 1) must be beaten by any ML (Step 2) *net, out-of-sample*, or you ship the rule. A trivial rule cannot leak; it is the honest floor. +2. **Walk-forward / filtered only.** The `favorable` flag for session D must be computed from data **strictly before D** (an embargo of ≥1 session dropped at the boundary). A full-sample fit, or any percentile computed over the whole set including D, **peeks at the future**. Sanity check: if a full-sample-fit gate crushes the walk-forward gate, your edge is look-ahead — report it, don't ship it. +3. **Label is same-day outcome; every feature is strictly pre-open (lagged to the prior close).** The session's realized ORB expR is the label; a feature that reads any intraday/at-close value of session D is leakage. +4. **Acceptance = NET expR improves, not AUC.** A 53%-accurate classifier can still be net-negative. The bar is `expR_delta > 0` on the harness AND on a held-out recent split. +5. **Let the DATA pick the sign.** Do NOT assume high-vol = bad. A long-dispersion small-cap momentum book likely *needs* volatility. Measure conditional expR per regime; the gate keeps whichever regime the data says is +EV. +6. **Small sample humility.** ~150–260 sessions, fewer runner-days. Keep features ≤8, trees shallow (`max_depth ≤ 3`), and expect the gate to lag (it enters/exits the runner regime a day late). The bar is *net* improvement, not perfection. + +## Current state (what already exists — reuse it, don't rebuild) + +- **Harness keep-or-zero pattern**: `research/experiments/replay.py` `_run_rank` / the `rank_one_bullet` intervention already does exactly "keep this fire's `baseline_r`, or zero it" per a session-level decision. The regime gate is the same shape, keyed on a session flag instead of a per-fire rank. +- **Setup-sourced fires** (`reconstruct_fires_from_setups`) give ~18k backtestable ORB fires with `entry_reference_price`/`invalidation_price`; their baseline is the LIVE exit simulated on the bar path (`_sim_baseline_r`). **This is both the fire source AND the label source** — the per-session label is the mean of `_sim_baseline_r` over that session's setup fires. +- **Per-session pre-open context already stored** in `setup_events` (plan 009): `session_date`, `gap_pct`, `relative_volume`, `above_vwap_flag`, `tape_regime_at_trigger`, `spy_intraday_return_pct_at_trigger`. +- **SPY/QQQ bars ingested** by `research/ingestion/market_context.py` (minute bars 04:00–20:00 ET + recent daily). SPY daily → 5d/20d momentum + 20d realized vol. +- **Shared population gate**: `strategy/population.py` `population_reject` — the regime gate composes ON TOP of it (population screens the name; regime screens the session). +- **Adjudication**: `research/experiments/adjudicate.py` computes `expR_delta = mean(variant_r - baseline_r)` and gates on `bars.expR_delta_min` + `bars.min_n`. For a keep-or-zero gate, `expR_delta = -mean(baseline_r over ZEROED fires) × frac_zeroed` — **positive iff the sessions the gate removes were net losers.** That is precisely the right verdict. +- **Deps**: `sklearn 1.9.0`, `numpy`, `pandas` present. **`statsmodels` is ABSENT** — so the HMM rung (Step 3) is deferred and needs an owner `pip install`; Steps 1–2 need no new dependency. + +## Commands you will need + +| Purpose | Command | Expected | +|---|---|---| +| Suite | `DATABASE_URL= python3 -m pytest -q` | green (the ~15 WSL-loopback dashboard failures are pre-existing; non-loopback = 0) | +| Build regime features | `python3 -m research.regime build-features` | writes `regime_features` rows, prints count | +| Score walk-forward | `python3 -m research.regime score --model threshold_v1` | writes `regime_scores`, prints favorable/unfavorable session counts | +| Register/run/adjudicate | `python3 -m research.experiments {register \|run\|adjudicate}` | as documented | +| ruff | `ruff check --select F research/ strategy/` | clean | + +**Never run against production from a test.** Use `MOMENTUM_PG_SCHEMA` or `:memory:` (see the [[pg-test-isolation-landmine]] memory) — a file/default `db_path` silently hits the live DB. + +## Scope + +**In scope**: +- `research/regime.py` — NEW. Two offline subcommands: `build-features` (per-session pre-open feature table) and `score` (walk-forward `favorable` flag → `regime_scores`). Pure offline analysis; imports nothing from the live loop. +- `storage/regime_schema.sql` (or an idempotent `CREATE TABLE IF NOT EXISTS` in `research/regime.py`) — `regime_features` + `regime_scores` tables. +- `research/experiments/replay.py` — ONE new intervention `iv_regime_gate` + wiring in `run()`/`_compute_fire` (mirror `rank_one_bullet`: keep `baseline_r` on favorable sessions, 0 otherwise). Additive; existing interventions untouched. +- `experiments/regime_threshold_v1.json`, `experiments/regime_meta_v1.json` — the two go/no-go experiments. +- `tests/unit/` + `tests/unit/experiments/` — coverage per the test plan. +- `plans/README.md`. + +**Out of scope** (HARD — do not touch): +- ANY live trading path: `trading_execution.py`, the live gate, sizing, exits, `run_live_paper.py`. The regime gate does NOT gate a live order in this plan. Live promotion is Step 5, owner-gated, its own change. +- `strategy/population.py` — reuse it, do not edit it. +- Adding `statsmodels` or any new dependency for Steps 1–2. The HMM rung (Step 3) is deferred and owner-gated on the install. +- Smoothed/full-sample regime fits used as the tradeable flag — leakage; they may appear ONLY as the labelled "look-ahead sanity check" in Step 4, never as the shipped `favorable` flag. + +## Steps + +### Step 0 — Data-availability probe (STOP gate) + +Before building anything, confirm the data supports it: + +1. Count distinct `session_date` in `setup_events` with ≥1 setup carrying `entry_reference_price` and `invalidation_price` (the labelable sessions). +2. Count SPY daily bars available (`daily_bars WHERE symbol='SPY'`) and their date span vs the setup sessions. +3. Count sessions where the SPY 20-day lookback is satisfiable (≥20 prior SPY daily bars). + +**STOP conditions:** +- Fewer than ~150 labelable sessions with a satisfiable SPY lookback → **STOP and report.** The sample is too thin for a walk-forward fit that means anything; note the count and recommend accruing more sessions (the scheduler ingests SPY nightly) before revisiting. The threshold rung (Step 1) may still be reported as descriptive-only, but do NOT fit the ML. +- SPY daily coverage does not overlap the setup sessions (market_context ingest only has ~30 sessions of SPY, per its docstring) → **the SPY-momentum features are unavailable for most of history.** Report this; fall back the Step-1 rule to features that DO cover the range (`tape_regime_at_trigger`, session-level `gap_pct`/`relative_volume` breadth aggregated from `setup_events`), and note SPY-based features await backfill. Do NOT fabricate SPY history. + +**Verify**: the probe prints the three counts; you have written down which feature set is actually populated over the setup date range. + +### Step 1 — Threshold baseline (no ML, no possible leak) — the shippable floor + +1. **`research/regime.py build-features`** — for each labelable `session_date`, compute a per-session feature row from **pre-open / prior-close-lagged** inputs only. Start with what is populated over the full range (per Step 0), a subset of: + - `breadth_gap` = fraction of that session's `setup_events` names with `gap_pct > 0` (session-level breadth; knowable pre-open from the gapper scan). + - `breadth_rvol` = median `relative_volume` across the session's names. + - `tape_regime` = the session's `tape_regime_at_trigger` (already an ingested regime tag) — encode to an ordinal. + - `spy_mom5`, `spy_mom20`, `spy_vol20` = SPY 5d/20d return + 20d realized vol from `daily_bars` **as of the prior close** (only where Step 0 says SPY coverage exists). + - `bot_trailing_expR_10d` = the mean session-label expR over the **prior** 10 labelable sessions (a hot/cold-streak detector on our own edge — strictly backward-looking; the single most on-target feature per `regime_notes.md`). + + Store in `regime_features(session_date PRIMARY KEY, , label_expR, label_pos)` where `label_expR` = mean `_sim_baseline_r` over the session's setup fires and `label_pos` = `label_expR > 0`. **`label_*` are outcome columns — they are NEVER read as features; they exist so `score` can fit/evaluate without recomputing the harness.** + +2. **`research/regime.py score --model threshold_v1`** — the trivial rule, computed **walk-forward**: for each session D, `favorable(D)` = the rule evaluated on D's features using ONLY thresholds derivable from sessions `< D` (with a 1-session embargo). Rule v1 (data picks the winning side — measure both directions, keep the +EV one): + - Primary: `bot_trailing_expR_10d > 0` (trade when our own recent edge is positive) **AND** `tape_regime` in the runner-favorable set. + - If SPY features exist: optionally `spy_vol20` on the side the data prefers (do NOT hardcode low-vol=good; pick the side whose *prior-sessions* conditional label_expR is higher). + - Write `regime_scores(model_version, session_date, p_profit, favorable)` — for a rule, `p_profit ∈ {0,1}` = `favorable`. + +3. **Add `iv_regime_gate` to `replay.py`** — mirror `_run_rank`. Load `{session_date: favorable}` for the experiment's `intervention.model_version` once per run; for each fire: `variant_r = baseline_r if favorable.get(session_date) else 0.0`; `detail = {"kind":"regime_gate","favorable":bool,"model_version":...}`. Fire sessions with no score row → keep (fail-open, like the population gate), and count them in the detail so a stale scores table is visible. + +4. **Register + run + adjudicate `experiments/regime_threshold_v1.json`**: population = the live v1 population (`{"min_entry_gap_pct":0.10,"min_day_volume":300000,"price_min":2.0,"price_max":20.0}`), intervention = `{"kind":"regime_gate","model_version":"threshold_v1"}`, bars = `{"expR_delta_min":0.05,"min_n":300}`, fidelity = `{"atr_frac":0.25,"spread_mult":3,"adverse_ticks":1}`. + +**GO/NO-GO (Step 1):** +- `regime_threshold_v1` `expR_delta > 0` with `n ≥ 300` → the trivial rule removes net-losing sessions. A real, shippable floor. Proceed to Step 2 to see if ML beats it. +- `expR_delta ≤ 0` → the free features carry no session signal even for a trivial rule. **This is a strong NO-GO signal.** Proceed to Step 2 ONCE (a classifier can find a rule the hand-rule missed), but with a low prior. If Step 2 also fails, STOP and report "no exploitable regime signal in free data at this sample size" — a complete, valuable outcome. + +**Verify**: `python3 -m research.experiments run && python3 -m research.experiments adjudicate` prints `regime_threshold_v1: (n=…)`; you record its `expR_delta` and the favorable/unfavorable session split. + +### Step 2 — Metalabel classifier (sklearn, walk-forward) — the primary + +1. **`research/regime.py score --model meta_v1`** — a `sklearn.ensemble.GradientBoostingClassifier(max_depth=3)` (or `RandomForestClassifier`, ≤8 features, shallow). Walk-forward: for each session D, fit on sessions `< D` (embargo 1) with `X` = the feature columns, `y` = `label_pos`; predict `p_profit(D)`. `favorable(D) = p_profit(D) ≥ p_min` (default 0.5, but pick `p_min` from a walk-forward-internal split, never from the test tail). Write `regime_scores(model_version='meta_v1', …)`. + - Require a minimum train size (e.g. ≥60 sessions) before scoring; below it, `favorable = True` (fail-open, don't gate on an unfit model). +2. **Register + run + adjudicate `experiments/regime_meta_v1.json`** — same population/bars/fidelity as threshold, `intervention.model_version = "meta_v1"`. + +**GO/NO-GO (Step 2):** +- `regime_meta_v1` `expR_delta > 0`, `n ≥ 300`, **AND** its `expR_delta` exceeds `regime_threshold_v1`'s by a real margin → the classifier adds signal over the trivial rule. Proceed to Step 4 (the OOS + leakage audit that decides promotion-readiness). +- `regime_meta_v1` `expR_delta ≤ threshold's` → the ML adds nothing; **the rule is the product.** If the threshold passed, carry `threshold_v1` into Step 4; if neither passed, STOP with the honest NO-GO. + +**Verify**: adjudicate prints both experiments' verdicts side by side; you record `expR_delta(meta) − expR_delta(threshold)`. + +### Step 3 — HMM regime as a FEATURE (DEFERRED, owner-gated on `pip install statsmodels`) + +Only if Step 2 shows signal AND you want a volatility-state feature the tree lacks. Add a 2-state `statsmodels MarkovRegression(spy_returns, k_regimes=2, switching_variance=True)` **filtered** (not smoothed) state probability as one extra column into `build-features`, re-fit `meta_v1`, re-adjudicate. **Requires the owner to install statsmodels first** — do NOT add the dep yourself. If it doesn't lift `expR_delta`, drop it. This rung is optional by design; the plan is complete without it. + +### Step 4 — Out-of-sample split + leakage audit (the promotion-readiness gate) + +The harness `expR_delta` is walk-forward already, but confirm it survives a held-out recent split and is not a look-ahead artifact: + +1. **OOS split**: recompute `expR_delta` on the most-recent ~30% of sessions only (fit/score used only their own priors — already true by walk-forward). The favorable-regime expR advantage must survive on this recent slice. "In-sample regimes need not persist out-of-sample" — if it evaporates here, the regime was in-sample noise. +2. **Leakage sanity check**: build a *deliberately leaky* variant — `favorable` computed from a FULL-SAMPLE fit (or an in-D percentile) — and compare its `expR_delta` to the walk-forward one. If the leaky version is *dramatically* better, that gap is exactly the look-ahead you correctly excluded; note it. If walk-forward ≈ leaky, the signal is robust. +3. **Every-fold check**: `expR_delta` should be positive across each of ~3 sequential time folds, not driven by one lucky month (e.g. 07-08/09 alone). + +**GO/NO-GO (Step 4):** +- Survives OOS + every fold, and the leaky-vs-walkforward gap is not the whole edge → **promotion-ready.** Present the owner the Step-5 decision (a live change), do NOT flip anything. +- Fails OOS or is one-month-driven → **do NOT promote.** Report it as "regime signal is in-sample only at this sample size." Keep the offline scorer running as a shadow so the verdict can be re-checked as sessions accrue. + +### Step 5 — Live promotion (OWNER-GATED — this plan does NOT perform it) + +If Step 4 is promotion-ready, the owner decides whether the live loop consults `regime_scores` for *today* to gate/size fresh entries. That is a live-path change (a new session-level gate in `trading_execution`, computed from strictly-pre-open features on the live clock, fail-open when the model is unfit/stale) and gets its **own** owner-gated plan — never an env flip inside this one. Document the interface (`regime_scores` keyed by today's date, written by a pre-open scorer job) and stop. + +## Test plan + +- `test_regime_gate_keeps_and_zeros` — `iv_regime_gate` returns `baseline_r` for a favorable session and `0.0` for an unfavorable one; a session with no score row is kept (fail-open) and flagged in `detail`. +- `test_regime_score_is_walk_forward` — for a synthetic feature/label series, `favorable(D)` uses NO information from session `≥ D`: shuffling the labels of sessions `≥ D` must not change `favorable(D)`. This is the leakage guard — it must be a real assert, not a smoke test. +- `test_regime_features_no_label_leak` — the feature matrix `X` passed to the classifier excludes `label_expR`/`label_pos` (assert the column set). +- `test_threshold_experiment_registers_and_runs` — `regime_threshold_v1` registers, runs over synthetic setup fires with a scored session map, adjudicates to a verdict. +- `test_regime_gate_expR_delta_sign` — construct fires where the zeroed sessions are net-negative → `expR_delta > 0`; where zeroed sessions are net-positive → `expR_delta < 0` (proves adjudication rewards removing losers, not just removing trades). +- Verification: `DATABASE_URL= python3 -m pytest -q` green; non-loopback failures = 0; `ruff check --select F research/ strategy/` clean. + +## Done criteria + +- [ ] Step 0 probe run; the labelable-session count + SPY coverage recorded in `plans/README.md`; STOP honored if under threshold +- [ ] `regime_features` + `regime_scores` tables created idempotently; `build-features` + `score` runnable +- [ ] `iv_regime_gate` added; `test_regime_gate_keeps_and_zeros` + `test_regime_gate_expR_delta_sign` green +- [ ] `test_regime_score_is_walk_forward` green (the leakage guard) +- [ ] `regime_threshold_v1` registered + adjudicated; `expR_delta` + session split recorded in `plans/README.md` +- [ ] `regime_meta_v1` registered + adjudicated; `expR_delta(meta) − expR_delta(threshold)` recorded +- [ ] Step 4 OOS + every-fold + leakage-gap results recorded; a clear GO (promotion-ready, owner-gated) or NO-GO ("in-sample only / no free-data regime signal at this sample size") written down +- [ ] ZERO live-path change: `git diff 1da168d..HEAD -- trading_execution.py run_live_paper.py strategy/exits.py` is empty +- [ ] Suite green; ruff clean + +## STOP conditions + +- Step 0: under ~150 labelable sessions, or SPY coverage absent over the range → STOP/report; do not fit ML on a thin sample. +- Step 1 AND Step 2 both `expR_delta ≤ 0` → STOP: "no exploitable regime signal in free data at this sample size." A complete outcome — it closes the regime lever with data, same value as a yes. +- Any step tempts a live gate/sizing/exit change or an env flip → HARD STOP (Step 5 is owner-gated, separate plan). +- The walk-forward test (`test_regime_score_is_walk_forward`) cannot be made to pass → your scorer leaks; do NOT trust any `expR_delta` until it does. A leaking scorer makes every downstream verdict a lie (`regime_notes.md`, discipline rule 2). +- Adding `statsmodels` is required to proceed → that is Step 3, deferred; do NOT install a dependency. Ship Steps 1–2 without it. + +## Maintenance notes + +- **The walk-forward flag is the load-bearing assumption.** If `score` ever computes a threshold/percentile/fit over the whole set (including the session being scored), the gate silently becomes look-ahead and every backtest lies. The leakage test is the tripwire — keep it. +- **Let the data pick the sign, always.** Do not hardcode "low vol = good." The bot is long-dispersion small-cap momentum; it may *need* volatility. `score` chooses the favorable side from prior-session conditional expR, not from a prior belief. +- **The gate lags by construction.** A daily regime flag enters the runner regime a session or two late and exits late. It will sit out some good days and take some bad ones. The bar is *net* whole-population expR improvement across folds, never a perfect call. +- **This is the most promising lever of the milestone AND the most likely to curve-fit.** ~150–260 sessions with fewer runner-days is a small sample. The threshold-first rule, walk-forward, OOS split, and every-fold check are the four guards. If, after all four, the gated expR does not clearly beat trade-always out-of-sample → the honest finding is "no exploitable regime signal in free data at this sample size," and we learned it cheaply, without a single live dollar at risk. diff --git a/plans/README.md b/plans/README.md index 48a0bac..99241f7 100644 --- a/plans/README.md +++ b/plans/README.md @@ -26,6 +26,7 @@ your row when done. | 008 | Conversion-signal model — measure if the labeled data separates winners | P1 | L | 006, 004 | **MEASURED 2026-07-14 (corrected): the 0.514 was the liq1-dominated MIX. Scoped to live v1: CV AUC 0.641 — REAL signal, but it is ~entirely the VWAP gate the bot ALREADY runs (above-VWAP +1R 35.4% vs below 12.7%). Residual within above-VWAP = 0.526 ≈ noise. RVOL-fallback added (read-only test 2026-07-14): 0.526→0.535, within noise — RVOL adds nothing beyond VWAP. Verdict: VWAP is the whole story for current features; no second indicator. Only market-context + catalyst remain untested (forward-accrual only, negative prior). The lever is a different setup (premarket), not more ORB filters.** | | 010 | Premarket shadow entry — prove a FILL-ABLE edge before any live premarket order | P2 | L | 006, 009 | **NO-GO (Step 1 measured 2026-07-15): premarket is a paper-fill mirage. pm1+fat-tail-exit expR: −0.50R @0.5% fill, −1.27R @1% (realistic thin book), −2.74R @2%, vs the ORB's −0.11R. The 22-min fat tail can't outrun the 1-2% premarket spread. STOP — do not build the live shadow track. The ORB's 5-min wait exists to trade in penny-spread regular-session liquidity.** | | 009 | Feature-population prerequisite — instrument v1/pm1 (market context, RVOL fallback, catalyst-on), then re-run 008's probe | P2 | L | 006, 008 | code DONE, awaiting supervised rebuild+probe (market context + thin-name RVOL fallback + SPY/QQQ ingest + enrichment-on + committed probe all landed & tested; Steps 4-5 rebuild/AUC run supervised) | +| 011 | Regime gate — a FREE-data daily signal deciding whether the ORB trades today; metalabel `P(our ORB profits this session)`, proven in the harness (walk-forward) before any live gate | P1 | L | 006, 004, 009 | **CODE DONE + REVIEW-APPROVED (worktree `worktree-agent-a2709a9c0f0bb7d65`, commit `5cef4aa`), awaiting SUPERVISED real-data adjudication. Built: `research/regime.py` (probe + walk-forward `build-features`/`score`), `iv_regime_gate` harness intervention (keep-or-zero, fail-open), `regime_threshold_v1`+`regime_meta_v1` JSONs, 5 tests incl. the mutate-the-future leakage guard. Reviewer re-ran: 9 new + 35 existing experiments tests green, ruff clean, live paths untouched. HMM rung skipped (statsmodels absent). Research: `docs/research/regime_notes.md`. SUPERVISED next step (hits production Postgres — NOT done by the executor): `python3 -m research.regime probe` (Step-0 STOP gate) → `build-features` → `score --model threshold_v1`/`meta_v1` → register+run+adjudicate the two experiments → record expR_delta + OOS/fold results here. Review findings to fold in at that point: (1) `tape_regime` = earliest setup's at-trigger (~9:35) reading, not strictly pre-open — fine for the shadow harness, must be lagged before Step-5 live promotion; (2) minor: `_load_regime_favorable` model_version has no None-guard (both JSONs set it, fail-open if absent). Honest prior unchanged: most promising AND most curve-fit-prone (~150-260 sessions); walk-forward + threshold-floor + OOS decide GO vs "no free-data regime signal at this sample size".** | Status values: TODO | IN PROGRESS | DONE | BLOCKED (with one-line reason) | REJECTED (with one-line rationale) @@ -55,5 +56,5 @@ plan in this series (check the status table above). - **Reviving the NiceGUI controlroom** — owner preference: :8765 is the dashboard. - **LISTEN/NOTIFY for dashboard change detection** — deferred, not rejected: the shared cached signal (007 step 2) is the sufficient lazy fix; NOTIFY is the named upgrade path. - **TimescaleDB / pg extensions** — not required at current scale (README's own assessment); revisit only if event/bar volume grows past what the hot-path index handles. -- **`strategy/regime.py` deletion** — already deleted at HEAD; recon reference was stale. +- **`strategy/regime.py` deletion** — already deleted at HEAD; recon reference was stale. NOTE: plan 011 introduces a NEW, unrelated `research/regime.py` — an *offline* metalabel scorer (session-level `P(ORB profits today)`), not the deleted live regime module. Different file, different purpose; do not conflate. - **Schwab account sync revival for human-vs-bot comparison** — the only non-delete rationale for schwab/; owner may veto 002's schwab deletion if that comparison is still wanted (plan 002 grep-gates it either way). diff --git a/research/regime.py b/research/regime.py index ac99f78..130c43c 100644 --- a/research/regime.py +++ b/research/regime.py @@ -128,26 +128,55 @@ class _ConShim: self.con = con -def _session_labels(con, sessions: list[date]) -> dict[date, float]: +def _session_bars(con, session_date: date, symbols: list[str]) -> dict: + """All minute bars for `symbols` on one session, grouped by symbol — ONE + query per session instead of one per fire. The per-fire round-trip to the + LAN Postgres was the build-features bottleneck (64k fires -> 64k queries -> + 5min timeout); this cuts it to one query per session (~266 total). Each + per-symbol frame matches query_minute_bars' shape (same cols, timestamp- + sorted, fresh RangeIndex) so _entry_index/_sim_baseline_r work unchanged.""" + if not symbols: + return {} + ph = ",".join(["?"] * len(symbols)) + df = con.execute( + "SELECT symbol, timestamp, session_date, is_premarket, is_regular_hours, " + "open, high, low, close, volume, vwap, quality_score FROM minute_bars " + f"WHERE session_date = ? AND symbol IN ({ph}) ORDER BY symbol, timestamp", + [session_date, *symbols]).df() + if df is None or not len(df): + return {} + return {sym: g.reset_index(drop=True) for sym, g in df.groupby("symbol")} + + +def _session_labels(con, sessions: list[date], skip: set | None = None) -> dict[date, float]: """{session_date: label_expr} = mean(_sim_baseline_r) over that session's setup fires — the SAME simulation the exit-experiment harness uses for a setup-sourced fire's baseline. Sessions with no computable fire (no bars, - no risk) are omitted, not zero-filled.""" + no risk) are omitted, not zero-filled. `skip` sessions (already labeled in + regime_features — a past session's label is deterministic and never changes) + are not re-simulated: this is what makes the nightly run incremental, + matching the harness's idempotent 'only new fires' model.""" from research.experiments.replay import _entry_index, _sim_baseline_r, reconstruct_fires_from_setups - from research.query import query_minute_bars fires = reconstruct_fires_from_setups(_ConShim(con)) by_session: dict[date, list] = defaultdict(list) for f in fires: by_session[f.session_date].append(f) + skip = skip or set() out: dict[date, float] = {} for d in sessions: + if d in skip: + continue + day_fires = [f for f in by_session.get(d, []) + if f.entry_price is not None and f.stop_price is not None + and f.entry_price - f.stop_price > 0] + if not day_fires: + continue + bars_by_sym = _session_bars(con, d, sorted({f.symbol for f in day_fires})) rs = [] - for f in by_session.get(d, []): - if f.entry_price is None or f.stop_price is None or f.entry_price - f.stop_price <= 0: - continue - bars = query_minute_bars(con, f.symbol, f.session_date) + for f in day_fires: + bars = bars_by_sym.get(f.symbol) if bars is None or not len(bars): continue idx = _entry_index(bars, f.fill_ts) @@ -160,6 +189,15 @@ def _session_labels(con, sessions: list[date]) -> dict[date, float]: return out +def _existing_labels(con) -> dict[date, float]: + """{session_date: label_expr} already in regime_features — the incremental + skip set so the nightly build re-simulates only NEW sessions.""" + rows = con.execute( + "SELECT session_date, label_expr FROM regime_features WHERE label_expr IS NOT NULL" + ).fetchall() + return {_as_date(r[0]): float(r[1]) for r in rows} + + def _session_context(con, sessions: list[date]) -> dict[date, dict]: """{session_date: {breadth_gap, breadth_rvol, tape_regime}} from that session's OWN labelable setup_events rows (gap_pct/relative_volume/ @@ -228,13 +266,21 @@ def _trailing_expr(labels: dict[date, float], sessions: list[date]) -> dict[date return out -def build_features(con, since: str | None = None) -> int: +def build_features(con, since: str | None = None, rebuild: bool = False) -> int: """Step 1: one regime_features row per labelable session with a computable label. Idempotent (PK session_date + ON CONFLICT DO UPDATE — features are - safe to rebuild as more setup/SPY data lands).""" + safe to rebuild as more setup/SPY data lands). + + INCREMENTAL by default: a past session's label is deterministic, so only + sessions not yet in regime_features get the (expensive) _sim_baseline_r + pass — the nightly run then simulates just the new day(s). ``rebuild=True`` + forces a full re-simulation (use after an exit-config change, which would + change historical labels).""" apply_schema(con) all_sessions = _labelable_sessions(con) - labels = _session_labels(con, all_sessions) # full history: label per session + existing = {} if rebuild else _existing_labels(con) + new_labels = _session_labels(con, all_sessions, skip=set(existing)) # simulate ONLY new sessions + labels = {**existing, **new_labels} # full history: label per session trailing = _trailing_expr(labels, all_sessions) # full history: correct rolling window sessions = all_sessions @@ -459,7 +505,9 @@ def main(argv=None) -> int: sub = ap.add_subparsers(dest="cmd", required=True) sub.add_parser("probe") bf = sub.add_parser("build-features") - bf.add_argument("--since", help="only rebuild sessions >= this date (YYYY-MM-DD)") + bf.add_argument("--since", help="only WRITE sessions >= this date (YYYY-MM-DD)") + bf.add_argument("--rebuild", action="store_true", + help="force full re-simulation of ALL session labels (default: incremental)") sc = sub.add_parser("score") sc.add_argument("--model", required=True, choices=["threshold_v1", "meta_v1"]) sc.add_argument("--p-min", type=float, default=P_MIN_DEFAULT) @@ -477,7 +525,7 @@ def main(argv=None) -> int: return 1 return 0 if args.cmd == "build-features": - n = build_features(store.con, since=args.since) + n = build_features(store.con, since=args.since, rebuild=args.rebuild) print(f"regime_features: {n} rows") return 0 if args.cmd == "score":