From 78bdf26141aacca26266f4e89926bdf5d0a6006c Mon Sep 17 00:00:00 2001 From: momentum dev Date: Thu, 18 Jun 2026 17:09:55 -0400 Subject: [PATCH] Perf: composite (event_type, timestamp) index + test-schema sweep + README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DB latency: every projection runs `WHERE event_type=? ORDER BY timestamp DESC LIMIT n`. With only a plain timestamp index, the latest of a STALE type backward-scans the whole events table — measured 373ms / ~52k buffers on 235k rows. Add a composite index in event_store._ensure_schema: CREATE INDEX idx_events_type_ts ON events(event_type, timestamp DESC) -> same query is now a 0.018ms index seek (3 buffers). Applied to the live DB CONCURRENTLY; also dropped two redundant legacy indexes (idx_events_type, now covered by the composite prefix; idx_events_correlation, a dup of idx_events_session_id) to cut write overhead. tests/conftest.py: sweep the throwaway `mem_*` schemas EventStore(":memory:") creates — 170 had leaked into the shared Postgres, bloating the catalog. README: rewritten around the current system — PostgreSQL event store, a full command reference (run_live_paper / momentum CLI / eod_replay / sweeps / tuning / cron), config knobs, and a database & performance section (the index above, extension guidance, retention). Co-Authored-By: Claude Opus 4.8 --- README.md | 244 ++++++++++++++++++++++++++--------------- storage/event_store.py | 13 +++ tests/conftest.py | 38 +++++++ 3 files changed, 205 insertions(+), 90 deletions(-) create mode 100644 tests/conftest.py diff --git a/README.md b/README.md index 5ca343b..47d1ba3 100644 --- a/README.md +++ b/README.md @@ -1,116 +1,180 @@ # Momentum -Event-sourced small-cap momentum day-trading system (Ross Cameron–style): -gap-and-go / bull-flag setups on 1-minute bars, with paper execution on -Alpaca and a live-broker integration for Schwab. - -> **Important rebuild note.** This codebase was reconstructed from scratch -> against the original architecture and interfaces after the original source -> files became unrecoverable in the working environment. Diff it against your -> local copy before discarding anything — module layout and public interfaces -> were preserved, but implementations are fresh. See `RUNBOOK.md` for what -> changed and why. - -## What this system does +Event-sourced small-cap **momentum day-trading** system (Ross Cameron–style +gap-and-go / opening-range breakout). It screens the day's most-active gappers +($1–5, gapping ≥3% on ≥2× relative volume), arms an opening-range-breakout +trigger the moment the range forms, fires a bracket order on the live cross, +and manages the exit with a trailing stop + profit-lock ladder. Execution runs +on an **Alpaca paper** account; everything is recorded in a **PostgreSQL** +append-only event store and replayable. ``` -Alpaca IEX bars ──► research/market.duckdb ──► Watcher ──► signal_ready - (1-min, live) (minute + daily) │ - ▼ -Dashboard (approve/reject) ◄── approval queue ◄── TradingExecutionService - │ │ - ▼ ▼ - exit orders ─────────────────────────► Alpaca paper account - │ - AlpacaPaperSync ◄─────┘ - (account/positions/orders → event store → dashboard) +Alpaca IEX bars ─► research DBs (minute+daily) ─► screen gappers ─► ArmedTriggerBook + (1-min, live) │ live price + ▼ crosses trigger + PostgreSQL event store ◄────────────────────── TradingExecutionService ─► Alpaca paper + (append-only: signals, orders, │ bracket (entry+stop+target) + fills, account, risk events) ▼ + ▲ LiveExitManager ─► trail stop ↑ / flatten + └──────────── AlpacaPaperSync ◄──────── (account / positions / orders) ``` -Everything observable flows through a single append-only **event store** -(`data/momentum.duckdb`): symbol discovery, criteria scores, signals, -approval requests, orders, fills, account snapshots, broker health. The -dashboard and all read paths are pure projections over those events, so any -session can be replayed and audited. - -## Milestones - -| # | Scope | Status | -|---|-------|--------| -| M1 | Strategy logic — setup evaluation, structure detection, quality, backtest | ✅ `strategy/` | -| M2 | Event store + projections | ✅ `storage/` | -| M3 | Schwab integration — OAuth, market data, positions/orders, health | ✅ `schwab/` (`python milestone3_verify.py`) | -| M4 | Watcher + live paper execution | ✅ `runtime/`, `alpaca_paper/`, `trading_execution.py`, `run_live_paper.py` | -| M5 | Dashboard UI + API | ✅ `api/`, `dashboard_api.py` | +Every read path (dashboard, journal, monitoring board) is a pure **projection** +over the event store, so any session can be replayed and audited. ## Quick start (paper trading) ```bash pip install -r requirements.txt -cp .env.example .env # fill in ALPACA_API_KEY / ALPACA_SECRET_KEY +cp .env.example .env # fill in ALPACA_API_KEY / ALPACA_SECRET_KEY + DATABASE_URL -python run_live_paper.py # full loop + dashboard at http://127.0.0.1:8010 +# PostgreSQL is the datastore. Point DATABASE_URL at your instance, e.g. +# DATABASE_URL=postgresql://user:pass@127.0.0.1:5432/momentum + +python run_live_paper.py # full loop (+ embedded dashboard) ``` -Signals appear in the dashboard's **Approval Queue**; click **Approve** to -send the bracket order to your Alpaca paper account (or start with -`--auto-approve` / `TRADING_AUTO_APPROVE=1`). Use the **Exit** button on a -position to close it at market. +No keys? `python run_live_paper.py --once` runs a single dry pass to verify wiring. -No keys? `python run_live_paper.py --once` still runs in dry mode so you can -verify wiring. +## Commands -### Dashboard (control room) +Everything is driven from the repo root. The `./momentum` wrapper runs the +inspection CLI in the project venv; the analysis scripts are plain `python`. -The dashboard is a single self-contained page (no build step) that streams -live state over Server-Sent Events and falls back to polling if the stream -drops. It shows: +### Live trading loop — `run_live_paper.py` -- **P&L strip** — realized, unrealized, total, win rate, average R, and - open/closed trade counts for the session. -- **Live sparkline charts** on every ready signal and open position, drawn - from 1-minute bars with dashed entry (blue) and stop (red) reference lines. -- **Pre-trade risk preview** in the approval queue — dollar risk, share count, - notional, and stop distance before you approve. -- **Click-to-expand criteria** on any watched symbol — all nine setup criteria - as pass / fail / not-evaluated, so a "blocked" symbol is legible at a glance. -- **Activity feed** of submissions and fills, plus the full order lifecycle. -- **Audio + toast alerts** when a new signal goes ready (toggle the sound - button, or press `S`). -- **Manual symbol injection** — type a ticker (or press `/`) to force it onto - the watchlist mid-session. -- **Keyboard shortcuts** — `A` approve / `R` reject the top of the queue, - `S` toggle sound, `/` focus the add-symbol box, `?` help. +| Command | What it does | +|---------|--------------| +| `python run_live_paper.py` | Full live loop: screen → arm → fire → manage exits → EOD flatten, with the embedded dashboard. | +| `python run_live_paper.py --no-dashboard` | Same, headless (how it runs in production / under a process manager). | +| `python run_live_paper.py --once` | One pass then exit — smoke-test the wiring. | +| `python run_live_paper.py --auto-approve` | Auto-arm brackets instead of waiting for dashboard approval (`TRADING_AUTO_APPROVE=1`). | +| `python run_live_paper.py --symbols AAPL,SNDL` | Force specific symbols onto the watchlist. | + +### Monitoring & inspection — `./momentum` + +| Command | What it does | +|---------|--------------| +| `./momentum watch` | Live monitoring board (armed triggers, ready signals, **today's trades + day P&L**, risk events). Refreshes every 3s. | +| `./momentum watch --once` | Render one frame and exit (for logging / a cron snapshot). | +| `./momentum journal` | Today's trade journal — every round-trip (ET times), broker-authoritative day P&L (`matched · open`), win rate. The 16:02 close capture. | +| `./momentum doctor` | Health check across each pipeline stage (DB reachable, ingest fresh, broker sync, loop heartbeat). | +| `./momentum inspect events` | Tail the raw event stream. | +| `./momentum inspect bars SYM` | Today's minute bars for a symbol. | +| `./momentum inspect discovery` | What the screener surfaced. | +| `./momentum inspect criteria SYM` | Per-criterion pass/fail for one symbol (why it is/isn't a setup). | +| `./momentum inspect signals` | Recent ready signals. | + +### Analysis, backtest & tuning + +| Command | What it does | +|---------|--------------| +| `python eod_replay.py [YYYY-MM-DD]` | End-of-day whole-market replay: run **today's logic** over every qualifying gapper (not just the ~20 watched live) + a perfect-hindsight line. Also **grows the stored dataset** for future verification. Defaults to today. | +| `python backtest_cli.py` | Intraday backtests over stored sessions. | +| `python sweep_exits.py` | Sweep managed-exit variants (trail / breakeven / profit-lock tiers), train/test split. | +| `python sweep_filters.py` | Sweep entry-filter variants (gap / rvol / price / liquidity). | +| `python diagnose_entries.py` | Entry-quality analysis — what the losers had in common. | +| `python nightly_tune.py` | Self-tune params into `data/learned_params.json` (applied on next boot). | +| `python research_cli.py` | Ad-hoc: session symbols, gapper scan, RSS news pull. | +| `python milestone3_verify.py` | Schwab integration health walk-through. | + +### Data ingestion + +| Command | What it does | +|---------|--------------| +| `python fetch_minute_bars.py` | Ad-hoc minute-bar ingestion. | +| `python backfill_history.py` | Backfill daily/minute history for the universe. | + +### Scheduled jobs (cron wrappers) + +The `.sh` wrappers set the venv + cwd so cron can call them directly: + +```cron +2 16 * * 1-5 /path/momentum/journal.sh # capture the close trade journal +5 16 * * 1-5 /path/momentum/eod_replay.sh # whole-market replay (grows dataset) +30 17 * * 1-5 /path/momentum/nightly_tune.sh # retune params for tomorrow +``` + +### Tests + +```bash +DATABASE_URL=postgresql://user:pass@host:5432/momentum python -m pytest +# ~130 tests: strategy, storage, triggers, execution (cooldown/re-entry/EOD +# flatten/naked-stop guards), exits, journal P&L, schwab, api, research. +``` + +Tests isolate via throwaway `mem_*` Postgres schemas; `tests/conftest.py` sweeps +them at session end. + +## Configuration + +All behavior is environment-driven — see **`.env.example`** for every key with +the *why*. The knobs that matter most: + +- **Entry:** `WATCHER_PRICE_MIN/MAX` (1–5), `TRIGGER_GAP_MIN/MAX`, `TRIGGER_RVOL_MIN`, + `TRIGGER_MIN_DOLLAR_VOL` (absolute liquidity floor — skip thin spikes). +- **Sizing:** `TRADING_RISK_PER_TRADE_PCT` (1%), `TRADING_MAX_POSITION_PCT`, + `TRADING_LIQUIDITY_MAX_VOLUME_PCT` (size down thin names). +- **Exits:** `TRADING_EXIT_TRAIL_MODE=prior_low`, `TRADING_EXIT_TRAIL_AFTER_R`, + `TRADING_EXIT_BREAKEVEN_R`, `TRADING_EXIT_PROFIT_TIERS`. +- **Discipline:** `TRADING_BACKOUT_COOLDOWN_SECONDS`, `TRADING_BLOCK_REENTRY_AFTER_EXIT` + + `TRADING_REENTRY_MIN_LOSS_PCT`, `TRADING_EOD_FLATTEN_TIME`. +- **Safety:** `TRADING_MAX_DAILY_LOSS_PCT` — **set to `0.03` before real-money + trading** (it ships loosened for testing). + +## Database & performance + +The single datastore is **PostgreSQL** (`DATABASE_URL`); research bars live in +DuckDB under `data/research/`. The event store is one append-only `events` +table that every projection reads. + +**Hot-path index.** Projections overwhelmingly run +`WHERE event_type=? ORDER BY timestamp DESC LIMIT n` (latest snapshot of a type). +A composite index makes this an index seek regardless of how stale the type is — +without it, looking up the latest of an idle type backward-scans the whole table +(measured **373 ms → 0.018 ms** on 235k rows). It's created automatically by +`storage/event_store.py`: + +```sql +CREATE INDEX idx_events_type_ts ON events (event_type, timestamp DESC); +``` + +**Plugins / extensions.** The big win here is the index above, not an extension. +For ongoing tuning: + +- **`pg_stat_statements`** — enable (`shared_preload_libraries`, server restart) + to see which queries actually cost time. Diagnostic, not a speedup itself. +- **`pg_prewarm`** — warm the cache for the `events` table after a restart. +- **TimescaleDB** — only worth it if the event/bar volume grows large (hypertables + + time partitioning + compression). Not required at current scale, and must be + installed on the server. + +**Maintenance.** ~80% of `events` rows are high-volume diagnostics +(`criteria_evaluated`, `signal_blocked`, `module_tick`). A retention/prune of old +diagnostic events keeps the table small and every query fast. Run `ANALYZE events` +after large ingests so the planner has fresh stats. ## Repository map ``` -strategy/ pure strategy engine: evaluation, structure, backtest -storage/ event schema, event store, DuckDB schema, projections -runtime/ watcher state machine + provider registry -research/ research DBs, ingestion (bars, RSS, gappers), providers -alpaca_paper/ Alpaca paper client, executor, account sync -schwab/ Schwab OAuth, market data, positions/orders, health -trading_execution.py signal → sizing → approval → order pipeline -run_live_paper.py the orchestrator (M4 end-to-end) -api/ dashboard JSON API + static UI -dashboard_api.py standalone read-only dashboard server -tests/ unit + integration suite (pytest) -``` - -CLI utilities: `fetch_minute_bars.py` (ad-hoc bar ingestion), -`backtest_cli.py` (intraday backtests over stored sessions), -`research_cli.py` (session symbols / gapper scan / RSS news), -`milestone3_verify.py` (Schwab health walk-through). - -## Tests - -```bash -python -m pytest # 60 tests: strategy, storage, watcher, execution, - # schwab, api, research, end-to-end pipeline +run_live_paper.py the orchestrator: screen → arm → fire → manage → EOD flatten +momentum_cli.py the `momentum` CLI (watch / journal / doctor / inspect) +trading_execution.py signal → sizing → approval → bracket order; risk discipline +runtime/ triggers (ArmedTriggerBook), exit_manager, flatten helper +strategy/ pure strategy engine: evaluation, structure, exits, backtest +storage/ event schema + store (PostgreSQL), projections +research/ research DBs, ingestion (bars, RSS, gappers), screeners +alpaca_paper/ Alpaca paper client, executor, account/order sync +schwab/ Schwab OAuth, market data, positions/orders, health +api/ + dashboard_api.py dashboard JSON API + static UI +eod_replay.py end-of-day whole-market replay +tests/ pytest suite (unit + integration) ``` ## Security -`data/schwab_tokens.json` (and all of `data/`) is git-ignored and written -with `0600` permissions. **If tokens were ever committed to your repository -history, revoke and re-issue them at Schwab** — see RUNBOOK.md. +`.env` and all of `data/` (DBs, `data/schwab_tokens.json`) are git-ignored and +never committed; token files are written `0600`. The `.env` holds **live broker / +database / webhook credentials** — keep it local. If any secret was ever +committed to a repo's history, **rotate it** (re-issue Alpaca keys / re-run the +Schwab OAuth flow). diff --git a/storage/event_store.py b/storage/event_store.py index 15e15c8..3bffb8a 100644 --- a/storage/event_store.py +++ b/storage/event_store.py @@ -43,6 +43,19 @@ class EventStore: created_at TIMESTAMP DEFAULT current_timestamp ); """) + # Composite (event_type, timestamp DESC) — THE hot-path index. Almost + # every projection runs `WHERE event_type=? ORDER BY timestamp DESC + # LIMIT n` (latest snapshot of a type). With only a plain timestamp + # index, looking up the latest of a STALE type backward-scans the whole + # table (measured: 235k rows filtered, ~370ms). This makes it an index + # range seek (sub-ms) regardless of how long ago that type last fired, + # and also serves bare `WHERE event_type=?` as the leftmost prefix. + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_events_type_ts + ON events(event_type, timestamp DESC); + """) + # plain timestamp index still earns its keep for type-agnostic range + # scans (e.g. `WHERE timestamp::date = CURRENT_DATE`). cursor.execute(""" CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events(timestamp); diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9c0c268 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,38 @@ +"""Shared pytest fixtures + session cleanup. + +Drop the throwaway ``mem_*`` schemas that ``EventStore(":memory:")`` creates for +test isolation. The closing fixture drops them on ``.close()``, but tests that +build a store inline (without that fixture) leak one schema per run — 170 had +piled up in the shared Postgres instance, bloating the catalog and slowing +query planning. This sweeps them at session end. +""" + +import os + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def _drop_leftover_mem_schemas(): + yield # run after the whole session + # Under xdist each worker shares the same Postgres, so a worker dropping at + # its own session-end could yank a schema another worker is still using. + # Only sweep in serial runs (the common case); parallel runs can sweep + # manually afterwards. + if os.environ.get("PYTEST_XDIST_WORKER"): + return + url = os.environ.get("DATABASE_URL") + if not url: + return + try: + import psycopg2 + conn = psycopg2.connect(url) + conn.autocommit = True + cur = conn.cursor() + cur.execute("SELECT schema_name FROM information_schema.schemata " + "WHERE schema_name LIKE 'mem_%'") + for (schema,) in cur.fetchall(): + cur.execute(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE') + conn.close() + except Exception: # noqa: BLE001 — cleanup must never fail the run + pass