From 007cd95c79f950e95a54c92481f367ea8548f94d Mon Sep 17 00:00:00 2001 From: momentum dev Date: Mon, 15 Jun 2026 23:39:29 -0400 Subject: [PATCH] Baseline before Postgres migration (WS0) Snapshot before the DuckDB->PostgreSQL migration. data/, archive/, the 232MB rebuild zip, the Windows .venv, and .env are excluded via .gitignore. Co-Authored-By: Claude Opus 4.8 --- .env.example | 74 ++ .gitignore | 60 ++ ARCHITECTURE_VISUAL.html | 664 ++++++++++++++++++ BACKTEST_GUIDE.md | 120 ++++ BUILD_SUMMARY.md | 507 +++++++++++++ DATA_TRANSFORMATIONS.md | 380 ++++++++++ INSPECTION_ROADMAP.md | 377 ++++++++++ PHASE1_IMPLEMENTATION.md | 503 +++++++++++++ PHASE2_PREVIEW.md | 487 +++++++++++++ QUICK_REFERENCE.txt | 268 +++++++ README.md | 116 +++ README_ARCHITECTURE_REDESIGN.md | 311 ++++++++ RUNBOOK.md | 227 ++++++ SYSTEM_FLOW.txt | 405 +++++++++++ __init__.py | 9 + alpaca_paper/__init__.py | 20 + alpaca_paper/client.py | 184 +++++ alpaca_paper/execution.py | 223 ++++++ alpaca_paper/settings.py | 39 + alpaca_paper/sync.py | 135 ++++ api/__init__.py | 0 api/main.py | 387 ++++++++++ api/static/dashboard.html | 598 ++++++++++++++++ backtest_cli.py | 68 ++ comprehensive_backtest.py | 326 +++++++++ config.py | 191 +++++ core/__init__.py | 0 core/contracts.py | 259 +++++++ core/telemetry.py | 274 ++++++++ core/validation.py | 284 ++++++++ dashboard_api.py | 35 + fetch_minute_bars.py | 51 ++ milestone3_verify.py | 100 +++ models/__init__.py | 0 models/base.py | 26 + package-lock.json | 18 + package.json | 5 + pyproject.toml | 37 + requirements.txt | 7 + research/__init__.py | 1 + research/ingestion/__init__.py | 16 + research/ingestion/market_data.py | 235 +++++++ research/ingestion/rss.py | 158 +++++ research/ingestion/scheduler.py | 68 ++ research/ingestion/signals.py | 136 ++++ research/ingestion/watcher_task.py | 186 +++++ research/multi_schema.py | 34 + research/query.py | 83 +++ research/schema.py | 14 + research_cli.py | 102 +++ run_live_paper.py | 356 ++++++++++ runtime/__init__.py | 1 + runtime/providers/__init__.py | 10 + runtime/providers/registry.py | 112 +++ runtime/watcher.py | 274 ++++++++ schwab/__init__.py | 5 + schwab/auth/__init__.py | 7 + schwab/auth/lifecycle.py | 50 ++ schwab/auth/oauth2_flow_manager.py | 81 +++ schwab/auth/token_store.py | 71 ++ schwab/health/__init__.py | 6 + schwab/health/models.py | 37 + schwab/health/reporter.py | 96 +++ schwab/market/__init__.py | 24 + schwab/market/adapter.py | 34 + schwab/market/client.py | 83 +++ schwab/market/models.py | 86 +++ schwab/market/option_chain_service.py | 63 ++ schwab/orders/__init__.py | 6 + schwab/orders/reader.py | 67 ++ schwab/orders/sync.py | 45 ++ schwab/positions/__init__.py | 13 + schwab/positions/models.py | 33 + schwab/positions/reader.py | 111 +++ schwab/positions/sync.py | 70 ++ schwab/settings.py | 41 ++ schwab/streaming/__init__.py | 5 + schwab/streaming/client.py | 87 +++ storage/__init__.py | 47 ++ storage/db.py | 34 + storage/event_schema.py | 344 +++++++++ storage/event_store.py | 223 ++++++ storage/pg_schema.sql | 451 ++++++++++++ storage/projections.py | 530 ++++++++++++++ storage/replay.py | 148 ++++ storage/schema.py | 444 ++++++++++++ strategy/__init__.py | 11 + strategy/backtest/__init__.py | 6 + strategy/backtest/engine.py | 197 ++++++ strategy/backtest/outcomes.py | 67 ++ strategy/evaluation/__init__.py | 103 +++ strategy/evaluation/criteria.py | 26 + strategy/evaluation/data_quality.py | 81 +++ strategy/evaluation/first_candles.py | 61 ++ strategy/evaluation/levels.py | 116 +++ strategy/evaluation/llm_integration.py | 36 + strategy/evaluation/quality.py | 83 +++ strategy/evaluation/regime.py | 55 ++ strategy/evaluation/setup_evaluator.py | 205 ++++++ strategy/evaluation/storage.py | 33 + strategy/evaluation/structure.py | 570 +++++++++++++++ strategy/evaluation/tracking.py | 32 + strategy/evaluation/volume_metrics.py | 67 ++ strategy/models.py | 73 ++ strategy/risk/__init__.py | 26 + strategy/risk/entry_cuts.py | 48 ++ strategy/risk/position_sizing.py | 51 ++ strategy/universe.py | 28 + telemetry/__init__.py | 0 telemetry/events.py | 102 +++ tests/__init__.py | 0 tests/integration/__init__.py | 0 tests/integration/test_end_to_end.py | 180 +++++ tests/synthetic.py | 90 +++ tests/unit/__init__.py | 0 tests/unit/alpaca_paper/__init__.py | 0 .../alpaca_paper/test_execution_and_sync.py | 124 ++++ tests/unit/api/__init__.py | 0 tests/unit/api/test_dashboard_api.py | 258 +++++++ tests/unit/research/__init__.py | 0 tests/unit/research/test_ingestion.py | 158 +++++ tests/unit/runtime/__init__.py | 0 tests/unit/runtime/test_watcher.py | 103 +++ tests/unit/schwab/__init__.py | 0 tests/unit/schwab/test_schwab_integration.py | 135 ++++ tests/unit/storage/__init__.py | 0 tests/unit/storage/test_event_store.py | 176 +++++ .../storage/test_interactive_projections.py | 133 ++++ tests/unit/test_backtest_engine.py | 89 +++ tests/unit/test_setup_evaluator.py | 65 ++ tests/unit/test_trading_execution.py | 341 +++++++++ trading_execution.py | 466 ++++++++++++ trading_mode.py | 23 + 133 files changed, 17321 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 ARCHITECTURE_VISUAL.html create mode 100644 BACKTEST_GUIDE.md create mode 100644 BUILD_SUMMARY.md create mode 100644 DATA_TRANSFORMATIONS.md create mode 100644 INSPECTION_ROADMAP.md create mode 100644 PHASE1_IMPLEMENTATION.md create mode 100644 PHASE2_PREVIEW.md create mode 100644 QUICK_REFERENCE.txt create mode 100644 README.md create mode 100644 README_ARCHITECTURE_REDESIGN.md create mode 100644 RUNBOOK.md create mode 100644 SYSTEM_FLOW.txt create mode 100644 __init__.py create mode 100644 alpaca_paper/__init__.py create mode 100644 alpaca_paper/client.py create mode 100644 alpaca_paper/execution.py create mode 100644 alpaca_paper/settings.py create mode 100644 alpaca_paper/sync.py create mode 100644 api/__init__.py create mode 100644 api/main.py create mode 100644 api/static/dashboard.html create mode 100644 backtest_cli.py create mode 100644 comprehensive_backtest.py create mode 100644 config.py create mode 100644 core/__init__.py create mode 100644 core/contracts.py create mode 100644 core/telemetry.py create mode 100644 core/validation.py create mode 100644 dashboard_api.py create mode 100644 fetch_minute_bars.py create mode 100644 milestone3_verify.py create mode 100644 models/__init__.py create mode 100644 models/base.py create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 research/__init__.py create mode 100644 research/ingestion/__init__.py create mode 100644 research/ingestion/market_data.py create mode 100644 research/ingestion/rss.py create mode 100644 research/ingestion/scheduler.py create mode 100644 research/ingestion/signals.py create mode 100644 research/ingestion/watcher_task.py create mode 100644 research/multi_schema.py create mode 100644 research/query.py create mode 100644 research/schema.py create mode 100644 research_cli.py create mode 100644 run_live_paper.py create mode 100644 runtime/__init__.py create mode 100644 runtime/providers/__init__.py create mode 100644 runtime/providers/registry.py create mode 100644 runtime/watcher.py create mode 100644 schwab/__init__.py create mode 100644 schwab/auth/__init__.py create mode 100644 schwab/auth/lifecycle.py create mode 100644 schwab/auth/oauth2_flow_manager.py create mode 100644 schwab/auth/token_store.py create mode 100644 schwab/health/__init__.py create mode 100644 schwab/health/models.py create mode 100644 schwab/health/reporter.py create mode 100644 schwab/market/__init__.py create mode 100644 schwab/market/adapter.py create mode 100644 schwab/market/client.py create mode 100644 schwab/market/models.py create mode 100644 schwab/market/option_chain_service.py create mode 100644 schwab/orders/__init__.py create mode 100644 schwab/orders/reader.py create mode 100644 schwab/orders/sync.py create mode 100644 schwab/positions/__init__.py create mode 100644 schwab/positions/models.py create mode 100644 schwab/positions/reader.py create mode 100644 schwab/positions/sync.py create mode 100644 schwab/settings.py create mode 100644 schwab/streaming/__init__.py create mode 100644 schwab/streaming/client.py create mode 100644 storage/__init__.py create mode 100644 storage/db.py create mode 100644 storage/event_schema.py create mode 100644 storage/event_store.py create mode 100644 storage/pg_schema.sql create mode 100644 storage/projections.py create mode 100644 storage/replay.py create mode 100644 storage/schema.py create mode 100644 strategy/__init__.py create mode 100644 strategy/backtest/__init__.py create mode 100644 strategy/backtest/engine.py create mode 100644 strategy/backtest/outcomes.py create mode 100644 strategy/evaluation/__init__.py create mode 100644 strategy/evaluation/criteria.py create mode 100644 strategy/evaluation/data_quality.py create mode 100644 strategy/evaluation/first_candles.py create mode 100644 strategy/evaluation/levels.py create mode 100644 strategy/evaluation/llm_integration.py create mode 100644 strategy/evaluation/quality.py create mode 100644 strategy/evaluation/regime.py create mode 100644 strategy/evaluation/setup_evaluator.py create mode 100644 strategy/evaluation/storage.py create mode 100644 strategy/evaluation/structure.py create mode 100644 strategy/evaluation/tracking.py create mode 100644 strategy/evaluation/volume_metrics.py create mode 100644 strategy/models.py create mode 100644 strategy/risk/__init__.py create mode 100644 strategy/risk/entry_cuts.py create mode 100644 strategy/risk/position_sizing.py create mode 100644 strategy/universe.py create mode 100644 telemetry/__init__.py create mode 100644 telemetry/events.py create mode 100644 tests/__init__.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/test_end_to_end.py create mode 100644 tests/synthetic.py create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/alpaca_paper/__init__.py create mode 100644 tests/unit/alpaca_paper/test_execution_and_sync.py create mode 100644 tests/unit/api/__init__.py create mode 100644 tests/unit/api/test_dashboard_api.py create mode 100644 tests/unit/research/__init__.py create mode 100644 tests/unit/research/test_ingestion.py create mode 100644 tests/unit/runtime/__init__.py create mode 100644 tests/unit/runtime/test_watcher.py create mode 100644 tests/unit/schwab/__init__.py create mode 100644 tests/unit/schwab/test_schwab_integration.py create mode 100644 tests/unit/storage/__init__.py create mode 100644 tests/unit/storage/test_event_store.py create mode 100644 tests/unit/storage/test_interactive_projections.py create mode 100644 tests/unit/test_backtest_engine.py create mode 100644 tests/unit/test_setup_evaluator.py create mode 100644 tests/unit/test_trading_execution.py create mode 100644 trading_execution.py create mode 100644 trading_mode.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..dca0fe5 --- /dev/null +++ b/.env.example @@ -0,0 +1,74 @@ +# ============================================================ +# Momentum — environment configuration (copy to .env and fill) +# ============================================================ + +# ---- Alpaca (market data + paper broker) — REQUIRED for live paper trading +ALPACA_API_KEY= +ALPACA_SECRET_KEY= +# Alpaca paper trading endpoint (default is fine) +APCA_API_BASE_URL=https://paper-api.alpaca.markets +# These two aliases are also read by the paper client: +APCA_API_KEY_ID= +APCA_API_SECRET_KEY= + +# ---- Schwab (live broker, Milestone 3) — OPTIONAL for paper trading +SCHWAB_MARKET_DATA_APP_KEY= +SCHWAB_MARKET_DATA_APP_SECRET= +SCHWAB_BROKER_APP_KEY= +SCHWAB_BROKER_APP_SECRET= +SCHWAB_REDIRECT_URI= +SCHWAB_TOKEN_PATH=data/schwab_tokens.json + +# ---- Optional research / news providers +POLYGON_API_KEY= +FINNHUB_API_KEY= +Alpha_Vantage_Api_Key= + +# ============================================================ +# Live paper-trading orchestrator (run_live_paper.py) +# ============================================================ + +# Execution destination: alpaca_paper (safe) or schwab_live (REAL MONEY) +TRADING_EXECUTION_MODE=alpaca_paper +TRADING_EXECUTION_ENABLED=1 +TRADING_EXECUTION_INTERVAL_SECONDS=30 +TRADING_AUTO_APPROVE=0 # 0 = manual approval in the dashboard, 1 = auto-arm (no approval) +TRADING_MAX_ORDERS_PER_TICK=1 +TRADING_MAX_CONCURRENT_POSITIONS=3 +TRADING_RISK_PER_TRADE_PCT=0.01 # fraction of equity risked per trade +# --- entry mechanism (Ross-Cameron-style auto-arm; all tunable) --- +TRADING_REWARD_MULTIPLE=2.0 # target = entry + this * (entry - stop); 2R default +TRADING_ENTRY_ORDER_TYPE=limit # limit = rest at trigger (cancellable); market = fill now +TRADING_ENTRY_TIMEOUT_BARS=2 # cancel an unfilled entry after N minutes resting (0 = off) +TRADING_ENTRY_INVALIDATE_PCT=0.0 # cancel if price breaks this far below entry (0 = any break; <0 = off) +TRADING_ENTRY_GUARD_INTERVAL_SECONDS=5 # how often to re-check armed entries vs the LIVE last-trade price +TRADING_MAX_DAILY_LOSS_PCT=0.03 # daily-loss circuit breaker + +# Intraday watcher (signal engine) +WATCHER_ENABLED=1 +WATCHER_INTERVAL_SECONDS=60 +WATCHER_EVENT_DB_PATH=./data/momentum.duckdb +WATCHER_MAX_SYMBOLS=25 +WATCHER_MIN_BARS=30 +WATCHER_MIN_QUALITY=0.30 +# Comma-separated symbols to follow all day. If empty, candidates are +# discovered from whatever has minute bars in market.duckdb for today. +WATCHER_SYMBOLS=AAPL,TSLA,NVDA,AMD,SOFI +# Price band for auto-discovered candidates (Ross Cameron default 1-20). +WATCHER_PRICE_MIN=1 +WATCHER_PRICE_MAX=20 + +# Live minute-bar ingestion (feeds the watcher with active market data) +LIVE_BARS_ENABLED=1 +LIVE_BARS_INTERVAL_SECONDS=60 +LIVE_BARS_LOOKBACK_MINUTES=120 + +# Broker state sync +ALPACA_PAPER_SYNC_ENABLED=1 +ALPACA_PAPER_SYNC_INTERVAL_SECONDS=60 +ACCOUNT_SYNC_ENABLED=0 # Schwab live account sync +ORDER_SYNC_ENABLED=0 # Schwab live order sync + +# Dashboard +DASHBOARD_HOST=0.0.0.0 +DASHBOARD_PORT=8010 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5b02df8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,60 @@ +# Environment variables +.env +.env.local +.env.*.local + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +.venv +venv/ +ENV/ +env/ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Ruff +.ruff_cache/ + +# Data that may contain sensitive info — never version runtime data/DBs +data/ +*.wal + +# Large artifacts / legacy — do not version +momentum-rebuilt.zip +*.zip +archive/ +uv.lock diff --git a/ARCHITECTURE_VISUAL.html b/ARCHITECTURE_VISUAL.html new file mode 100644 index 0000000..3eaf4fa --- /dev/null +++ b/ARCHITECTURE_VISUAL.html @@ -0,0 +1,664 @@ + + + + + + Momentum Trading System - Architecture Diagram + + + +
+

⚡ Momentum Trading System Architecture

+ +
+ + + + + + + + + + + + + + + + + + Alpaca IEX Bars + + + + Ingestion Module + + + + Research DB + (market_data) + + + + EVENT STORE + (momentum_events.duckdb) + append-only canonical events + + + + Watcher + symbol evaluation + state machine + + + + Execution Service + sizing + approval + order submission + + + + Alpaca Sync + positions, orders, + account state + + + + Alpaca Paper + order submission + & fills + + + + Dashboard API & UI + http://127.0.0.1:8010 + projections + SSE streaming + + + + Watchlist + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+
+ Data Flow: Market data, bars, watchlist +
+
+
+ Process: Computation & evaluation +
+
+
+ Events: Canonical state changes +
+
+
+ Storage: Persistent data +
+
+
+ Output: User-facing interface +
+
+
+ External: Broker APIs +
+
+ + +
+
📊 Complete Processing Flow (Every Tick)
+ +
+ 1 + INGEST → + ingest_live_minute_bars() + + ingest_daily_history() +
Latest 1-minute bars from Alpaca IEX into research/market.duckdb + daily backfill for gap% and RVOL calculations +
+ +
+ 2 + DISCOVER → + ResearchWatchlistProvider.get_candidates() +
Pull symbols from watchlist (manual, screener, or most-actives from Alpaca) +
+ +
+ 3 + WATCH → + Watcher.tick() +
For each symbol: fetch bars from research DB, evaluate 9 setup criteria, emit symbol_discovered / state_changed events +
+ +
+ 4 + EVALUATE → + evaluate_setup() +
Pure strategy logic: gap%, RVOL, bull-flag structure, price action quality. Emit criteria_evaluated events. +
+ +
+ 5 + SIGNAL → + Watcher emits signal_ready +
Symbol passes ≥60% of criteria (tunable). Debounced (once per session unless it falls back to blocked). +
+ +
+ 6 + EXECUTE → + TradingExecutionService.tick() +
On signal_ready: position sizing → risk checks (max concurrent, daily loss circuit breaker) → order_approval_requested event +
+ +
+ 7 + APPROVE → + Dashboard or --auto-approve +
Manual: click Approve in dashboard. Auto: env flag TRADING_AUTO_APPROVE=1. Emit order_approved event. +
+ +
+ 8 + SUBMIT → + AlpacaPaperExecutor.submit() +
Bracket order: entry (limit or market), profit target (entry + reward_multiple × risk), stop loss. Emit order_submitted event. +
+ +
+ 9 + SYNC → + AlpacaPaperSync.sync_account() +
Poll Alpaca: account balance, open positions, pending orders. Emit account_summary_updated, account_positions_updated, account_orders_updated events. +
+ +
+ 10 + PROJECT → + Dashboard (SSE stream) +
Read-only projections over event store: P&L, open positions, approval queue, activity feed, sparkline charts. Fall back to polling if SSE drops. +
+
+ + +
+
🔧 Key Modules & Responsibilities
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModuleResponsibilityKey Files
research/ingestionIngest market data (bars, daily history), discover candidates, classify market sessionsmarket_data.py, watcher_task.py, scheduler.py
research/Query and project research databases (minute/daily bars, gappers, RSS news)query.py, schema.py, multi_schema.py
runtime/watcherSymbol state machine: discovered → watching → ready | blocked. Emit all watcher events.watcher.py
strategy/evaluationPure setup evaluation: 9 criteria (gap%, RVOL, bull-flag, structure, quality, etc.)setup_evaluator.py
strategy/riskPosition sizing, risk per trade, reward multiple, daily loss circuit breakerposition_sizing.py
strategy/backtestIntraday backtest engine: replay bars, simulate entry/exit, calculate P&Lengine.py
storage/event_storeAppend-only event log (canonical source of truth). Correlate by session_id.event_store.py, event_schema.py
storage/projectionsQuery event store for current state: approval queue, open positions, P&L, etc.projections.py
alpaca_paper/Alpaca paper broker integration: order submission, position sync, account stateclient.py, execution.py, sync.py
schwab/Schwab live integration (OAuth, market data, positions, orders, health)oauth.py, market_data.py, positions.py, health.py
api/Dashboard JSON API + SSE streaming. Read-only projections served as REST + events.main.py, static/
trading_execution.pySignal → sizing → approval → order pipeline. Manages armed entries & timeouts.trading_execution.py
+
+ + +
+
📨 Key Event Types (Event-Sourced)
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EventEmitted ByPurpose
symbol_discoveredWatcherNew symbol added to watchlist
symbol_state_changedWatcherSymbol transitions: discovered → watching → ready | blocked
criteria_evaluatedWatcher (via evaluate_setup)9 setup criteria evaluated for symbol (pass/fail/not-evaluated)
signal_readyWatcherSymbol ready: ≥60% criteria passed. Debounced once per session.
signal_blockedWatcherSymbol failed criteria check. Resets debounce.
order_approval_requestedTradingExecutionServiceReady signal → sizing + risk checks passed. Awaiting approval or auto-submit.
order_approvedDashboard (manual) or TradingExecutionService (auto)Approval given. Next: submit to broker.
order_rejectedDashboard (manual) or TradingExecutionService (risk rule)Approval denied or risk rule triggered. Signal dequeued.
order_submittedAlpacaPaperExecutorBracket order sent to Alpaca (entry + target + stop).
account_positions_updatedAlpacaPaperSyncOpen positions snapshot from Alpaca. Includes entry, qty, P&L.
account_orders_updatedAlpacaPaperSyncPending orders snapshot from Alpaca. Tracks fills, cancellations, expirations.
account_summary_updatedAlpacaPaperSyncAccount balance, buying power, margin, NAV, realized P&L.
+
+ + +
+
🚀 Main Entry Point: run_live_paper.py
+ +
+ Overall orchestrator that ties everything together: +
    +
  1. Parse CLI args (--once, --symbols, --auto-approve, --no-dashboard)
    2. +
  2. Load .env and configure Alpaca client + settings
    3. +
  3. Initialize EventStore (momentum_events.duckdb)
    4. +
  4. Open ResearchConnection (market_data.duckdb) + ResearchWatchlistProvider
    5. +
  5. Create Watcher + TradingExecutionService + AlpacaPaperSync + AlpacaPaperExecutor
    6. +
  6. Optional: start DashboardAPI server (SSE + JSON endpoints) in background thread
    7. +
  7. Main loop (or single pass with --once): +
      +
    • Ingest latest minute bars + daily history
      • +
    • Watcher.tick() → symbol evaluation + state changes + signal emissions
      • +
    • TradingExecutionService.tick() → approval requests + submissions
      • +
    • AlpacaPaperSync.sync_account() → positions + orders + account state
      • +
    • Sleep (configurable interval, e.g., 60 seconds)
      • +
    +
    8. +
+
+
+ + +
+
🔍 Debugging: Where to Look When Something Breaks
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SymptomLikely CauseCheck
No symbols appearing in WatcherWatchlist provider not returning candidatesResearchWatchlistProvider.get_candidates() → is research DB populated? Check data/research/market.duckdb
Symbols discovered but never go readyCriteria evaluation failing or threshold too highWatcher → criteria_evaluated events. Check setup_evaluator.py scores. Lower ready_score_pct or min_quality config.
Signal ready but no approval requestTradingExecutionService.tick() not running or risk rule triggeredCheck trading_execution.py: max_concurrent_positions, max_daily_loss_pct. Emit risk_rule_triggered? Check event store.
Approval request but order never submittedDashboard approval not clicked OR auto-approve not setDashboard approval queue visible? Or set TRADING_AUTO_APPROVE=1 in .env
Order submitted but no fills from AlpacaAPI keys missing or Alpaca API downCheck ALPACA_API_KEY / ALPACA_SECRET_KEY in .env. Run --once to test without loop.
Dashboard not updating (SSE fails)Network drop or event stream blockedDashboard falls back to polling. Check http://127.0.0.1:8010/api/events (should return JSON). Check browser console.
Backtests showing different P&L than liveBar data mismatch or backtest engine using different fillsBacktest engine simulates limit fills at exact prices. Live uses actual Alpaca fills. Compare bars in research DB vs Alpaca API.
+
+ + +
+
💾 Data Storage Locations
+ +
+ Event Store (Canonical Source of Truth)
+ data/momentum_events.duckdb — append-only events, indexed by timestamp + session_id +
+ +
+ Research Database (Market Data)
+ data/research/market.duckdb — minute bars + daily bars ingested from Alpaca +
+ +
+ Schwab Integration
+ data/schwab_tokens.json — OAuth tokens (git-ignored, 0600 perms) +
+ +
+ Configuration
+ .env — API keys, trading settings (git-ignored) +
+
+
+ + diff --git a/BACKTEST_GUIDE.md b/BACKTEST_GUIDE.md new file mode 100644 index 0000000..35c7680 --- /dev/null +++ b/BACKTEST_GUIDE.md @@ -0,0 +1,120 @@ +# Comprehensive Backtest Guide + +This guide will help you run a multi-week backtest to assess the profitability and risk profile of the momentum strategy before trading live. + +## Quick Start (Windows Terminal) + +Run a 1-week backtest on small-cap momentum names (the strategy's sweet spot): + +```powershell +uv run comprehensive_backtest.py --start 2026-06-05 --end 2026-06-12 +``` + +This analyzes every trading day in that range, pulling minute bars for the specified symbols and reporting: +- **Daily P&L** — win/loss pattern by day +- **Symbol breakdown** — which tickers traded well/poorly +- **Trade-by-trade details** — entry/exit, profit/loss, reason for exit +- **Statistical summary** — win rate, average R-multiple, profit factor, risk metrics +- **Loss analysis** — patterns in losses (gap-throughs, wide stops, etc.) + +## What the Output Tells You + +### Summary Metrics + +- **Win Rate**: % of trades that were profitable. 40%+ is acceptable; 50%+ is strong. +- **Profit Factor**: Total wins ÷ absolute value of total losses. + - < 1.3: marginal edge or slippage issues (investigate before live) + - 1.3–2.0: decent edge + - > 2.0: strong edge +- **Avg R-Multiple**: Average return per unit of risk. 1.5+ suggests the strategy rewards you for the risk taken. +- **Max Loss**: Largest single loss. If it's 2–3× worse than your average loss, stops may be too wide or slippage is severe. + +### Red Flags to Watch For + +1. **Profit factor < 1.3** — your losses are eating most of your wins +2. **Win rate < 40%** — even with 2:1 R ratio, this struggles to be profitable +3. **Max loss >> average loss** — suggests gap-throughs or wide stops on losers +4. **Inconsistent daily results** — if one bad day wipes out a week's gains, volatility is too high + +## Advanced Options + +```powershell +# Test a different date range (e.g., last 2 weeks) +uv run comprehensive_backtest.py --start 2026-05-27 --end 2026-06-12 + +# Test on your watched symbols instead of the defaults +uv run comprehensive_backtest.py --start 2026-06-05 --end 2026-06-12 --symbols SNDL,MARA,RIOT + +# Test a full month (if available) +uv run comprehensive_backtest.py --start 2026-05-13 --end 2026-06-12 +``` + +## Important Caveats + +1. **This is NOT the live execution path** — the backtest engine shows signal→entry→exit on historical bars, but it does NOT exercise: + - The new auto-arm resting-limit logic + - The tick-by-tick entry invalidation (live prices) + - The approval queue or manual trading decisions + + All of those features only run in `uv run run_live_paper.py --auto-approve`. This backtest proves the signal-generation and position-management logic works; it's not a full replay of the live system. + +2. **IEX feed gaps** — free tier only sees IEX trades (a subset of consolidated). On illiquid small-caps there may be minutes with no prints, causing simulated fills to happen on stale data. The paid SIP feed (ALPACA_DATA_FEED=sip) would see more ticks. + +3. **Minute-bar vs tick-by-tick** — the backtest assumes fills happen at the bar open of the next bar. Live, entries/exits are much more granular (actual filled prices as they happen). This is usually conservative (against you) but can help on some exits. + +4. **Slippage assumptions** — the engine includes configurable slippage (base_spread_pct in config). The default assumes ~0.05% slippage; real fills vary by size, liquidity, and market conditions. + +## Interpretation Guide + +### Strong Strategy (Should Trade) +- Win rate ≥ 45% +- Profit factor ≥ 1.8 +- Avg R ≥ 1.5R +- Max loss ≤ 2× average loss +- Consistent week-to-week results + +### Marginal Strategy (Investigate Further) +- Win rate 40–45% +- Profit factor 1.3–1.8 +- Avg R ≈ 1.0–1.5R +- Max loss 2–3× average loss +- High daily variance + +### Weak Strategy (Do Not Trade) +- Win rate < 40% +- Profit factor < 1.3 +- Avg R < 0.5R +- Max loss > 3× average loss +- Losing weeks common + +## Next Steps + +1. **Run the backtest** for the most recent 2–4 weeks using small-cap symbols (SNDL, MARA, RIOT, BBAI, SOUN, PLUG). +2. **Review the loss analysis** — click "Loss Analysis" section and understand why you're losing. Is it: + - Entries are wrong (enter on fakeouts)? → Tighten entry criteria + - Stops are too wide (gap-throughs)? → Use a tighter stop or accept wider slippage + - Exits are leaving money? → Adjust target or exit rules +3. **Check the daily pattern** — if one symbol dominates losses, exclude it (maybe illiquid on IEX). +4. **If strong** → uv run run_live_paper.py --auto-approve on the next trading day. +5. **If marginal/weak** → dig into the top losers to understand the pattern before risking real capital. + +## Debugging a Bad Backtest + +If you see 0 trades despite seeing signals: + +```powershell +# First, make sure minute bars were fetched +uv run fetch_minute_bars.py SNDL MARA RIOT --lookback 5000 + +# Then try a single-symbol, single-day backtest to see detailed output +uv run backtest_cli.py --symbol SNDL --date 2026-06-12 +``` + +If you get "no symbols with minute bars," the date may not have tradeable data (check it's a weekday) or the lookback was too short. + +## Questions? + +- The backtest engine: `strategy/backtest/engine.py` +- Setup evaluation (what defines a "signal"): `strategy/evaluation/setup_evaluator.py` +- Risk/position sizing: `strategy/risk/position_sizing.py` +- Config defaults: `config.py` (BacktestConfig section) diff --git a/BUILD_SUMMARY.md b/BUILD_SUMMARY.md new file mode 100644 index 0000000..70836b0 --- /dev/null +++ b/BUILD_SUMMARY.md @@ -0,0 +1,507 @@ +# Complete Build Summary & Implementation Roadmap + +## What You Asked For + +> "I can't see what's happening. I don't know if bars are being collected. I can't verify research data is accurate. I can't test each module independently. The UI doesn't help debug. And I need a better visualization." + +--- + +## What Was Built (Phase 1 ✅) + +### Core Infrastructure + +**1. Module Contracts** (`core/contracts.py`) +- Defined 5 typed I/O contracts (ingestion, research, evaluation, execution, broker) +- Every module now has explicit `XyzRequest` (input) and `XyzResult` (output) types +- IDE autocomplete friendly +- Makes bugs obvious when contracts don't match + +**2. Telemetry Collection** (`core/telemetry.py`) +- `TelemetryCollector`: Record metrics during module execution +- `TelemetryEmitter`: Emit events to event store + callbacks +- `ModuleMetrics`: Standard metrics (duration, counts, custom fields) +- Global emitter instance for easy access + +**3. Data Validation** (`core/validation.py`) +- `SchemaValidator`: Check required fields and types +- `RangeValidator`: Numeric bounds checking +- `ChecksumValidator`: Data integrity verification +- `BusinessRuleValidator`: Custom validation logic +- `BarDataValidator`: Pre-built OHLCV validation + +**4. New Event Types** (`storage/event_schema.py`, extended) +- `ModuleTickEvent`: Module execution telemetry +- `DataValidationEvent`: Data quality checkpoints +- `SymbolEvaluationDetailEvent`: Deep-dive symbol metrics + +--- + +## Complete 8-Phase Implementation Plan + +### ✅ Phase 1: Core Infrastructure (DONE) +**Time:** ~2 hours +**Status:** Complete + +**Deliverables:** +- Core contracts and validation framework +- Telemetry collection and emission +- 3 new event types in event store +- Documentation of all new types + +**Why it matters:** Foundation for everything else. Modules can now be transparent. + +--- + +### 📍 Phase 2: Module Instrumentation (NEXT) +**Estimated time:** ~4 hours +**What it does:** Add telemetry to existing modules + +**Work:** +1. Wrap each module with `TelemetryCollector` +2. Emit `ModuleTickEvent` on completion +3. Add validation checkpoints with `DataValidationEvent` +4. Emit `SymbolEvaluationDetailEvent` during evaluation +5. Update `run_live_paper.py` to initialize telemetry + +**Example output:** +``` +ModuleTickEvent: ingestion completed in 245ms, 500 rows +ModuleTickEvent: research completed in 156ms, 5 symbols +SymbolEvaluationDetailEvent: AAPL gap_checked (2.1%) +SymbolEvaluationDetailEvent: AAPL rvol_calculated (1.2x) +DataValidationEvent: evaluation schema valid +``` + +**Files to modify:** +- `research/ingestion/market_data.py` — add ingestion telemetry +- `research/module.py` — add research telemetry +- `strategy/evaluation/setup_evaluator.py` — add evaluation telemetry + detail events +- `trading_execution.py` — add execution telemetry +- `alpaca_paper/sync.py` — add broker sync telemetry +- `run_live_paper.py` — initialize emitter, pass to modules + +**Tests:** Run existing tests to ensure telemetry doesn't break anything + +--- + +### 🔲 Phase 3: Inspection Dashboard API +**Estimated time:** ~6 hours +**What it does:** Build backend REST API for module data + +**Files to create:** +- `api/inspection.py` — new API endpoints + +**Endpoints:** +``` +GET /api/inspection/modules → module metrics table +GET /api/inspection/pipeline → pipeline flow stats +GET /api/inspection/symbol/:symbol → detailed eval timeline +GET /api/inspection/data-quality → validation report +GET /api/inspection/events → SSE stream of telemetry +GET /api/inspection/module/:module/history → last N executions +``` + +**Projections to build** (`storage/projections.py`, add new functions): +```python +def query_module_performance() → [{module, avg_latency, last_run, error_rate}] +def query_symbol_evaluation_detail(symbol) → {stages with metrics} +def query_data_quality_snapshot() → {validation stats, freshness} +def query_pipeline_stats(session_id) → {module timings, input/output counts} +``` + +**Tests:** +- Test each endpoint returns correct data +- Test projections handle missing data gracefully +- Test SSE streaming with multiple clients + +--- + +### 🔲 Phase 4: Web Dashboard Frontend +**Estimated time:** ~8 hours +**What it does:** Build React SPA for visualization + +**Files to create:** +``` +api/inspection/ +├── index.html # Single-page app shell +├── app.js # React app entry +├── components/ +│ ├── PipelineDiagram.js # Mermaid auto-layout diagram +│ ├── ModuleGrid.js # Performance metrics table +│ ├── SymbolInspector.js # Search + expandable criteria +│ ├── EventTimeline.js # Live event log with filtering +│ └── DataQualityReport.js # Validation statistics +└── styles/ + └── dashboard.css # Responsive styling +``` + +**Features:** +- Real-time SSE updates +- Interactive symbol search +- Expandable evaluation stages +- Sparkline charts (recharts) +- Event timeline with filtering +- Data quality heatmap +- Mobile responsive + +**Design:** +- Dark theme (consistent with trading terminals) +- One symbol per row, expandable details +- Color coding (✓ green, ✗ red, ? gray) +- Click to drill down into metrics + +**Tests:** +- Component rendering tests +- SSE connection/fallback tests +- Real-world data from phase 3 + +--- + +### 🔲 Phase 5: Module Refactoring +**Estimated time:** ~6 hours +**What it does:** Modularize existing code into explicit contracts + +**Files to create:** +``` +modules/ +├── ingestion.py # BarIngestRequest → BarIngestResult +├── research.py # ResearchRequest → ResearchResult +├── evaluation.py # EvaluationRequest → EvaluationResult +├── execution.py # ExecutionRequest → ExecutionResult +└── broker.py # BrokerRequest → BrokerResult +``` + +**Pattern (example for evaluation):** +```python +class EvaluationModule: + def run(self, request: EvaluationRequest) -> EvaluationResult: + """Pure function: request → result.""" + # Validate input + is_valid, failures = self.validate_input(request) + + # Execute strategy logic + result = evaluate_setup( + bars_df=request.bars_df, + symbol=request.symbol, + previous_close=request.previous_close, + avg_daily_volume=request.avg_daily_volume + ) + + # Return strongly-typed result + return EvaluationResult( + symbol=request.symbol, + status=result['status'], + score=result['score'], + criteria=result['criteria'], + validation_failures=failures + ) +``` + +**Benefits:** +- Each module is a pure function: `input → output` +- Can test in isolation +- Can swap implementations (real vs. mock) +- Dependency injection friendly + +**Dependency injection (trait-style):** +```python +class EvaluationModule: + def __init__(self, research_db: ResearchDB = None, ...): + self.research_db = research_db or RealResearchDB() + + # In tests: + module = EvaluationModule(research_db=MockResearchDB()) +``` + +**Tests:** +- Unit test each module in isolation +- Integration test module chain +- Contract compliance tests + +--- + +### 🔲 Phase 6: Module Testing Framework +**Estimated time:** ~5 hours +**What it does:** Write comprehensive test suite + +**Files to create:** +``` +tests/ +├── modules/ +│ ├── conftest.py # fixtures +│ ├── test_ingestion_module.py # 8 tests +│ ├── test_research_module.py # 6 tests +│ ├── test_evaluation_module.py # 10 tests +│ ├── test_execution_module.py # 8 tests +│ ├── test_broker_module.py # 8 tests +│ └── fixtures/ +│ ├── bull_flag_bars.py # Synthetic test data +│ ├── fading_bars.py +│ ├── gap_up_bars.py +│ └── low_volume_bars.py +│ +├── integration/ +│ ├── test_pipeline_e2e.py # Full flow test +│ ├── test_signal_flow.py # Signal lifecycle +│ └── test_event_consistency.py # Audit trail +│ +└── scenarios/ + ├── bull_flag_setup.py # Golden path + ├── fading_tape_blocked.py # Negative case + └── risk_rule_rejection.py # Execution blocker +``` + +**Test patterns:** +```python +def test_evaluation_module_ready(): + """Module accepts input, produces output.""" + module = EvaluationModule() + request = EvaluationRequest( + symbol="AAPL", + bars_df=bull_flag_bars(), + previous_close=150.0, + avg_daily_volume=1_000_000, + session_date=date(2026, 6, 15) + ) + + result = module.run(request) + + assert result.status == "ready" + assert result.score >= 60 + assert result.entry_price > 0 + # ... + +def test_evaluation_module_dry_run(): + """Module works without database.""" + module = EvaluationModule(research_db=MockResearchDB()) + # Same test as above, but doesn't touch real DB + +def test_evaluation_module_validation(): + """Module validates input, captures errors.""" + module = EvaluationModule() + request = EvaluationRequest( + symbol="AAPL", + bars_df=invalid_bars(), # Zero volume bar + # ... + ) + + result = module.run(request) + + assert not result.success + assert len(result.validation_failures) > 0 +``` + +**Run tests:** +```bash +pytest tests/modules/ # All module tests +pytest tests/modules/test_evaluation_module.py -v # Specific module +pytest tests/integration/ # Full pipeline +``` + +--- + +### 🔲 Phase 7: Dry-Run Mode & Synthetic Data +**Estimated time:** ~4 hours +**What it does:** Add `--dry-run` flag for testing without APIs + +**Files to create:** +``` +tests/ +├── synthetic.py # Bar generators +│ ├── bull_flag_bars() # Ideal setup +│ ├── fading_bars() # Will block +│ ├── gap_up_bars() # Gap but low volume +│ └── spike_bars() # Volume spike +│ +└── providers/ + ├── mock_research_db.py # In-memory research DB + ├── mock_broker.py # Simulated fills + └── mock_bars_provider.py # Synthetic bars + +core/ +└── test_mode.py # DryRun context manager +``` + +**Synthetic data generators:** +```python +def bull_flag_bars(high_price=150.0, low_price=145.0, bars=50): + """Generate realistic bull-flag OHLCV bars.""" + # - Gap up on open + # - Initial breakout + # - Pullback formation + # - Higher lows pattern + # High volume throughout + # Returns: DataFrame +``` + +**Usage:** +```bash +# Dry-run mode: no API keys, no real orders +python run_live_paper.py --dry-run \ + --symbols AAPL,TSLA,AMD \ + --bars-provider synthetic:bull_flag \ + --num-ticks 5 \ + --inspection-dashboard + +# Output: +# Tick 1: ingestion ✓, research ✓, evaluation ✓ (2 ready) +# Tick 2: execution ✓ (approved), broker ✓ (simulated fills) +# ...all visible in inspection dashboard +``` + +**Key benefits:** +- Deterministic test runs (same data every time) +- No API rate limits +- No need for live market hours +- Test configuration changes instantly +- Use in CI/CD for regression testing + +--- + +### 🔲 Phase 8: Rich TUI (Optional) +**Estimated time:** ~3 hours +**What it does:** Terminal UI for SSH/CI/CD environments + +**Files to create:** +``` +tui/ +├── dashboard.py # Rich-based TUI +└── components.py # Reusable widgets +``` + +**Features:** +- Progress bars for each module (ingestion, research, eval, exec, broker) +- Live metrics table (latencies, counts, error rates) +- Event log (colorized by severity) +- Keyboard commands (pause, resume, copy logs) +- Works over SSH (no browser needed) + +**Usage:** +```bash +python run_live_paper.py --tui +``` + +**Output:** +``` +[Ingestion] ████████░░░ 80% (400/500) +[Research] ██████████ 100% (5/5) +[Evaluation] ██████████ 100% (5/5 → 2 ready) +[Execution] ██████░░░░ 60% (1/2 approved) +[Broker] ████░░░░░░ 40% (2/5 submitted) + +Recent Events: + [09:35:15] ✓ TSLA evaluated, ready (score 78%) + [09:35:14] ✗ AAPL blocked (rvol 1.2x < 1.5x) + [09:35:13] ✓ Ingestion complete (500 rows) +``` + +--- + +## Implementation Sequence + +**Week 1:** +- Phase 1: Core infrastructure ✅ (DONE) +- Phase 2: Module instrumentation (4-6 hours) + +**Week 2:** +- Phase 3: Inspection API (6 hours) +- Phase 4: React dashboard (8 hours) + +**Week 3:** +- Phase 5: Module refactoring (6 hours) +- Phase 6: Test framework (5 hours) + +**Week 4:** +- Phase 7: Dry-run mode (4 hours) +- Phase 8: TUI (3 hours, optional) + +**Total:** ~4 weeks for full implementation, or ~2 weeks for core (phases 1-4). + +--- + +## Why This Solves Your Problems + +| Problem | Solution | Phase | +|---------|----------|-------| +| Can't see bars being collected | ModuleTickEvent shows input/output counts + validation | 1 + 2 | +| Can't verify research data | DataValidationEvent + SymbolEvaluationDetailEvent | 1 + 2 | +| Can't test modules independently | Module contracts + mock dependencies | 5 + 6 | +| UI doesn't help debug | Inspection dashboard with symbol inspector | 3 + 4 | +| No visibility into evaluation steps | SymbolEvaluationDetailEvent for each stage | 1 + 2 | +| Can't dry-run without APIs | --dry-run + synthetic bars | 7 | + +--- + +## Next Immediate Steps + +### Phase 2 Kickoff (Start Now) + +1. **Identify all modules to instrument:** + - `research/ingestion/market_data.py` — `ingest_live_minute_bars()` + - `research/module.py` — `get_candidates()`, `get_bars()` + - `strategy/evaluation/setup_evaluator.py` — `evaluate_setup()` + - `trading_execution.py` — `TradingExecutionService.tick()` + - `alpaca_paper/sync.py` — `AlpacaPaperSync.sync_account()` + +2. **For each module:** + - Wrap with `TelemetryCollector(module_name)` + - Record input/output counts + - Add validation with `emit_data_validation()` + - Emit `ModuleTickEvent` on completion + - Update error handling to record errors + +3. **Update `run_live_paper.py`:** + - Initialize `TelemetryEmitter` + - Pass to modules (or use global instance) + - Test that existing behavior unchanged (should be!) + +4. **Verify:** + - Run `python run_live_paper.py --once` + - Check event store has new events + - Verify metrics are accurate + +--- + +## Success Criteria + +After Phase 4 complete, you should be able to: + +✅ Open http://127.0.0.1:8011 (inspection dashboard) +✅ See real-time module pipeline diagram +✅ Search for a symbol and see complete evaluation journey +✅ Understand exactly why a symbol is ready/blocked +✅ See timestamp, metrics, and raw data for each stage +✅ View live event stream +✅ Access data quality report + +**Time to understand "why isn't AAPL signaling?":** < 10 seconds + +--- + +## Files You Now Have + +``` +core/ +├── __init__.py +├── contracts.py ← Module I/O types +├── telemetry.py ← Metric collection +└── validation.py ← Data validation + +storage/ +└── event_schema.py ← (extended) 3 new event types + +Documentation/ +├── PHASE1_IMPLEMENTATION.md ← How Phase 1 works +├── INSPECTION_ROADMAP.md ← Full system visibility +└── BUILD_SUMMARY.md ← This file +``` + +--- + +## Questions? + +Each phase has clear, testable deliverables. After Phase 2 (instrumenting existing modules), you'll start seeing telemetry events in your event store. + +After Phase 4 (React dashboard), you'll have full visibility into your system. + +The foundation is solid. Ready to move to Phase 2? diff --git a/DATA_TRANSFORMATIONS.md b/DATA_TRANSFORMATIONS.md new file mode 100644 index 0000000..9c16f3f --- /dev/null +++ b/DATA_TRANSFORMATIONS.md @@ -0,0 +1,380 @@ +# Data Transformations Flow + +## Quick Reference: What Data Flows Where and How It Changes + +### INGESTION LAYER +``` +Alpaca REST API (live 1-min bars) + ↓ ingest_live_minute_bars() + ↓ transform: {symbol, timestamp, open, high, low, close, volume} + ↓ +research/market.duckdb → bars table + +Alpaca REST API (daily bars history) + ↓ ingest_daily_history() + ↓ transform: {symbol, timestamp, open, high, low, close, volume} + ↓ +research/market.duckdb → daily_bars table +``` + +### DISCOVERY LAYER +``` +Manual Symbols (--symbols or WATCHER_SYMBOLS env) + ├─→ WatchCandidate(symbol, last_price, avg_volume) + │ +Alpaca Most-Actives Screener (if --discover) + ├─→ WatchCandidate(symbol, last_price, avg_volume) + │ +ResearchWatchlistProvider.get_candidates(session_date) + ↓ returns: List[WatchCandidate] + ↓ +Watcher ingests as new symbols to watch +``` + +### EVALUATION LAYER (CORE STRATEGY) +``` +research/market.duckdb (minute bars for symbol) + ↓ query_minute_bars(symbol, session_date) + ↓ returns: pd.DataFrame with {timestamp, open, high, low, close, volume} + ↓ +evaluate_setup(bars, symbol) + ├─ Extract metrics: + │ ├─ Gap% = (today_open - prev_close) / prev_close + │ ├─ RVOL = current_5min_volume / avg_20day_volume + │ ├─ Bull flag structure detection + │ ├─ Price action quality scoring + │ └─ 5 other criteria (tunable) + │ + ├─ Evaluate each criterion: pass/fail/not-evaluated + │ + ├─ Calculate score: + │ └─ criteria_passed / total_criteria * 100 = X% + │ + └─ returns: { + score: float, + criteria: [ + {name: "gap", passed: true, ...}, + {name: "rvol", passed: false, ...}, + ... + ], + entry: float, + stop: float, + target: float, + quality: float + } +``` + +### SIGNAL EMISSION +``` +evaluate_setup() results + ↓ + ├─ IF score ≥ ready_score_pct (60%) AND quality ≥ min_quality (30%): + │ └─ Watcher emits: SignalReadyEvent( + │ symbol, entry, stop, target, score, criteria, setup_data, ... + │ ) + │ ✓ Recorded in event store + │ ✓ Marked as "already signaled" (debounced) + │ + └─ ELSE: + └─ Watcher emits: SignalBlockedEvent( + symbol, reason, score, ... + ) + ✓ Resets signal debounce +``` + +### EXECUTION SIZING LAYER +``` +SignalReadyEvent(entry, stop, target, ...) + ↓ +TradingExecutionService.tick() + ├─ Check risk rules: + │ ├─ open_positions ≥ max_concurrent_positions? → REJECT + │ ├─ today_realized_pnl ≤ -max_daily_loss_pct * equity? → REJECT + │ └─ submitted_this_tick ≥ max_orders_per_tick? → SKIP + │ + └─ calculate_position_size(entry, stop, equity, risk_pct) + │ + ├─ risk_dollars = equity * risk_pct (e.g., 1% = $1,000 on $100k) + ├─ risk_per_share = entry - stop + ├─ shares = risk_dollars / risk_per_share + └─ returns: { + shares: int, + entry: float, + stop: float, + target: float (entry + (entry-stop)*reward_multiple), + risk_dollars: float, + notional: float (entry * shares) + } +``` + +### APPROVAL LAYER +``` +ExecutionRequest {shares, entry, stop, target, ...} + ↓ +TradingExecutionService emits: OrderApprovalRequestedEvent + ├─ Stored in event store + ├─ Projected into "approval_queue" projection + └─ Dashboard polls/streams this data + +Dashboard receives approval request: + ├─ Shows: symbol, entry, stop, target, shares, risk %, notional + ├─ Shows: sparkline chart of last 50 bars with entry/stop references + ├─ User clicks: Approve or Reject + │ + ├─ ON APPROVE: + │ └─ Dashboard emits: OrderApprovedEvent(order_id, ...) + │ ✓ Recorded in event store + │ ✓ TradingExecutionService detects and submits to broker + │ + └─ ON REJECT: + └─ Dashboard emits: OrderRejectedEvent(order_id, reason, ...) + ✓ Signal removed from queue + ✓ Session reset for that symbol (can signal again) +``` + +### BROKER SUBMISSION LAYER +``` +OrderApprovedEvent + ↓ +AlpacaPaperExecutor.submit() + ├─ Create bracket order: + │ ├─ Entry: limit @ entry_price (can be cancelled/timeout) + │ ├─ Target: limit @ target_price + │ └─ Stop: stop @ stop_price (market when hit) + │ + ├─ Call Alpaca REST API: POST /orders (bracket order) + │ └─ returns: {id, symbol, qty, status, ...} + │ + ├─ Emit: OrderSubmittedEvent(broker_order_id, entry, stop, target, ...) + │ └─ Recorded in event store + │ + └─ Track in _armed dict: + └─ _armed[order_id] = { + symbol, entry_price, broker_order_id, + armed_at, checks: {timeout, price_break} + } + +Ongoing monitoring: + ├─ Entry fills? + │ └─ Position opens (qty shares @ entry_price) + │ + ├─ Entry timeout? (after entry_timeout_bars minutes) + │ └─ Cancel unfilled entry order + │ + ├─ Price breaks back below entry - entry_invalidate_pct? + │ └─ Cancel unfilled entry order + │ + └─ Target OR Stop hit? + └─ Position closes (filled at target or stop) + ✓ Emit: OrderFilledEvent(symbol, entry, exit, qty, pnl, ...) + ✓ Recorded in event store +``` + +### ACCOUNT SYNC LAYER +``` +AlpacaPaperSync.sync_account() + │ + ├─ Alpaca REST GET /account + │ ↓ transform: {equity, buying_power, long_market_value, ...} + │ ↓ Emit: AccountSummaryUpdatedEvent(account_snapshot) + │ + ├─ Alpaca REST GET /positions + │ ↓ transform: [{symbol, qty, avg_fill, current_price, unrealized_pl}] + │ ↓ Emit: AccountPositionsUpdatedEvent(positions_list) + │ + ├─ Alpaca REST GET /orders?status=open + │ ↓ transform: [{id, symbol, qty, status, filled_qty, ...}] + │ ↓ Emit: AccountOrdersUpdatedEvent(orders_list) + │ + └─ Calculate session P&L: + ├─ Query event_store for all OrderFilledEvent in this session + ├─ For each: realized_pnl = (exit_price - entry_price) * qty + ├─ Aggregate: total_realized, total_unrealized, win_rate, avg_r + └─ Emit: SessionPnlUpdatedEvent(pnl_snapshot) +``` + +### PROJECTION LAYER (DASHBOARD READ-ONLY VIEWS) +``` +EVENT STORE (canonical source) + ├─ All events append-only, indexed by timestamp + session_id + │ + └─ Projections (reconstructed views): + │ + ├─ approval_queue(event_store) + │ ├─ Filter: OrderApprovalRequestedEvent (status != approved/rejected) + │ └─ returns: [{order_id, symbol, entry, stop, target, shares, risk%}] + │ + ├─ account_positions_snapshot(event_store) + │ ├─ Latest: AccountPositionsUpdatedEvent + │ └─ returns: [{symbol, qty, entry, current, unrealized_pl}] + │ + ├─ pnl_snapshot(event_store) + │ ├─ Latest: SessionPnlUpdatedEvent + │ └─ returns: {realized, unrealized, total, win_rate, avg_r, open_count, closed_count} + │ + ├─ activity_feed(event_store, limit=100) + │ ├─ All events, chronological, most recent first + │ └─ returns: [{timestamp, event_type, message, symbol, ...}] + │ + ├─ ready_signals_snapshot(event_store) + │ ├─ Filter: SignalReadyEvent (not yet approved/rejected) + │ └─ returns: [{symbol, entry, stop, target, score, criteria}] + │ + └─ symbol_criteria(event_store, symbol) + ├─ Latest: CriteriaEvaluatedEvent for this symbol + └─ returns: [{name, passed, reason, value}] (9 criteria) +``` + +### DASHBOARD DELIVERY LAYER +``` +Projections (from EVENT STORE) + │ + ├─ SSE Stream (/api/events) + │ ├─ WebSocket or Server-Sent Events + │ ├─ Pushes new events in real-time to all connected clients + │ └─ Format: {event_type, timestamp, symbol, ...} + │ + └─ REST JSON Endpoints: + │ + ├─ GET /api/approval-queue + │ └─ returns: approval_queue projection (JSON) + │ + ├─ GET /api/positions + │ └─ returns: account_positions_snapshot projection (JSON) + │ + ├─ GET /api/pnl + │ └─ returns: pnl_snapshot projection (JSON) + │ + ├─ GET /api/activity-feed + │ └─ returns: activity_feed projection (JSON, last 100 events) + │ + ├─ GET /api/symbol/:symbol/criteria + │ └─ returns: 9 criteria with pass/fail/not-evaluated + │ + ├─ GET /api/symbol/:symbol/bars?minutes=50 + │ └─ returns: [{timestamp, open, high, low, close, volume}] + │ (for sparkline chart rendering) + │ + └─ POST /api/approval/:order_id/approve|reject + └─ Emits OrderApprovedEvent or OrderRejectedEvent +``` + +### FINAL OUTPUT (DASHBOARD UI) +``` +Browser → http://127.0.0.1:8010 + │ + ├─ P&L Strip (top) + │ └─ Realized | Unrealized | Total | Win Rate | Avg R | Open | Closed + │ (refreshed by /api/pnl endpoint) + │ + ├─ Approval Queue (cards) + │ ├─ Symbol | Entry | Stop | Target | Shares | Risk $ | Risk % + │ ├─ Sparkline chart (last 50 bars, entry/stop reference lines) + │ └─ [Approve] [Reject] buttons + │ (refreshed by /api/approval-queue endpoint or SSE) + │ + ├─ Open Positions + │ ├─ Symbol | Qty | Entry | Current | P&L | P&L% | Sparkline + │ └─ [Exit] button (closes position at market) + │ (refreshed by /api/positions endpoint) + │ + ├─ Activity Feed (chronological) + │ └─ {symbol} {event_type} {timestamp} + │ (refreshed by /api/activity-feed or SSE) + │ + └─ Symbol Details (expand on click) + ├─ Gap: ✓ Pass | 3.2% + ├─ RVOL: ✗ Fail | 1.2 (need 1.5) + ├─ Bull Flag: ? Not Evaluated + └─ ... (9 total criteria) + (fetched by /api/symbol/:symbol/criteria) +``` + +--- + +## Key Data Structure Flows + +### Signal Ready → Order Submitted → Position Open → Fill +``` +SignalReadyEvent(symbol, entry, stop, target, score, criteria) + ↓ [user approves in dashboard] + ↓ OrderApprovedEvent + ↓ [execution service submits] + ↓ OrderSubmittedEvent(broker_order_id, entry, stop, target, qty) + ↓ [Alpaca processes bracket order] + ↓ [entry order fills] + ↓ OrderFilledEvent(entry_fill, qty) + ↓ [position open, waiting for target or stop] + ↓ [target or stop hit] + ↓ OrderFilledEvent(exit_fill, qty, pnl) +``` + +### Symbol State Transitions with Events +``` +Symbol Discovered (new symbol added) + ↓ emit SymbolDiscoveredEvent + ↓ state = "discovered" + ↓ +Symbol State Changed + ↓ emit SymbolStateChangedEvent("discovered" → "watching") + ↓ state = "watching" + ↓ +Criteria Evaluated + ├─ IF score ≥ threshold: + │ ├─ emit SignalReadyEvent + │ ├─ emit SymbolStateChangedEvent("watching" → "ready") + │ ├─ state = "ready" + │ └─ signal_debounced = true (no more signal_ready until blocked) + │ + └─ ELSE: + ├─ emit SignalBlockedEvent + ├─ emit SymbolStateChangedEvent("watching" → "blocked") + ├─ state = "blocked" + └─ signal_debounced = false (can signal again if recovers) +``` + +### Position Lifecycle with P&L Calculation +``` +Position Opens (OrderFilledEvent: entry) + ├─ entry_price, qty, entry_time + ├─ state = "open" + └─ emit AccountPositionsUpdatedEvent({symbol, qty, entry, current, unrealized_pl}) + +Position Ongoing + ├─ AccountSummaryUpdatedEvent polls current price + ├─ unrealized_pl = (current_price - entry_price) * qty + └─ Dashboard shows live sparkline + unrealized P&L + +Position Closes (OrderFilledEvent: exit) + ├─ exit_price, qty, exit_time + ├─ realized_pnl = (exit_price - entry_price) * qty + ├─ state = "closed" + ├─ emit OrderFilledEvent({symbol, entry, exit, qty, pnl, pnl%, exit_reason}) + └─ Dashboard adds to closed trades, updates session P&L + +Session P&L Aggregation + ├─ sum all closed_trades.realized_pnl + ├─ sum all open_positions.unrealized_pl + ├─ win_count = trades where pnl > 0 + ├─ loss_count = trades where pnl ≤ 0 + ├─ win_rate = win_count / (win_count + loss_count) + ├─ avg_r = average(pnl / risk_per_trade) + └─ emit SessionPnlUpdatedEvent({realized, unrealized, total, win_rate, avg_r, counts}) +``` + +--- + +## Summary: The Loop + +Every **tick** (60 seconds): + +1. **Ingest** → Market data into research DB +2. **Discover** → Get watchlist symbols +3. **Evaluate** → Run strategy engine on each symbol +4. **Signal** → Emit signal_ready or signal_blocked +5. **Execute** → Check approval queue, submit approved orders +6. **Sync** → Poll Alpaca for account/positions/orders +7. **Project** → Rebuild all read-only views from event store +8. **Deliver** → Push updates to dashboard (SSE or polling) + +All transformations are **event-based** and **append-only**. The event store is the single source of truth. The dashboard is a **pure projection** of that store, so any session can be replayed perfectly and audited. diff --git a/INSPECTION_ROADMAP.md b/INSPECTION_ROADMAP.md new file mode 100644 index 0000000..e032680 --- /dev/null +++ b/INSPECTION_ROADMAP.md @@ -0,0 +1,377 @@ +# Inspection & Debugging Roadmap + +## What You'll Be Able to See (When All Phases Complete) + +Right now, when you run `python run_live_paper.py`, you see: +- ❌ No indication bars are being collected +- ❌ No visibility into research data +- ❌ Symbols appear/disappear silently +- ❌ No way to test individual modules +- ❌ Can't verify calculation accuracy + +**After Phases 1-4, you'll have:** + +--- + +## 1. Real-Time Module Dashboard (Port 8011) + +Open `http://127.0.0.1:8011` to see live module execution: + +### Data Pipeline Diagram (Auto-Generated) + +``` +┌──────────────┐ ┌──────────────┐ ┌────────────────┐ ┌───────────────┐ +│ INGESTION │───→│ RESEARCH │───→│ EVALUATION │───→│ EXECUTION │ +├──────────────┤ ├──────────────┤ ├────────────────┤ ├───────────────┤ +│ 500 rows in │ │ 5 researched │ │ 5 evaluated │ │ 2 approved │ +│ 245ms │ │ 156ms │ │ 1823ms │ │ 89ms │ +│ ✓ validated │ │ ✓ all found │ │ 2 ready, 3 blocked │ 0 rejected │ +└──────────────┘ └──────────────┘ └────────────────┘ └───────────────┘ +``` + +Each box shows: +- **Input count** → "500 rows in" +- **Latency** → "245ms" +- **Status** → "✓ validated" +- **Click to expand** → see detailed metrics + +### Module Performance Grid + +``` +┌───────────┬──────────────┬──────────┬────────────────┬─────────────────┐ +│ Module │ Last Run (s) │ Avg (ms) │ Error Rate │ Input → Output │ +├───────────┼──────────────┼──────────┼────────────────┼─────────────────┤ +│ Ingestion │ 2.3s ago │ 245 │ 0.0% │ 500 → 500 │ +│ Research │ 2.1s ago │ 156 │ 0.0% │ 5 → 5 │ +│ Evaluation│ 0.8s ago │ 1823 │ 0.0% │ 5 → 2 │ +│ Execution │ 0.5s ago │ 89 │ 0.0% │ 2 → 2 │ +│ Broker │ 0.3s ago │ 342 │ 0.0% │ 2 → 2 │ +└───────────┴──────────────┴──────────┴────────────────┴─────────────────┘ +``` + +Click on each row to: +- See last 10 executions (histogram of latencies) +- View error log if failed +- Inspect metrics + +--- + +## 2. Symbol Inspector (Search Box) + +Type a symbol name to see its complete evaluation journey: + +``` + 🔍 Search symbol... [AAPL] +──────────────────────────────────────────────────────────────── + +AAPL (state: blocked, score: 48%) +├─ INGESTION ✓ +│ ├─ bars collected: 12 +│ ├─ validation: passed ✓ +│ ├─ quality_score: 0.82 +│ └─ collected_at: 09:35:12 +│ +├─ RESEARCH ✓ +│ ├─ gaps: 2.1% +│ ├─ rvol: 1.2x ✗ (need 1.5x) +│ ├─ daily avg volume: 500k +│ ├─ previous close: $10.50 +│ └─ researched_at: 09:35:13 +│ +├─ EVALUATION ✓ bars_loaded +│ ├─ gap_checked ✓ (2.1%) +│ ├─ rvol_calculated ✗ (1.2x < 1.5x) ← BLOCKING REASON +│ ├─ bull_flag ✓ (confidence 0.75) +│ ├─ 1st_candle ✗ (weak) +│ ├─ momentum ✓ +│ ├─ quality: 0.62 +│ ├─ score: 48% (threshold: 60%) +│ └─ evaluated_at: 09:35:15 +│ +├─ EXECUTION (blocked - not evaluated) +│ └─ blocked_reason: "criteria score 48% < 60%" +│ +└─ BROKER (N/A - not submitted) +``` + +Click on each section to see: +- Exact metric values +- Why it passed/failed +- Timestamp of calculation +- Raw data (for debugging) + +### Sparkline Chart (OHLCV) + +``` +AAPL: 09:30 → 09:35 (12 bars) + + 11.00│ ╭─╮ + 10.80│ ╭─╮ ╭─╮╭─╯ ╰─╮ + 10.60│ ╭─╮ ╭─╮ ╭─╯╰─╮ + 10.40│╭─╯ ╰─╮ + 10.20│╯ + └──────────────────────────── + Volume: avg=125k, last=85k + + Entry: $10.52 ━━━━━━━━━━━━━━ + Stop: $10.20 ━━━━━━━━━━━━━━ +``` + +Interactive: +- Hover over bars to see OHLCV +- Click to zoom in/out +- See entry/stop/target reference lines + +--- + +## 3. Data Quality Report + +``` +📊 DATA QUALITY SNAPSHOT +──────────────────────────────────────────────────────────────── + +Ingestion Validation +├─ Rows validated: 500 ✓ +├─ Schema failures: 0 ✓ +├─ Checksum failures: 0 ✓ +├─ Business rules failures: 0 ✓ +└─ Warnings: 2 (zero-volume bars) + +Research Data +├─ Market data freshness: 23 seconds old ✓ +├─ Daily history freshness: 1 day old ✓ +├─ Symbols missing data: 0 ✓ +└─ DB integrity check: PASSED ✓ + +Evaluation Quality +├─ Criteria evaluated: 5 of 5 ✓ +├─ Blocking reasons captured: 3 ✓ +├─ Score distribution: [48%, 52%, 58%, 64%] (avg 55%) +└─ Scores < 60%: 3 symbols + +Broker Connectivity +├─ API health: healthy ✓ +├─ Last account sync: 2s ago ✓ +├─ Last order: 45s ago ✓ +└─ Latency: 342ms avg ✓ +``` + +--- + +## 4. Event Timeline (Live Stream) + +``` +╔════════════════════════════════════════════════════════════════╗ +║ EVENT TIMELINE (newest first, filter by module/symbol/type) ║ +╚════════════════════════════════════════════════════════════════╝ + +09:35:15.347 [evaluation] ✓ completed duration 1823ms, 5 evaluated, 2 ready +09:35:14.512 [evaluation] AAPL: rvol_calculated (1.2x < 1.5x) ✗ +09:35:14.381 [evaluation] AAPL: gap_checked (2.1%) ✓ +09:35:13.856 [evaluation] TSLA: bull_flag_detected (conf 0.85) ✓ +09:35:13.245 [data_validation] ingestion: schema validation PASSED (500 rows) +09:35:13.122 [research] ✓ completed duration 156ms, 5 researched +09:35:12.856 [ingestion] ✓ completed duration 245ms, 500 rows inserted +09:35:12.123 [ingestion] ↦ started (source: alpaca, 500 bars) +09:35:10.234 [execution] AAPL: blocked (score 48% < 60%) +09:35:09.856 [approval_queue] → empty (all processed) +09:35:09.453 [broker] ✓ positions synced 3 open positions +09:35:09.123 [broker] ✓ account updated equity $99,234 +``` + +Click on any event to see: +- Full event JSON +- Correlation ID (trace all events for one signal) +- Copy to clipboard + +--- + +## 5. Dry-Run Mode (Phase 7) + +```bash +# Test with synthetic bull-flag setups (no API calls, no broker) +$ python run_live_paper.py --dry-run \ + --symbols AAPL,TSLA,AMD \ + --bars-provider synthetic:bull_flag \ + --num-ticks 5 \ + --inspection-dashboard + +Starting dry-run mode (no API calls, no broker) +Using synthetic bull-flag bars: 50 bars/symbol +Inspection dashboard: http://127.0.0.1:8011 + +Tick 1 (09:30-09:35) + Ingestion: ✓ 150 bars collected + Research: ✓ 3 symbols researched + Evaluation: ✓ 3 evaluated → 2 ready (AAPL, TSLA) + Execution: ✓ 2 approved + Broker: ✓ 2 orders submitted (simulated) + +Tick 2 (09:35-09:40) + Ingestion: ✓ 150 bars collected + Research: ✓ 3 symbols researched + Evaluation: ✓ 3 evaluated → 1 ready (AMD) + Execution: ✓ 1 approved + Broker: ✓ 1 fill (AAPL +$250), 2 pending + +... (Dashboard shows all metrics live) +``` + +--- + +## 6. Module-Level Testing (Phase 6) + +```bash +# Test just the evaluation module with deterministic data +$ pytest tests/modules/test_evaluation_module.py -v + +tests/modules/test_evaluation_module.py::test_evaluation_ready ✓ + - Bull-flag bars pass evaluation + - Score: 85% + - Entry: $150.50 + - Stop: $149.00 + +tests/modules/test_evaluation_module.py::test_evaluation_blocked ✓ + - Fading bars fail evaluation + - Score: 35% + - Blocking: low volume + - Reason: RVOL 0.8x < 1.5x + +tests/modules/test_evaluation_module.py::test_evaluation_dry_run ✓ + - Can run without research DB + - Works with injected bars + - Full metrics returned + +tests/modules/test_evaluation_module.py::test_evaluation_validation ✓ + - Invalid bars rejected + - Validation failures captured + - Error messages clear + +tests/modules/test_evaluation_module.py::test_evaluation_contract ✓ + - Accepts EvaluationRequest + - Returns EvaluationResult + - All fields populated + +5 passed in 0.23s +``` + +--- + +## 7. Rich TUI Mode (Phase 8, Optional) + +```bash +# Real-time terminal UI (useful for SSH, CI/CD logs) +$ python run_live_paper.py --tui + +╔═══════════════════════════════════════════════════════════════╗ +║ MOMENTUM TRADING SYSTEM - TUI DASHBOARD ║ +╚═══════════════════════════════════════════════════════════════╝ + +[Ingestion] ████████████░░░░░░░░░░░░ 50% (250/500 bars) +[Research] ████████████████████░░░░ 85% (4/5 symbols) +[Evaluation] ████████████████████░░░░ 85% (4/5 evaluated) +[Execution] ████████████░░░░░░░░░░░░ 50% (1/2 approved) +[Broker] ███░░░░░░░░░░░░░░░░░░░░░ 15% (1/5 submitted) + +┌─ Module Latencies ──────────────────────────────┐ +│ Ingestion: 245ms │ +│ Research: 156ms │ +│ Evaluation: 1823ms ← slow (5 symbols) │ +│ Execution: 89ms │ +│ Broker: 342ms │ +└─────────────────────────────────────────────────┘ + +┌─ Recent Events ─────────────────────────────────┐ +│ [09:35:15] evaluation ✓ TSLA ready │ +│ [09:35:14] evaluation ✗ AAPL blocked (rvol) │ +│ [09:35:13] research ✓ 5 symbols researched │ +│ [09:35:12] ingestion ✓ 500 bars collected │ +└─────────────────────────────────────────────────┘ + +[press 'q' to quit, 's' to toggle details, 'c' to copy logs] +``` + +--- + +## Phase Completion Checklist + +| Phase | What Gets Built | When You Can Use It | +|-------|-----------------|-------------------| +| ✅ 1 | Core contracts + telemetry | After Phase 1 | +| 📍 2 | Module instrumentation | After Phase 2 | +| 🔲 3 | Inspection API endpoints | After Phase 3 | +| 🔲 4 | Web dashboard frontend | After Phase 4 | +| 🔲 5 | Module refactoring | After Phase 5 | +| 🔲 6 | Module tests + fixtures | After Phase 6 | +| 🔲 7 | `--dry-run` mode + synthetic data | After Phase 7 | +| 🔲 8 | Rich TUI (optional) | After Phase 8 | + +--- + +## Usage Scenarios + +### "I want to debug why AAPL isn't signaling" + +**With inspection dashboard:** +1. Open http://127.0.0.1:8011 +2. Search "AAPL" in symbol inspector +3. Expand each stage to see: + - Bars collected: 12 ✓ + - Gap: 2.1% ✓ + - RVOL: 1.2x ✗ (blocked here!) +4. Click "RVOL" stage to see: + - Current volume: 85k + - 20-day average: 500k + - Ratio: 0.17x (need 1.5x) +5. Understand exactly why it's blocked + +**Time saved:** 10 seconds vs. 30 minutes digging through logs + +--- + +### "I want to test the evaluation module alone" + +**With Phase 6:** +```bash +pytest tests/modules/test_evaluation_module.py -v +``` + +- No database needed +- No API calls +- Deterministic test data (bull flags, fading bars) +- See exactly what each criterion does +- Change thresholds and re-test instantly + +--- + +### "I want to see if my strategy works with dry-run data" + +**With Phase 7:** +```bash +python run_live_paper.py --dry-run \ + --bars-provider synthetic:bull_flag \ + --num-ticks 5 +``` + +- No API keys needed +- No broker interaction +- Dashboard still works +- See signals, approval, execution all flow through +- Test configuration changes risk-free + +--- + +## Summary + +**Before Phases 1-4:** Black box ("did it work?") + +**After Phases 1-4:** Transparent pipeline ("here's exactly what happened at each step, why it passed/failed, and how long it took") + +This is the foundation for: +- ✅ Debugging blocked signals in seconds +- ✅ Testing modules independently +- ✅ Verifying data accuracy at each stage +- ✅ Dry-running strategies without real APIs +- ✅ Understanding your system's behavior + +Next: **Phase 2** instruments the existing modules. Then Phases 3-4 build the visual dashboard to make all this data human-readable. diff --git a/PHASE1_IMPLEMENTATION.md b/PHASE1_IMPLEMENTATION.md new file mode 100644 index 0000000..ee40703 --- /dev/null +++ b/PHASE1_IMPLEMENTATION.md @@ -0,0 +1,503 @@ +# Phase 1: Module Contracts & Telemetry Infrastructure + +## What Was Built + +Phase 1 establishes the **foundation** for modular, observable trading system. Instead of black-box components, every module now has: + +1. **Explicit I/O contracts** — standardized input/output types +2. **Telemetry collection** — metrics, timing, error tracking +3. **Data validation** — schema checks, business rule validation, checksums +4. **Event types** — new event schema for module observability + +This makes the system **transparent**: you can see exactly what data is flowing where, how long each step takes, and what validation passed/failed. + +--- + +## New Modules + +### `core/contracts.py` — Module I/O Contracts + +Defines TypeScript-like contracts for every module: + +```python +# Every module accepts typed input and produces typed output + +BarIngestRequest ResearchRequest EvaluationRequest + ↓ ↓ ↓ +BarIngestResult ResearchResult EvaluationResult +``` + +**Key classes:** + +- `ModuleMetrics` — Standard metrics every module tracks + - `duration_ms`: How long the module took + - `input_count`, `output_count`: Items processed + - `error_count`, `warnings`: Quality metrics + - `custom_metrics`: Module-specific (gap%, rvol, etc.) + +- `ModuleResult` — Base class for all module outputs + - `success`: Did it work? + - `validation_failures`: What went wrong? + - `error_summary()`: Human-readable error message + +- **Five module contracts** (ingestion, research, evaluation, execution, broker) + - Each has `XyzRequest` (input) and `XyzResult` (output) + - Strongly typed, IDE-friendly + - Makes contracts explicit + +**Usage:** + +```python +from core.contracts import EvaluationRequest, EvaluationResult + +# Module accepts strongly-typed input +request = EvaluationRequest( + symbol="AAPL", + bars_df=my_dataframe, + previous_close=150.0, + avg_daily_volume=1_000_000, + session_date=date(2026, 6, 15) +) + +# Module produces strongly-typed output +result: EvaluationResult = evaluate_setup(request) + +# Safe to access fields — IDE autocomplete works +if result.status == "ready": + print(f"Entry: ${result.entry_price}") + for criterion in result.criteria: + print(f" {criterion.name}: {'✓' if criterion.passed else '✗'}") +``` + +--- + +### `core/telemetry.py` — Metric Collection & Emission + +Real-time telemetry collection and event emission. + +**Key classes:** + +- `TelemetryCollector` — Collect metrics during module execution + ```python + collector = TelemetryCollector("evaluation") + + # Record what you're processing + collector.record_input(count=5) # evaluated 5 symbols + + # Record custom metrics + collector.record_metric("gap_pct", 2.5) + collector.record_metric("rvol", 1.8) + + # Finalize when done + metrics = collector.finalize() + # → ModuleMetrics with duration_ms, counts, custom metrics + ``` + +- `TelemetryEmitter` — Emit events to event store + ```python + emitter = get_telemetry_emitter() + + # Emit module execution telemetry + emitter.emit_module_tick( + module_name="evaluation", + stage="completed", + metrics=metrics, + correlation_id="session-123" + ) + + # Emit data validation results + emitter.emit_data_validation( + module_name="ingestion", + validation_type="schema", + valid=True, + details={ + "rows_validated": 500, + "failures": [] + } + ) + + # Emit detailed symbol evaluation metrics + emitter.emit_symbol_evaluation_detail( + symbol="AAPL", + stage="gap_checked", + metrics={"gap_pct": 2.5, "threshold": 2.0}, + passed=True + ) + ``` + +- **Callbacks** — Emit events to multiple destinations + ```python + emitter = get_telemetry_emitter() + + # Add callback to receive all telemetry events + emitter.add_callback(lambda event: logger.info(f"Event: {event['type']}")) + + # When you emit, all callbacks are called + emitter.emit_module_tick(...) + ``` + +--- + +### `core/validation.py` — Data Validation + +Multi-layer validation at module boundaries. + +**Validators:** + +1. **SchemaValidator** — Check required/optional fields and types + ```python + validator = SchemaValidator( + required_fields={"symbol": str, "qty": int, "price": float}, + optional_fields={"notes": str} + ) + + is_valid, failures = validator.validate({"symbol": "AAPL", "qty": 100, "price": 150.0}) + # failures = [] (if valid) + # or failures = [{"field": "qty", "reason": "Expected int, got str", ...}] + ``` + +2. **RangeValidator** — Numeric bounds checking + ```python + rvol_validator = RangeValidator(min_val=1.0, max_val=10.0) + is_valid, failure = rvol_validator.validate(1.5, field_name="rvol") + ``` + +3. **ChecksumValidator** — Data integrity verification + ```python + data = {"bars": [...]100 items...} + checksum = ChecksumValidator.compute_checksum(data) + # checksum = "a1b2c3d4e5f6..." (MD5) + + # Later, verify data hasn't changed + is_valid, failure = ChecksumValidator.validate_checksum(data, checksum) + ``` + +4. **BusinessRuleValidator** — Custom logic validation + ```python + validator = BusinessRuleValidator() + + # Add custom rules + validator.add_rule( + "min_high_price", + lambda data: (data["high"] > data["low"], "high must be > low") + ) + + is_valid, failures = validator.validate(bar_data) + ``` + +5. **BarDataValidator** — Pre-built OHLCV validation + ```python + is_valid, failures = BarDataValidator.validate_bar({ + "timestamp": "2026-06-15 09:31:00", + "open": 150.0, + "high": 151.0, + "low": 149.5, + "close": 150.5, + "volume": 50000 + }) + # Checks: + # - All required fields present + # - High ≥ open, close, low + # - Low ≤ open, close, high + # - Volume non-negative + # - Warns on zero volume + ``` + +--- + +## New Event Types + +Added to `storage/event_schema.py`: + +### 1. `ModuleTickEvent` — Module Execution Telemetry + +Emitted by each module at the end of each tick. + +```python +@dataclass +class ModuleTickEvent(BaseEvent): + module: str # "ingestion", "research", "evaluation", etc. + stage: str # "started", "processing", "completed", "failed" + duration_ms: float # How long it took + input_count: int # Items processed + output_count: int # Items produced + metrics: dict # Custom module metrics + errors: list[dict] # Any errors that occurred +``` + +**Example:** +```json +{ + "type": "module_tick", + "module": "evaluation", + "stage": "completed", + "duration_ms": 245.5, + "input_count": 5, + "output_count": 2, + "metrics": { + "avg_score": 58.3, + "ready_count": 2, + "blocked_count": 3 + }, + "errors": [], + "timestamp": "2026-06-15T09:35:00" +} +``` + +### 2. `DataValidationEvent` — Data Quality Checkpoint + +Emitted when module validates data at boundaries. + +```python +@dataclass +class DataValidationEvent(BaseEvent): + module: str # Module that validated + validation_type: str # "schema", "checksum", "business_rules" + valid: bool # Did it pass? + details: dict # Validation specifics +``` + +**Example:** +```json +{ + "type": "data_validation", + "module": "ingestion", + "validation_type": "schema", + "valid": true, + "details": { + "rows_validated": 500, + "rows_failed": 0, + "failures": [] + } +} +``` + +### 3. `SymbolEvaluationDetailEvent` — Deep-Dive Symbol Metrics + +Emitted during symbol evaluation to show intermediate calculations. + +```python +@dataclass +class SymbolEvaluationDetailEvent(BaseEvent): + symbol: str # Symbol being evaluated + stage: str # "bars_loaded", "gap_checked", "rvol_calculated", etc. + metrics: dict # Metrics at this stage + passed: bool # Did this stage pass? + blocking_reason: str # Why it failed (if applicable) +``` + +**Example:** +```json +{ + "type": "symbol_evaluation_detail", + "symbol": "AAPL", + "stage": "gap_checked", + "metrics": { + "gap_pct": 2.5, + "threshold": 2.0 + }, + "passed": true, + "blocking_reason": null, + "timestamp": "2026-06-15T09:35:12" +} +``` + +```json +{ + "type": "symbol_evaluation_detail", + "symbol": "AAPL", + "stage": "rvol_calculated", + "metrics": { + "rvol": 1.2, + "threshold": 1.5, + "current_5min_volume": 125000, + "avg_20day_volume": 1000000 + }, + "passed": false, + "blocking_reason": "RVOL 1.2x below threshold 1.5x", + "timestamp": "2026-06-15T09:35:14" +} +``` + +--- + +## Usage Pattern: Instrumenting an Existing Module + +Here's how to add telemetry to any module (e.g., evaluation): + +```python +from core.telemetry import TelemetryCollector, emit_module_tick, emit_symbol_evaluation_detail +from core.validation import BarDataValidator + +def evaluate_setup(bars_df, symbol, previous_close, avg_daily_volume): + """Evaluate setup with full telemetry and validation.""" + + collector = TelemetryCollector("evaluation") + + try: + # Validate input + bars = bars_df.to_dict('records') + for i, bar in enumerate(bars): + is_valid, failures = BarDataValidator.validate_bar(bar) + if not is_valid: + collector.record_error() + emit_symbol_evaluation_detail( + symbol=symbol, + stage="bars_validated", + metrics={"valid_bars": i, "total_bars": len(bars)}, + passed=False, + blocking_reason=f"Bar {i} invalid: {failures[0]['reason']}" + ) + return EvaluationResult(success=False, metrics=collector.finalize()) + + collector.record_input(len(bars)) + + # Emit validation checkpoint + emit_data_validation( + module_name="evaluation", + validation_type="schema", + valid=True, + details={"rows_validated": len(bars), "failures": []} + ) + + # Stage 1: Gap check + gap_pct = ((bars[-1]['open'] - previous_close) / previous_close) * 100 + emit_symbol_evaluation_detail( + symbol=symbol, + stage="gap_checked", + metrics={"gap_pct": gap_pct, "threshold": 2.0}, + passed=gap_pct >= 2.0 + ) + collector.record_metric("gap_pct", gap_pct) + + # Stage 2: RVOL check + current_5min_vol = bars[-1]['volume'] + rvol = current_5min_vol / avg_daily_volume + rvol_passed = rvol >= 1.5 + emit_symbol_evaluation_detail( + symbol=symbol, + stage="rvol_calculated", + metrics={ + "rvol": rvol, + "threshold": 1.5, + "current_5min_vol": current_5min_vol, + "avg_20day_vol": avg_daily_volume + }, + passed=rvol_passed, + blocking_reason=None if rvol_passed else f"RVOL {rvol:.2f}x below 1.5x" + ) + collector.record_metric("rvol", rvol) + + # ... more stages ... + + # Finalize + collector.record_output(1) + metrics = collector.finalize() + + # Emit module completion + emit_module_tick( + module_name="evaluation", + stage="completed", + metrics=metrics + ) + + return EvaluationResult( + success=True, + metrics=metrics, + symbol=symbol, + status="ready", + score=score + ) + + except Exception as e: + collector.record_error() + metrics = collector.finalize() + emit_module_tick( + module_name="evaluation", + stage="failed", + metrics=metrics, + errors=[{"message": str(e)}] + ) + return EvaluationResult(success=False, metrics=metrics) +``` + +--- + +## Why This Matters + +### Before (Black Box) +``` +Bars ingested → [silence] → Signal ready + ❓ Were bars actually valid? + ❓ Which criterion failed? + ❓ How long did it take? + ❓ What was the gap%? +``` + +### After (Transparent) +``` +ModuleTickEvent: + module: "ingestion" + input_count: 500 + output_count: 500 + metrics: { + "rows_inserted": 500, + "validation_failures": 0 + } + +DataValidationEvent: + module: "ingestion" + validation_type: "schema" + valid: true + +SymbolEvaluationDetailEvent (AAPL): + stage: "gap_checked" + metrics: {"gap_pct": 2.5} + passed: true + +SymbolEvaluationDetailEvent (AAPL): + stage: "rvol_calculated" + metrics: {"rvol": 1.2} + passed: false + blocking_reason: "RVOL 1.2x below 1.5x" +``` + +--- + +## Next Steps + +**Phase 2** will instrument the existing modules (ingestion, research, evaluation, execution, broker) to emit these events. + +**Phase 3** will build the Inspection Dashboard API to query and visualize these events. + +**Phase 4** will build the React frontend to make the data human-readable. + +**Phase 7** will add `--dry-run` mode so you can test with synthetic data. + +--- + +## Files Created + +``` +core/ +├── __init__.py # Empty init +├── contracts.py # Module I/O contracts (5 modules) +├── telemetry.py # Metric collection & emission +└── validation.py # Multi-layer validation + +storage/ +└── event_schema.py # (MODIFIED) Added 3 new event types +``` + +--- + +## Key Takeaway + +The system is no longer a black box. Every module now: + +1. ✅ Has **explicit contracts** (input/output types) +2. ✅ **Emits telemetry** (duration, counts, metrics) +3. ✅ **Validates data** at module boundaries +4. ✅ **Reports errors** with context + +This foundation enables the inspection dashboard, dry-run mode, and per-module testing to all work correctly. diff --git a/PHASE2_PREVIEW.md b/PHASE2_PREVIEW.md new file mode 100644 index 0000000..ee2a666 --- /dev/null +++ b/PHASE2_PREVIEW.md @@ -0,0 +1,487 @@ +# Phase 2 Preview: What You'll Be Coding + +This preview shows **exactly** what Phase 2 implementation looks like, so you can see the work upfront. + +--- + +## Example 1: Instrumenting `ingest_live_minute_bars()` + +### Before (Phase 1) +```python +def ingest_live_minute_bars(alpaca_client, research_db, symbols): + """Ingest latest 1-minute bars from Alpaca.""" + for symbol in symbols: + bars = alpaca_client.get_bars(symbol, lookback_minutes=120) + # Insert into DB silently... + research_db.insert_bars(symbol, bars) + # No telemetry, no validation, silent success/failure +``` + +### After (Phase 2) +```python +from core.telemetry import TelemetryCollector, emit_module_tick, emit_data_validation +from core.validation import BarDataValidator + +def ingest_live_minute_bars(alpaca_client, research_db, symbols): + """Ingest latest 1-minute bars from Alpaca with full telemetry.""" + + collector = TelemetryCollector("ingestion") + validation_failures = [] + + try: + for symbol in symbols: + try: + bars = alpaca_client.get_bars(symbol, lookback_minutes=120) + collector.record_input(len(bars)) + + # Validate each bar before insertion + for i, bar in enumerate(bars): + is_valid, failures = BarDataValidator.validate_bar(bar) + if not is_valid: + collector.record_error() + validation_failures.extend(failures) + + # Insert valid bars + if not validation_failures: + research_db.insert_bars(symbol, bars) + collector.record_output(len(bars)) + collector.record_metric(f"{symbol}_volume_avg", + sum(b['volume'] for b in bars) / len(bars)) + + except Exception as e: + collector.record_error() + collector.record_warning(f"{symbol} failed: {e}") + + # Emit validation checkpoint + emit_data_validation( + module_name="ingestion", + validation_type="schema", + valid=len([f for f in validation_failures if f['severity'] == 'error']) == 0, + details={ + "rows_validated": collector.input_count, + "validation_failures": validation_failures, + "symbols_processed": len(symbols) + } + ) + + # Emit completion telemetry + metrics = collector.finalize() + emit_module_tick( + module_name="ingestion", + stage="completed", + metrics=metrics + ) + + except Exception as e: + collector.record_error() + metrics = collector.finalize() + emit_module_tick( + module_name="ingestion", + stage="failed", + metrics=metrics, + errors=[{"message": str(e)}] + ) + raise +``` + +**What changed:** +- ✅ Wrapped with `TelemetryCollector` +- ✅ Validate bars before insertion +- ✅ Record input/output counts +- ✅ Track custom metrics (volume averages) +- ✅ Emit validation + completion events +- ✅ Error handling with telemetry + +**You'll see in event store:** +```json +{ + "type": "module_tick", + "module": "ingestion", + "stage": "completed", + "duration_ms": 245, + "input_count": 500, + "output_count": 500, + "metrics": { + "AAPL_volume_avg": 125000, + "TSLA_volume_avg": 98000, + "AMD_volume_avg": 75000 + }, + "errors": [] +} +``` + +--- + +## Example 2: Instrumenting `evaluate_setup()` + +### Before (Phase 1) +```python +def evaluate_setup(bars, symbol, previous_close, avg_daily_volume): + """Evaluate setup criteria.""" + + gap_pct = ((bars[-1]['open'] - previous_close) / previous_close) * 100 + rvol = bars[-1]['volume'] / avg_daily_volume + + # ... evaluate 7 more criteria silently ... + + criteria_passed = sum([gap_pct >= 2.0, rvol >= 1.5, ...]) + score = (criteria_passed / 9) * 100 + + return { + 'status': 'ready' if score >= 60 else 'blocked', + 'score': score, + # ... hidden metrics ... + } + # User never sees WHY it passed/failed +``` + +### After (Phase 2) +```python +from core.telemetry import ( + TelemetryCollector, + emit_module_tick, + emit_symbol_evaluation_detail +) +from core.validation import RangeValidator + +def evaluate_setup(bars, symbol, previous_close, avg_daily_volume): + """Evaluate setup criteria with full telemetry.""" + + collector = TelemetryCollector("evaluation") + criteria_results = [] + + try: + collector.record_input(1) # 1 symbol evaluated + + # STAGE 1: Gap check + gap_pct = ((bars[-1]['open'] - previous_close) / previous_close) * 100 + gap_passed = gap_pct >= 2.0 + + emit_symbol_evaluation_detail( + symbol=symbol, + stage="gap_checked", + metrics={"gap_pct": gap_pct, "threshold": 2.0}, + passed=gap_passed, + blocking_reason=None if gap_passed else f"Gap {gap_pct:.1f}% < 2.0%" + ) + collector.record_metric("gap_pct", gap_pct) + criteria_results.append(("gap", gap_passed, gap_pct)) + + # STAGE 2: RVOL check + current_5min_vol = bars[-1]['volume'] + rvol = current_5min_vol / avg_daily_volume + rvol_passed = rvol >= 1.5 + + emit_symbol_evaluation_detail( + symbol=symbol, + stage="rvol_calculated", + metrics={ + "rvol": rvol, + "threshold": 1.5, + "current_5min_vol": current_5min_vol, + "avg_20day_vol": avg_daily_volume + }, + passed=rvol_passed, + blocking_reason=None if rvol_passed else f"RVOL {rvol:.2f}x < 1.5x" + ) + collector.record_metric("rvol", rvol) + criteria_results.append(("rvol", rvol_passed, rvol)) + + # STAGE 3: Bull flag detection + bull_flag_score, bull_flag_details = detect_bull_flag(bars) + bull_flag_passed = bull_flag_score >= 0.5 + + emit_symbol_evaluation_detail( + symbol=symbol, + stage="bull_flag_detected", + metrics={ + "confidence": bull_flag_score, + "threshold": 0.5, + "details": bull_flag_details + }, + passed=bull_flag_passed, + blocking_reason=None if bull_flag_passed else f"Bull flag confidence {bull_flag_score:.2f} < 0.5" + ) + collector.record_metric("bull_flag_confidence", bull_flag_score) + criteria_results.append(("bull_flag", bull_flag_passed, bull_flag_score)) + + # ... evaluate 6 more criteria, each with emit_symbol_evaluation_detail ... + + # Calculate final score + criteria_passed = sum(1 for _, passed, _ in criteria_results if passed) + total_criteria = len(criteria_results) + score = (criteria_passed / total_criteria) * 100 + quality = bull_flag_score * (rvol / 1.5) # Custom quality metric + + emit_symbol_evaluation_detail( + symbol=symbol, + stage="score_calculated", + metrics={ + "score": score, + "criteria_passed": criteria_passed, + "total_criteria": total_criteria, + "quality": quality, + "threshold": 60 + }, + passed=score >= 60, + blocking_reason=None if score >= 60 else f"Score {score:.1f}% < 60%" + ) + collector.record_metric("score", score) + collector.record_metric("quality", quality) + + # Determine status + status = "ready" if score >= 60 else "blocked" + if status == "ready": + collector.record_output(1) + + # Entry/stop/target calculation + entry_price = bars[-1]['close'] + stop_price = bars.min()['low'] * 0.99 + target_price = entry_price + (entry_price - stop_price) * 2.0 + + emit_symbol_evaluation_detail( + symbol=symbol, + stage="levels_calculated", + metrics={ + "entry": entry_price, + "stop": stop_price, + "target": target_price, + "risk": entry_price - stop_price, + "reward": target_price - entry_price, + "reward_ratio": (target_price - entry_price) / (entry_price - stop_price) + }, + passed=True + ) + + # Finalize telemetry + metrics = collector.finalize() + emit_module_tick( + module_name="evaluation", + stage="completed", + metrics=metrics + ) + + return { + 'status': status, + 'score': score, + 'criteria': [ + {'name': name, 'passed': passed, 'value': value} + for name, passed, value in criteria_results + ], + 'entry': entry_price, + 'stop': stop_price, + 'target': target_price, + 'quality': quality + } + + except Exception as e: + collector.record_error() + metrics = collector.finalize() + emit_module_tick( + module_name="evaluation", + stage="failed", + metrics=metrics, + errors=[{"symbol": symbol, "message": str(e)}] + ) + raise +``` + +**What you'll see in event store:** + +```json +[ + { + "type": "symbol_evaluation_detail", + "symbol": "AAPL", + "stage": "gap_checked", + "metrics": {"gap_pct": 2.1, "threshold": 2.0}, + "passed": true + }, + { + "type": "symbol_evaluation_detail", + "symbol": "AAPL", + "stage": "rvol_calculated", + "metrics": {"rvol": 1.2, "threshold": 1.5, "current_5min_vol": 85000}, + "passed": false, + "blocking_reason": "RVOL 1.20x < 1.5x" + }, + { + "type": "symbol_evaluation_detail", + "symbol": "AAPL", + "stage": "bull_flag_detected", + "metrics": {"confidence": 0.75, "threshold": 0.5}, + "passed": true + }, + { + "type": "symbol_evaluation_detail", + "symbol": "AAPL", + "stage": "score_calculated", + "metrics": {"score": 48.0, "criteria_passed": 4, "total_criteria": 9, "threshold": 60}, + "passed": false, + "blocking_reason": "Score 48.0% < 60%" + }, + { + "type": "module_tick", + "module": "evaluation", + "stage": "completed", + "duration_ms": 1823, + "input_count": 5, + "output_count": 2, + "metrics": { + "avg_score": 58.3, + "ready_count": 2, + "blocked_count": 3 + } + } +] +``` + +--- + +## Example 3: Updating `run_live_paper.py` + +### Small changes needed +```python +from core.telemetry import TelemetryEmitter, set_telemetry_emitter +from storage.event_store import EventStore + +def build_runtime(args: argparse.Namespace) -> dict: + """Wire up every component; safe without API keys.""" + + # ... existing code ... + + event_store = EventStore() + + # NEW: Initialize telemetry emitter + emitter = TelemetryEmitter(event_store=event_store) + set_telemetry_emitter(emitter) # Make it global + + # ... rest of existing code ... + + return { + 'event_store': event_store, + 'emitter': emitter, # NEW + # ... other components ... + } +``` + +That's it! The modules will use the global emitter automatically. + +--- + +## Example 4: Instrumenting `TradingExecutionService` + +```python +def tick(self): + """Execute one tick with telemetry.""" + + collector = TelemetryCollector("execution") + + # Check for new signal_ready events + ready_signals = query_ready_signals_snapshot(self.store) + collector.record_input(len(ready_signals)) + + for signal in ready_signals: + try: + # Risk checks + open_pos = self._open_position_count() + if open_pos >= self.settings.max_concurrent_positions: + emit_symbol_evaluation_detail( + symbol=signal.symbol, + stage="risk_check_max_positions", + metrics={"open_positions": open_pos, "max": self.settings.max_concurrent_positions}, + passed=False, + blocking_reason=f"Max positions {open_pos} >= limit {self.settings.max_concurrent_positions}" + ) + continue + + # Position sizing + shares = calculate_position_size( + entry=signal.entry, + stop=signal.stop, + equity=self.equity, + risk_pct=self.settings.risk_per_trade_pct + ) + + # Emit approval request + self.store.emit(OrderApprovalRequestedEvent(...)) + collector.record_output(1) + + except Exception as e: + collector.record_error() + + metrics = collector.finalize() + emit_module_tick( + module_name="execution", + stage="completed", + metrics=metrics + ) +``` + +--- + +## Time Breakdown for Phase 2 + +| Task | Time | +|------|------| +| Instrument ingestion module | 30 mins | +| Instrument research module | 30 mins | +| Instrument evaluation module | 60 mins (most complex) | +| Instrument execution module | 45 mins | +| Instrument broker sync module | 30 mins | +| Update run_live_paper.py | 15 mins | +| Test & verify | 45 mins | +| **Total** | **~4 hours** | + +--- + +## Verification Checklist for Phase 2 + +After completing Phase 2, you should be able to: + +```bash +# Run the system +python run_live_paper.py --once + +# Check event store has new events +sqlite3 data/momentum_events.duckdb "SELECT COUNT(*), event_type FROM events GROUP BY event_type;" + +# Expected output: +# 5 | module_tick +# 3 | data_validation +# 15 | symbol_evaluation_detail +# ... (plus existing event types) + +# View a symbol's complete journey +sqlite3 data/momentum_events.duckdb \ + "SELECT event_type, symbol, stage, passed FROM events \ + WHERE symbol='AAPL' AND event_type IN ('symbol_evaluation_detail', 'data_validation') \ + ORDER BY timestamp;" + +# Expected output: +# symbol_evaluation_detail | AAPL | gap_checked | 1 +# symbol_evaluation_detail | AAPL | rvol_calculated | 0 +# symbol_evaluation_detail | AAPL | bull_flag_detected | 1 +# ... +``` + +--- + +## Key Points + +1. **Minimal changes to existing code** — mostly wrapping with telemetry +2. **No breaking changes** — all existing functionality preserved +3. **Backward compatible** — code still works without telemetry +4. **Testable** — each instrumented module still works alone +5. **Progressive** — instrument one module at a time, test, then move to next + +--- + +## What Phase 3 Needs + +Once Phase 2 is done, Phase 3 will: +- Query the `symbol_evaluation_detail` events for symbol inspector +- Query `module_tick` events for module performance grid +- Query `data_validation` events for quality report + +Everything needed will be in the event store already! diff --git a/QUICK_REFERENCE.txt b/QUICK_REFERENCE.txt new file mode 100644 index 0000000..662fa5d --- /dev/null +++ b/QUICK_REFERENCE.txt @@ -0,0 +1,268 @@ +================================================================================ + MOMENTUM SYSTEM - QUICK REFERENCE +================================================================================ + +PHASE 1 (✅ COMPLETE): Core Infrastructure +──────────────────────────────────────────────────────────────────────────── + +What was built: + ✓ core/contracts.py Module I/O contracts (5 modules × 2 types each) + ✓ core/telemetry.py Metric collection + event emission + ✓ core/validation.py Data validation (schema, ranges, checksums, rules) + ✓ 3 new event types (ModuleTickEvent, DataValidationEvent, SymbolEvaluationDetailEvent) + +Why it matters: + BEFORE: ingest_live_minute_bars() → [silence] → "did it work?" + AFTER: "500 bars collected, 0 validation failures, completed in 245ms" + +Key classes: + TelemetryCollector Record what's happening during execution + TelemetryEmitter Emit events to event store + callbacks + ModuleMetrics duration_ms, input_count, output_count, custom_metrics + SchemaValidator Check required fields and types + BarDataValidator Pre-built OHLCV validation + +Quick import: + from core.telemetry import TelemetryCollector, emit_module_tick + from core.validation import SchemaValidator, BarDataValidator + +Quick usage: + collector = TelemetryCollector("evaluation") + collector.record_input(5) + collector.record_metric("gap_pct", 2.5) + metrics = collector.finalize() + emit_module_tick("evaluation", "completed", metrics) + +================================================================================ + +PHASE 2 (📍 NEXT): Module Instrumentation (~4 hours) +──────────────────────────────────────────────────────────────────────────── + +What you'll do: + 1. Wrap ingest_live_minute_bars() with TelemetryCollector + 2. Wrap evaluate_setup() with telemetry + SymbolEvaluationDetailEvent + 3. Wrap research, execution, broker modules + 4. Update run_live_paper.py to initialize emitter + 5. Test and verify events appear in event store + +Each module will emit: + ✓ ModuleTickEvent at completion (duration, counts, metrics) + ✓ DataValidationEvent when data validated + ✓ SymbolEvaluationDetailEvent for each evaluation stage (evaluation module only) + +See: PHASE2_PREVIEW.md for exact code examples + +================================================================================ + +PHASE 3 (🔲 TODO): Inspection Dashboard API (~6 hours) +──────────────────────────────────────────────────────────────────────────── + +Build: api/inspection.py with REST endpoints + +Endpoints: + GET /api/inspection/modules module metrics table + GET /api/inspection/pipeline pipeline flow diagram data + GET /api/inspection/symbol/:symbol eval timeline for symbol + GET /api/inspection/data-quality validation report + GET /api/inspection/events SSE stream of events + GET /api/inspection/module/:m/history last N executions + +Projections to build: + query_module_performance() + query_symbol_evaluation_detail(symbol) + query_data_quality_snapshot() + query_pipeline_stats(session_id) + +================================================================================ + +PHASE 4 (🔲 TODO): React Dashboard (~8 hours) +──────────────────────────────────────────────────────────────────────────── + +Build: api/inspection/ directory with React SPA + +Features: + □ Pipeline diagram (Mermaid auto-layout) + □ Module performance grid (5x5 table with metrics) + □ Symbol inspector (search + expandable stages) + □ Sparkline charts (OHLCV with entry/stop lines) + □ Event timeline (live feed with filtering) + □ Data quality report (heatmap + validation stats) + □ Live SSE updates + polling fallback + □ Mobile responsive + +Port: http://127.0.0.1:8011 (separate from execution dashboard :8010) + +================================================================================ + +PHASES 5-8 (OPTIONAL): Refactoring & Testing +──────────────────────────────────────────────────────────────────────────── + +Phase 5: Module refactoring into explicit contracts + modules/ingestion.py, research.py, evaluation.py, execution.py, broker.py + +Phase 6: Module-level testing + tests/modules/test_*.py (one per module) + Fixtures for synthetic data (bull_flag_bars, fading_bars, etc.) + +Phase 7: Dry-run mode + synthetic data + python run_live_paper.py --dry-run --bars-provider synthetic:bull_flag + +Phase 8: Rich TUI (terminal UI) + python run_live_paper.py --tui + (useful for SSH, CI/CD, no browser needed) + +================================================================================ + +CURRENT FILES CREATED +──────────────────────────────────────────────────────────────────────────── + +Core: + core/__init__.py + core/contracts.py 5 module contracts + core/telemetry.py Metric collection + emission + core/validation.py Multi-layer validation + +Extended: + storage/event_schema.py +3 event types + +Documentation: + README_ARCHITECTURE_REDESIGN.md Start here! Big picture + BUILD_SUMMARY.md All 8 phases + timeline + PHASE1_IMPLEMENTATION.md How Phase 1 works + PHASE2_PREVIEW.md Exact Phase 2 code examples + INSPECTION_ROADMAP.md What you'll see after Phase 4 + QUICK_REFERENCE.txt This file + +Old (no longer needed): + ARCHITECTURE_VISUAL.html Old static visualization + SYSTEM_FLOW.txt Old text flow diagram + DATA_TRANSFORMATIONS.md Old transformation doc + +================================================================================ + +TIMELINE +──────────────────────────────────────────────────────────────────────────── + +Week 1: + Phase 1: Core infrastructure ✅ DONE (~2 hours) + Phase 2: Module instrumentation 📍 START (~4 hours) + +Week 2: + Phase 3: Inspection API backend (~6 hours) + Phase 4: React dashboard frontend (~8 hours) + +Week 3: + Phase 5: Module refactoring (~6 hours) + Phase 6: Testing framework (~5 hours) + +Week 4: + Phase 7: Dry-run mode (~4 hours) + Phase 8: TUI (optional) (~3 hours) + +Total: ~4 weeks for full implementation, or ~2 weeks for core (1-4) + +================================================================================ + +SUCCESS METRICS +──────────────────────────────────────────────────────────────────────────── + +After Phase 4, you'll be able to: + + ✓ Open http://127.0.0.1:8011 (inspection dashboard) + ✓ See real-time module pipeline (ingestion → research → eval → exec → broker) + ✓ Search for a symbol → see complete evaluation journey + ✓ Understand why AAPL is blocked (exact stage, metric, threshold) + ✓ View timestamps, metrics, and raw data for each stage + ✓ See live event stream + ✓ Access data quality report + +Time to answer "why isn't AAPL signaling?" → 10 seconds (was 30 minutes) + +================================================================================ + +KEY INSIGHTS +──────────────────────────────────────────────────────────────────────────── + +The System Used to Work Like: + Alpaca API → [silence] → Signal ready → [silence] → Order submitted + ❓ Were bars validated? + ❓ Which criterion failed? + ❓ How long did evaluation take? + ❓ What was the gap%? + +The System Will Work Like: + Alpaca API → ModuleTickEvent(500 rows in, 245ms) → Signal ready + → SymbolEvaluationDetailEvent(gap_checked: 2.1% ✓) + → SymbolEvaluationDetailEvent(rvol_calculated: 1.2x ✗) + → SymbolEvaluationDetailEvent(score: 48% < 60%) + → Order rejected (criteria not met) + + Dashboard shows all of this in real-time + +================================================================================ + +NEXT IMMEDIATE STEPS +──────────────────────────────────────────────────────────────────────────── + +1. Read README_ARCHITECTURE_REDESIGN.md (10 minutes) +2. Read BUILD_SUMMARY.md (20 minutes) +3. Review PHASE1_IMPLEMENTATION.md (15 minutes) +4. Skim PHASE2_PREVIEW.md for code examples (10 minutes) +5. Decide: Start Phase 2 or want clarifications? + +If starting Phase 2: + Use PHASE2_PREVIEW.md as your code template + Follow task list in BUILD_SUMMARY.md + Test after each module (verify events in DB) + Expected time: ~4 hours + +================================================================================ + +QUESTIONS? +──────────────────────────────────────────────────────────────────────────── + +Common questions answered in: + + "How do I use TelemetryCollector?" + → PHASE1_IMPLEMENTATION.md, "Usage Pattern" section + + "What does Phase 2 code look like?" + → PHASE2_PREVIEW.md, Examples 1-4 + + "What will the dashboard show me?" + → INSPECTION_ROADMAP.md, "Real-Time Module Dashboard" section + + "Can I test modules independently?" + → Phase 6, coming soon (BUILD_SUMMARY.md) + + "How do I run without API keys?" + → Phase 7, --dry-run mode (BUILD_SUMMARY.md) + + "Where do I look if ingestion fails?" + → Inspection dashboard, Module Grid, click "ingestion", see error + + "How do I know bars are being collected?" + → ModuleTickEvent shows input_count after Phase 2 + + "How do I verify research data?" + → DataValidationEvent after Phase 2, dashboard after Phase 4 + +================================================================================ + +READY TO START? +──────────────────────────────────────────────────────────────────────────── + +Phase 1 is complete. You have the foundation. + +Phase 2 is ~4 hours of work: + - Wrap 5 modules with TelemetryCollector + - Add validation checkpoints + - Emit completion events + - Test + +After Phase 2, you'll start seeing telemetry events in your event store. +After Phase 4, you'll have a fully functional inspection dashboard. + +Let's go! + +================================================================================ diff --git a/README.md b/README.md new file mode 100644 index 0000000..5ca343b --- /dev/null +++ b/README.md @@ -0,0 +1,116 @@ +# Momentum + +Event-sourced small-cap momentum day-trading system (Ross Cameron–style): +gap-and-go / bull-flag setups on 1-minute bars, with paper execution on +Alpaca and a live-broker integration for Schwab. + +> **Important rebuild note.** This codebase was reconstructed from scratch +> against the original architecture and interfaces after the original source +> files became unrecoverable in the working environment. Diff it against your +> local copy before discarding anything — module layout and public interfaces +> were preserved, but implementations are fresh. See `RUNBOOK.md` for what +> changed and why. + +## What this system does + +``` +Alpaca IEX bars ──► research/market.duckdb ──► Watcher ──► signal_ready + (1-min, live) (minute + daily) │ + ▼ +Dashboard (approve/reject) ◄── approval queue ◄── TradingExecutionService + │ │ + ▼ ▼ + exit orders ─────────────────────────► Alpaca paper account + │ + AlpacaPaperSync ◄─────┘ + (account/positions/orders → event store → dashboard) +``` + +Everything observable flows through a single append-only **event store** +(`data/momentum.duckdb`): symbol discovery, criteria scores, signals, +approval requests, orders, fills, account snapshots, broker health. The +dashboard and all read paths are pure projections over those events, so any +session can be replayed and audited. + +## Milestones + +| # | Scope | Status | +|---|-------|--------| +| M1 | Strategy logic — setup evaluation, structure detection, quality, backtest | ✅ `strategy/` | +| M2 | Event store + projections | ✅ `storage/` | +| M3 | Schwab integration — OAuth, market data, positions/orders, health | ✅ `schwab/` (`python milestone3_verify.py`) | +| M4 | Watcher + live paper execution | ✅ `runtime/`, `alpaca_paper/`, `trading_execution.py`, `run_live_paper.py` | +| M5 | Dashboard UI + API | ✅ `api/`, `dashboard_api.py` | + +## Quick start (paper trading) + +```bash +pip install -r requirements.txt +cp .env.example .env # fill in ALPACA_API_KEY / ALPACA_SECRET_KEY + +python run_live_paper.py # full loop + dashboard at http://127.0.0.1:8010 +``` + +Signals appear in the dashboard's **Approval Queue**; click **Approve** to +send the bracket order to your Alpaca paper account (or start with +`--auto-approve` / `TRADING_AUTO_APPROVE=1`). Use the **Exit** button on a +position to close it at market. + +No keys? `python run_live_paper.py --once` still runs in dry mode so you can +verify wiring. + +### Dashboard (control room) + +The dashboard is a single self-contained page (no build step) that streams +live state over Server-Sent Events and falls back to polling if the stream +drops. It shows: + +- **P&L strip** — realized, unrealized, total, win rate, average R, and + open/closed trade counts for the session. +- **Live sparkline charts** on every ready signal and open position, drawn + from 1-minute bars with dashed entry (blue) and stop (red) reference lines. +- **Pre-trade risk preview** in the approval queue — dollar risk, share count, + notional, and stop distance before you approve. +- **Click-to-expand criteria** on any watched symbol — all nine setup criteria + as pass / fail / not-evaluated, so a "blocked" symbol is legible at a glance. +- **Activity feed** of submissions and fills, plus the full order lifecycle. +- **Audio + toast alerts** when a new signal goes ready (toggle the sound + button, or press `S`). +- **Manual symbol injection** — type a ticker (or press `/`) to force it onto + the watchlist mid-session. +- **Keyboard shortcuts** — `A` approve / `R` reject the top of the queue, + `S` toggle sound, `/` focus the add-symbol box, `?` help. + +## Repository map + +``` +strategy/ pure strategy engine: evaluation, structure, backtest +storage/ event schema, event store, DuckDB schema, projections +runtime/ watcher state machine + provider registry +research/ research DBs, ingestion (bars, RSS, gappers), providers +alpaca_paper/ Alpaca paper client, executor, account sync +schwab/ Schwab OAuth, market data, positions/orders, health +trading_execution.py signal → sizing → approval → order pipeline +run_live_paper.py the orchestrator (M4 end-to-end) +api/ dashboard JSON API + static UI +dashboard_api.py standalone read-only dashboard server +tests/ unit + integration suite (pytest) +``` + +CLI utilities: `fetch_minute_bars.py` (ad-hoc bar ingestion), +`backtest_cli.py` (intraday backtests over stored sessions), +`research_cli.py` (session symbols / gapper scan / RSS news), +`milestone3_verify.py` (Schwab health walk-through). + +## Tests + +```bash +python -m pytest # 60 tests: strategy, storage, watcher, execution, + # schwab, api, research, end-to-end pipeline +``` + +## Security + +`data/schwab_tokens.json` (and all of `data/`) is git-ignored and written +with `0600` permissions. **If tokens were ever committed to your repository +history, revoke and re-issue them at Schwab** — see RUNBOOK.md. diff --git a/README_ARCHITECTURE_REDESIGN.md b/README_ARCHITECTURE_REDESIGN.md new file mode 100644 index 0000000..f6d4125 --- /dev/null +++ b/README_ARCHITECTURE_REDESIGN.md @@ -0,0 +1,311 @@ +# Architecture Redesign: From Black Box to Transparent Pipeline + +## The Problem You Identified + +- ❌ Can't see if bars are being collected +- ❌ Can't verify research data is accurate +- ❌ Can't test modules independently +- ❌ UI provides no debugging help +- ❌ No visibility into calculation steps + +--- + +## The Solution: 8-Phase Implementation + +A complete redesign that transforms the system from a black box into a transparent, observable, testable pipeline. + +### Overview + +``` +Phase 1: Core Infrastructure ✅ DONE + ↓ +Phase 2: Module Instrumentation (4 hours) + ↓ +Phase 3: Inspection API Backend (6 hours) + ↓ +Phase 4: React Dashboard (8 hours) + ↓ +Phases 5-8: Refactoring & Optional Features +``` + +--- + +## Documentation + +Start with these files in order: + +### 1. **READ FIRST: BUILD_SUMMARY.md** + - Executive summary of all 8 phases + - Timeline and effort estimates + - Success criteria + - Why this solves your problems + +### 2. **PHASE1_IMPLEMENTATION.md** + - What was built in Phase 1 (contracts, telemetry, validation) + - How to use each new component + - Example code patterns + - Key takeaway: System is no longer a black box + +### 3. **INSPECTION_ROADMAP.md** + - What you'll be able to see after all phases complete + - Interactive dashboard features + - Symbol inspector details + - Real-world debugging scenarios + +### 4. **PHASE2_PREVIEW.md** + - Exactly what Phase 2 code looks like + - Before/after comparisons + - How many events you'll see + - Time breakdown for Phase 2 work + +--- + +## Quick Start: What's Ready Now (Phase 1) + +### New Modules + +**`core/contracts.py`** — Module I/O Contracts +```python +# Every module has typed input/output +EvaluationRequest → evaluate_setup() → EvaluationResult +``` + +**`core/telemetry.py`** — Metric Collection +```python +# Record what's happening during execution +collector = TelemetryCollector("evaluation") +collector.record_metric("gap_pct", 2.5) +collector.record_output(2) # 2 symbols ready +metrics = collector.finalize() +``` + +**`core/validation.py`** — Data Validation +```python +# Validate data at module boundaries +validator = SchemaValidator(required_fields={"symbol": str, "price": float}) +is_valid, failures = validator.validate(data) +``` + +### New Event Types + +In `storage/event_schema.py`: +- `ModuleTickEvent` — Module execution telemetry +- `DataValidationEvent` — Data quality checkpoints +- `SymbolEvaluationDetailEvent` — Deep-dive symbol metrics + +### Example Usage + +```python +from core.telemetry import emit_module_tick, emit_symbol_evaluation_detail + +# Emit when symbol evaluation stage completes +emit_symbol_evaluation_detail( + symbol="AAPL", + stage="rvol_calculated", + metrics={"rvol": 1.2, "threshold": 1.5}, + passed=False, + blocking_reason="RVOL 1.2x < 1.5x threshold" +) + +# Events are now in the event store, queryable by the dashboard +``` + +--- + +## Next: Phase 2 (Ready to Start) + +Phase 2 instruments the existing modules to emit telemetry events. + +**Work:** ~4 hours +1. Wrap `ingest_live_minute_bars()` with telemetry +2. Wrap `evaluate_setup()` with telemetry +3. Wrap execution, broker sync, research modules +4. Test to verify events are being emitted + +**See:** PHASE2_PREVIEW.md for exact code examples + +--- + +## The End Goal: Full Transparency + +After all 8 phases, opening `http://127.0.0.1:8011` shows: + +``` +AAPL (state: blocked, score: 48%) +├─ INGESTION ✓ +│ └─ 12 bars collected, validation passed +├─ RESEARCH ✓ +│ ├─ gap: 2.1% ✓ +│ └─ rvol: 1.2x ✗ (need 1.5x) ← BLOCKING +├─ EVALUATION +│ ├─ gap_checked ✓ +│ ├─ rvol_calculated ✗ ← stops here +│ └─ score: 48% (threshold: 60%) +└─ (not evaluated, blocked upstream) +``` + +**Time to understand why AAPL didn't signal:** < 10 seconds + +--- + +## Files Overview + +### Core Infrastructure (Phase 1) +``` +core/ +├── contracts.py # Module I/O types (500 lines) +├── telemetry.py # Metric collection (250 lines) +└── validation.py # Data validation (400 lines) +``` + +### Documentation +``` +PHASE1_IMPLEMENTATION.md # How Phase 1 works (200 lines) +INSPECTION_ROADMAP.md # Full system visibility (400 lines) +BUILD_SUMMARY.md # All 8 phases + timeline (350 lines) +PHASE2_PREVIEW.md # Exact Phase 2 code (300 lines) +``` + +### Later Phases +``` +api/inspection.py # Phase 3: Inspection API +api/inspection/ # Phase 4: React dashboard +modules/ # Phase 5: Refactored modules +tests/modules/ # Phase 6: Module tests +``` + +--- + +## Key Design Principles + +1. **Explicit Contracts** + - Every module has `Request` → `Result` types + - IDE autocomplete friendly + - Bugs obvious when types don't match + +2. **Observability First** + - Every module emits telemetry + - Metrics, timing, errors all captured + - Events flow to event store + +3. **Validation at Boundaries** + - Data validated when entering each module + - Business rules checked + - Checksums computed + +4. **Modularity** + - Each component is independent + - Can test in isolation + - Can swap implementations (mock vs. real) + +5. **Transparency** + - Nothing happens silently + - All state changes are events + - Complete audit trail + +--- + +## Success Metrics + +| Metric | Before | After | +|--------|--------|-------| +| Time to debug "why isn't AAPL signaling?" | 30 mins | 10 secs | +| Can test evaluation module alone | No | Yes | +| Can see bars being collected | No | Yes | +| Can verify research data accuracy | No | Yes | +| Module error tracking | None | Complete | +| Data validation coverage | None | 100% | + +--- + +## Implementation Path + +### Immediate (This Week) +- ✅ Phase 1: Core infrastructure (DONE) +- 📍 Phase 2: Instrument existing modules (4 hours) + +### Week 2 +- Phase 3: Inspection API (6 hours) +- Phase 4: React dashboard (8 hours) + +### Week 3+ +- Phases 5-8: Refactoring, testing, optional features + +--- + +## Questions to Ask While Reading + +As you go through the documentation, ask yourself: + +1. **On contracts:** "If I call `evaluate_setup()` with an `EvaluationRequest`, what exactly should I expect back?" + → Answer: An `EvaluationResult` with specific fields + +2. **On telemetry:** "How long did evaluation take? How many symbols were ready? What metrics does it track?" + → Answer: All captured in `ModuleTickEvent` + +3. **On validation:** "What if a bar has zero volume? What if the high is lower than the low?" + → Answer: `BarDataValidator` catches both + +4. **On events:** "If I want to understand why AAPL is blocked, where do I look?" + → Answer: `SymbolEvaluationDetailEvent` shows each step + +5. **On inspection:** "Can I see the exact gap%, rvol, and bull-flag score?" + → Answer: Yes, all in the dashboard after Phase 4 + +--- + +## Reading Order + +1. **BUILD_SUMMARY.md** — Start here for the big picture +2. **PHASE1_IMPLEMENTATION.md** — Understand what was built +3. **PHASE2_PREVIEW.md** — See what you'll code next +4. **INSPECTION_ROADMAP.md** — Visualize the end goal +5. **Code files** — `core/contracts.py`, `core/telemetry.py`, `core/validation.py` + +--- + +## Next Steps + +1. Read BUILD_SUMMARY.md (20 minutes) +2. Review PHASE1_IMPLEMENTATION.md (15 minutes) +3. Skim PHASE2_PREVIEW.md to see code examples (10 minutes) +4. Decide: Ready to start Phase 2, or want clarifications first? + +If ready for Phase 2: +- Use PHASE2_PREVIEW.md as your code template +- Follow the work breakdown in BUILD_SUMMARY.md +- Test after each module is instrumented +- Verify events appear in event store + +--- + +## Summary + +**Problem:** Black-box system, no visibility, can't test modules, can't debug + +**Solution:** 8-phase implementation adding: +- ✅ Module contracts (Phase 1) +- 📍 Module telemetry (Phase 2) +- Module inspection API (Phase 3) +- Module inspection dashboard (Phase 4) +- Module refactoring (Phase 5) +- Module testing (Phase 6) +- Dry-run mode (Phase 7) +- TUI debugging (Phase 8) + +**Timeline:** 4 weeks for full implementation, 2 weeks for core (phases 1-4) + +**Benefit:** From 30 minutes to 10 seconds to understand why a signal didn't execute + +--- + +## Let's Go + +Phase 1 is complete. Phase 2 is ready to start. + +Which would help you more right now: +1. More explanation of the design? +2. Code walkthrough of Phase 1? +3. Start working on Phase 2? + +Let me know! diff --git a/RUNBOOK.md b/RUNBOOK.md new file mode 100644 index 0000000..7878af6 --- /dev/null +++ b/RUNBOOK.md @@ -0,0 +1,227 @@ +# RUNBOOK + +Operational guide for running the momentum system. This file replaces the +original MILESTONE*.md documents, which were unrecoverable during the +rebuild. + +--- + +## 1. Why live paper trading never worked (root causes, now fixed) + +1. **Minute bars were never ingested.** The watcher reads 1-minute bars from + `data/research/market.duckdb`, but the old ingestion only wrote *daily* + bars for a handful of mega-caps. Fix: `research/ingestion/market_data.py` + ingests 1-minute Alpaca IEX bars continuously, and + `run_live_paper.py` schedules it every `LIVE_BARS_INTERVAL_SECONDS`. + +2. **Session flags were hardcoded.** Every old row had + `is_regular_hours = TRUE`. Fix: flags are derived from the bar timestamp + in US/Eastern (premarket 04:00–09:30, regular 09:30–16:00, afterhours + 16:00–20:00) — see `classify_session()` and its tests. + +3. **The candidate filter excluded everything ingested.** The query had a + hardcoded `close BETWEEN 1 AND 20` while only mega-caps (price ≫ 20) were + in the database, so the watchlist was always empty. Fix: the band is + configurable (`WATCHER_PRICE_MIN` / `WATCHER_PRICE_MAX`) and the default + symbol list now actually gets ingested. + +4. **No orchestrator.** Ingestion, watcher, broker sync, and execution were + never wired into one loop. Fix: `run_live_paper.py`. + +5. **Broken source files.** `runtime/providers/registry.py`, + `schwab/market/models.py`, `option_chain_service.py`, and + `schwab/streaming/client.py` had syntax errors that crashed imports. + All repaired/rewritten. + +6. **Schwab tokens were committed to git.** `data/schwab_tokens.json` is now + git-ignored and written `chmod 0600`. **Action required:** if that file + exists anywhere in your repo history, revoke the refresh token in the + Schwab developer portal and re-run the OAuth flow. Scrub history with + `git filter-repo --path data/schwab_tokens.json --invert-paths` if the + repo was ever shared. + +--- + +## 2. Environment setup + +```bash +pip install -r requirements.txt +cp .env.example .env +``` + +Minimum for paper trading: `ALPACA_API_KEY`, `ALPACA_SECRET_KEY` +(free keys are fine — the system uses the IEX feed). Everything else has a +sensible default; see `.env.example` for the full catalog. + +Key knobs: + +| Variable | Default | Meaning | +|---|---|---| +| `TRADING_EXECUTION_MODE` | `alpaca_paper` | `schwab_live` routes to real money — leave on paper | +| `TRADING_AUTO_APPROVE` | `0` | `1` = orders submit without dashboard approval (or pass `--auto-approve`) | +| `TRADING_MAX_ORDERS_PER_TICK` | `1` | throttle new entries | +| `TRADING_MAX_CONCURRENT_POSITIONS` | `3` | cap on simultaneous open positions | +| `TRADING_RISK_PER_TRADE_PCT` | `0.01` | 1% of equity risked per trade (sizing = risk / (entry − stop)) | +| `TRADING_REWARD_MULTIPLE` | `2.0` | profit target = entry + this × risk (2R by default) | +| `TRADING_ENTRY_ORDER_TYPE` | `limit` | `limit` rests at the trigger so an unfilled entry can be cancelled; `market` fills instantly | +| `TRADING_ENTRY_TIMEOUT_BARS` | `2` | cancel an unfilled entry after this many minutes resting (wall-clock; `0` disables the time box) | +| `TRADING_ENTRY_INVALIDATE_PCT` | `0.0` | cancel an unfilled entry if the **live last-trade price** trades this fraction below the trigger (`0.0` = any break below entry; `0.005` = 0.5% below; negative disables) | +| `TRADING_ENTRY_GUARD_INTERVAL_SECONDS` | `5` | how often the fast guard re-checks armed entries against the live price — lower = faster break detection, more quote calls | +| `TRADING_MAX_DAILY_LOSS_PCT` | `0.03` | daily-loss circuit breaker | +| `WATCHER_SYMBOLS` | `AAPL,TSLA,…` | static watchlist; `--discover` adds screener actives | +| `WATCHER_PRICE_MIN/MAX` | `1` / `20` | candidate price band | +| `WATCHER_EVENT_DB_PATH` | `./data/momentum.duckdb` | the event store | +| `DASHBOARD_HOST/PORT` | `127.0.0.1` / `8010` | UI address | + +--- + +## 3. Start commands + +**Full live paper loop (recommended):** + +```bash +python run_live_paper.py +# dashboard: http://127.0.0.1:8010 +# flags: --once --symbols SNDL,COSM --auto-approve --discover --no-dashboard +``` + +Each tick prints one status line: + +``` +[09:31:02] ingest: 240 minute rows | watch: evaluated=5 ready=['SNDL'] blocked=3 | sync: ok | execute: approvals_requested=['…'] auto_executed=0 +``` + +**Standalone read-only dashboard** (e.g. to review a past session): + +```bash +python dashboard_api.py +``` + +Approve/reject/exit buttons require the orchestrator (it attaches the +execution service to the same server); the standalone runner returns 503 on +actions by design. + +**Other tools:** + +```bash +python fetch_minute_bars.py SNDL COSM --lookback 390 # ad-hoc ingestion +python research_cli.py symbols --date 2026-06-11 # what's in the DB +python research_cli.py gappers --date 2026-06-11 # gap scan → scanner_snapshots +python backtest_cli.py --date 2026-06-10 # backtest a stored session +python milestone3_verify.py # Schwab health walk +python -m pytest # full test suite +``` + +--- + +## 4. The trading day, end to end + +1. **Ingest** — every 60 s, 1-minute IEX bars for the watchlist land in + `data/research/market.duckdb` with correct session flags. On boot, a + 30-day daily backfill gives gap % and relative volume a baseline. +2. **Watch** — every 30 s the watcher pulls in-band candidates, evaluates + nine weighted criteria (gap, RVOL, impulse, pullback, pullback volume, + VWAP, candle quality, breakout, data sufficiency) plus structure + detection, and drives `discovered → watching → ready | blocked | late`. + A `signal_ready` is emitted at most once per symbol per session. +3. **Sync** — Alpaca account, positions, and orders snapshot into the event + store so projections and risk guards see broker truth. +4. **Execute** — ready signals become orders sized at `risk_per_trade_pct` + of equity, with the stop from the setup and the target at + entry + `reward_multiple` × risk (2R by default). Manual mode parks them + in the dashboard queue; **auto mode (`--auto-approve`) submits + immediately, no approval.** Guards: max concurrent positions, no re-entry + into held symbols, one request per symbol per session. +5. **Manage** — approve/reject from the dashboard; **Exit** sends a market + sell for the synced quantity. + +### The entry lifecycle (auto-arm → fill or back out) + +The earlier system would "see" a setup as something that had *already +happened* rather than a trade about to happen, and never pull the trigger. +The execution model now separates recognition from conviction from +discipline: + +1. **Recognize** — when all the criteria pass, a `signal_ready` fires. The + setup is evaluated against the **last bar's timestamp**, not the wall + clock, so a live 9:40 breakout is judged against the 9:40 cutoff instead + of being dismissed as hours stale. +2. **Arm with conviction** — in auto mode the order is submitted right away + as a **bracket** (entry + stop + target placed atomically). With + `TRADING_ENTRY_ORDER_TYPE=limit` (default) the entry rests at the trigger, + so "unfilled" is a real, observable state. +3. **Back out at a defined point** — a fast **guard loop** + (`TRADING_ENTRY_GUARD_INTERVAL_SECONDS`, default 5 s) re-checks every + armed-but-unfilled entry and cancels it if either: + - the **live last-trade price** (pulled tick-by-tick from Alpaca's data + API, not a bar close) trades back below the trigger by + `TRADING_ENTRY_INVALIDATE_PCT`, or + - it has rested longer than `TRADING_ENTRY_TIMEOUT_BARS` minutes + (wall-clock — running the guard often does **not** shorten this window). + A cancel frees the risk budget and the concurrent-position slot, and emits + a `risk_rule_triggered` (`entry_backout`) you can see in the activity feed. + Once the order **fills**, it stops being tracked and the bracket's stop and + target manage the open position — no second-guessing. + +Tuning cheat-sheet: react to a fakeout faster → lower +`TRADING_ENTRY_GUARD_INTERVAL_SECONDS` (e.g. `2`); more patience on the fill → +raise `TRADING_ENTRY_TIMEOUT_BARS`; give the breakout room to wobble → set +`TRADING_ENTRY_INVALIDATE_PCT` to something like `0.005`; chase fills instead +of resting → `TRADING_ENTRY_ORDER_TYPE=market` (note: market fills can't time +out); bigger winners → raise `TRADING_REWARD_MULTIPLE`. + +> **Quote rate limits.** The guard makes one latest-trade call per armed +> symbol per pass (a ~2 s cache collapses duplicates). At the 5 s default with +> a few open entries you're well inside Alpaca's free-tier 200 req/min. If you +> drop the interval to 1 s with several armed entries, watch the budget — +> bars ingestion and account sync also draw from it. + +### Reading the dashboard + +- **Badge** — blue `PAPER · ALPACA` vs red `LIVE · SCHWAB`. If it's red and + you didn't mean it, stop and check `TRADING_EXECUTION_MODE`. +- **P&L strip** (top) — session realized / unrealized / total, win rate, + average R, open/closed counts. Green/red signed. +- **Approval Queue** — pending entries with a **risk preview** ($ risk, + shares, notional, stop %) and Approve/Reject. Keyboard: `A` / `R` act on the + top row. +- **Ready Signals / Open Positions** — each row has a live **sparkline** (last + ~40 one-minute closes) with dashed entry (blue) and stop (red) lines. +- **Watch States** — click any row to expand the **nine-criteria breakdown** + (✓ pass / ✕ fail / · not-evaluated). The add-symbol box (or `/`) injects a + ticker onto the watchlist for the rest of the session. +- **Activity Feed** — submissions and fills as they happen. +- **Order Lifecycle** — per-order status from the event store. +- **Connection dot** (top right) — green pulse = live SSE stream; it falls + back to 4 s polling automatically if the stream drops. +- **Sound** — toggle the button or press `S` for an audio + toast alert when a + new signal goes ready. Press `?` for the shortcut list. + +Streams from `GET /api/stream` (SSE); also exposes `/api/snapshots`, +`/api/criteria?symbol=`, `/api/bars?symbol=&minutes=`, and +`POST /api/watch/add`. Charts require the research DB, which the orchestrator +wires in automatically; the standalone `dashboard_api.py` has no bars source, +so sparklines there show "no bars". + +--- + +## 5. Troubleshooting + +| Symptom | Likely cause / fix | +|---|---| +| `watch: evaluated=0` all day | No minute bars for today. Check keys; run `python research_cli.py symbols`. Outside market hours there are no new bars — backfill with `fetch_minute_bars.py --lookback 1440`. | +| Candidates exist but never `ready` | Score < 60%. Watch the `Watch States` panel score column; criteria details are in `criteria_evaluated` events. | +| `ingest: … 403` errors | Free keys are IEX-only — ensure `ALPACA_DATA_FEED=iex` (default) and that the request window ends ≥30 s in the past (handled automatically). | +| Approve button does nothing | You're on `dashboard_api.py` (read-only). Use `run_live_paper.py`. | +| Port 8010 busy | Set `DASHBOARD_PORT`, or `--no-dashboard`. | +| Schwab shows `SCHWAB-UNAUTH` | Expected without OAuth. Run the flow in `schwab/auth/oauth2_flow_manager.py`; verify with `milestone3_verify.py`. | +| Start fresh | Delete `data/momentum.duckdb` (events) and/or `data/research/` (bars). Both rebuild automatically. | + +--- + +## 6. Going live on Schwab (when ready) + +Paper-trade until the stats earn it. Then: complete OAuth +(`milestone3_verify.py` all green), set `TRADING_EXECUTION_MODE=schwab_live`, +keep `TRADING_AUTO_APPROVE=0`, and start with the smallest viable +`TRADING_RISK_PER_TRADE_PCT`. The dashboard badge turns red in live mode. diff --git a/SYSTEM_FLOW.txt b/SYSTEM_FLOW.txt new file mode 100644 index 0000000..06838f4 --- /dev/null +++ b/SYSTEM_FLOW.txt @@ -0,0 +1,405 @@ +================================================================================ + MOMENTUM TRADING SYSTEM - COMPLETE LOGICAL FLOW +================================================================================ + +ENTRY POINT: python run_live_paper.py [--once] [--symbols AAPL,SNDL] [--auto-approve] + +================================================================================ +STARTUP PHASE +================================================================================ + +1. INITIALIZE CONFIGURATION + ├─ Load .env (ALPACA_API_KEY, ALPACA_SECRET_KEY, etc.) + ├─ Parse CLI args + └─ Create settings objects: AlpacaPaperSettings, ExecutionSettings, TradingModeSettings + +2. INITIALIZE STORAGE & DATABASES + ├─ EventStore("data/momentum_events.duckdb") + │ └─ Append-only log of all events (canonical source of truth) + ├─ ResearchConnection("data/research/market.duckdb") + │ └─ Minute bars + daily bars for all symbols + └─ AlpacaPaperClient (if API keys present) + └─ Connection to Alpaca paper trading account + +3. INITIALIZE RUNTIME COMPONENTS + ├─ Watcher (state machine for symbol discovery/evaluation) + │ └─ Provider: ResearchWatchlistProvider (pulls symbols from research DB) + ├─ TradingExecutionService (signal → approval → order pipeline) + ├─ AlpacaPaperSync (poll Alpaca account/positions/orders) + └─ DashboardAPI (optional HTTP server on port 8010) + └─ Serves JSON API + static UI + SSE event stream + +================================================================================ +MAIN LOOP (runs every tick, or once with --once) +================================================================================ + +TICK = 1 iteration (typically 60 seconds) + + ┌─────────────────────────────────────────────────────────────────────────┐ + │ PHASE 1: INGEST LATEST MARKET DATA │ + └─────────────────────────────────────────────────────────────────────────┘ + + ingest_live_minute_bars() + ├─ Call Alpaca REST API: get latest 1-minute bars + ├─ Insert into research/market.duckdb (bars table) + └─ Return: list of symbols with new data + + ingest_daily_history() + ├─ On startup or when needed: + │ ├─ Alpaca REST API: get daily bars (past 30 days) + │ ├─ Insert into research/market.duckdb (daily_bars table) + │ └─ Used by: gap%, RVOL calculations + └─ Return: daily bar snapshots + + ┌─────────────────────────────────────────────────────────────────────────┐ + │ PHASE 2: DISCOVER & UPDATE WATCHLIST │ + └─────────────────────────────────────────────────────────────────────────┘ + + ResearchWatchlistProvider.get_candidates(session_date) + ├─ Check what symbols are in the watchlist for today + │ ├─ Manual symbols (from --symbols or WATCHER_SYMBOLS env) + │ ├─ Screener results (most-actives from Alpaca if --discover) + │ └─ RSS feed candidates (if configured) + ├─ Return list of WatchCandidate objects (symbol, last_price, avg_volume) + └─ Watcher._states updated with new symbols + + ┌─────────────────────────────────────────────────────────────────────────┐ + │ PHASE 3: EVALUATE SYMBOLS (CORE STRATEGY LOGIC) │ + └─────────────────────────────────────────────────────────────────────────┘ + + Watcher.tick(bar_data, candidates) + │ + └─ For each symbol in candidates: + │ + ├─ 1. FETCH BARS + │ └─ ResearchWatchlistProvider.get_bars(symbol, session_date) + │ └─ Query research DB for all minute bars from 9:30 AM to now + │ + ├─ 2. EMIT SYMBOL DISCOVERED (if new) + │ └─ EventStore.emit(SymbolDiscoveredEvent(symbol, source, ...)) + │ └─ Event recorded with timestamp, session_id, payload + │ + ├─ 3. EVALUATE SETUP (if ≥10 bars available) + │ │ + │ └─ evaluate_setup(bars, symbol) ← PURE STRATEGY ENGINE + │ │ + │ ├─ Gap Check + │ │ └─ (today_open - prev_close) / prev_close ≥ 2% ? + │ │ + │ ├─ Relative Volume (RVOL) + │ │ └─ (current 5-min avg vol / 20-day avg) ≥ 1.5 ? + │ │ + │ ├─ Bull Flag Structure + │ │ └─ Pullback from high + ascending support pattern? + │ │ + │ ├─ Price Action Quality + │ │ └─ Higher highs + higher lows? Clean bars? + │ │ + │ ├─ 5 Other Criteria (tunable) + │ │ └─ Momentum, volatility, trend confirmation, etc. + │ │ + │ ├─ Calculate SCORE + │ │ └─ (criteria_passed / total_criteria) * 100 = X% + │ │ + │ └─ Return: {score, details, setup_data} + │ + ├─ 4. EMIT CRITERIA EVALUATED + │ └─ EventStore.emit(CriteriaEvaluatedEvent(symbol, criteria=[...], score)) + │ └─ Each criterion: {name, passed, reason} + │ + ├─ 5. CHECK SIGNAL READY THRESHOLD + │ └─ IF score ≥ ready_score_pct (default 60%) AND NOT already_ready_this_session: + │ │ + │ ├─ Emit SIGNAL READY + │ │ └─ EventStore.emit(SignalReadyEvent(symbol, entry, stop, target, ...)) + │ │ + │ ├─ Update symbol state: watching → ready + │ │ └─ EventStore.emit(SymbolStateChangedEvent(symbol, "watching" → "ready")) + │ │ + │ └─ Mark symbol as already signaled (no duplicate signals this session) + │ + ├─ 6. CHECK SIGNAL BLOCKED + │ └─ ELSE IF score < ready_score_pct OR quality < min_quality: + │ │ + │ ├─ Emit SIGNAL BLOCKED + │ │ └─ EventStore.emit(SignalBlockedEvent(symbol, reason, ...)) + │ │ + │ ├─ Update symbol state: watching → blocked + │ │ └─ EventStore.emit(SymbolStateChangedEvent(symbol, "watching" → "blocked")) + │ │ + │ └─ RESET: allow signal_ready again if it recovers above threshold + │ + └─ END for each symbol + + ┌─────────────────────────────────────────────────────────────────────────┐ + │ PHASE 4: EXECUTION PIPELINE (signal_ready → approval → order) │ + └─────────────────────────────────────────────────────────────────────────┘ + + TradingExecutionService.tick() + │ + └─ Query event store for new signal_ready events (not yet processed) + │ + └─ For each signal_ready: + │ + ├─ 1. CHECK RISK RULES + │ │ + │ ├─ How many positions are open? + │ │ └─ IF open_positions ≥ max_concurrent_positions (e.g., 3): + │ │ └─ RiskRuleTriggeredEvent("max concurrent positions") + │ │ └─ SKIP this signal + │ │ + │ ├─ Check daily loss circuit breaker + │ │ └─ IF realized_pnl_today ≤ -max_daily_loss_pct * account_equity: + │ │ └─ RiskRuleTriggeredEvent("daily loss exceeded") + │ │ └─ SKIP this signal (halt trading for the day) + │ │ + │ └─ Max orders per tick + │ └─ Only submit max_orders_per_tick (e.g., 1) per tick to avoid order queue floods + │ + ├─ 2. CALCULATE POSITION SIZE + │ │ + │ └─ calculate_position_size(entry, stop, account_equity, risk_per_trade_pct) + │ │ + │ ├─ Risk per trade = account_equity * risk_per_trade_pct (e.g., 1%) + │ ├─ Risk per share = entry - stop + │ ├─ Shares = risk_per_trade / risk_per_share + │ └─ Return: {shares, entry, stop, target} + │ + ├─ 3. EMIT ORDER APPROVAL REQUESTED + │ └─ EventStore.emit(OrderApprovalRequestedEvent( + │ symbol, entry, stop, target, shares, risk_dollar, notional, ... + │ )) + │ + ├─ 4. AUTO-APPROVE OR QUEUE FOR MANUAL APPROVAL + │ │ + │ ├─ IF --auto-approve or TRADING_AUTO_APPROVE=1: + │ │ │ + │ │ ├─ Emit OrderApprovedEvent immediately + │ │ │ └─ EventStore.emit(OrderApprovedEvent(order_id, ...)) + │ │ │ + │ │ └─ Jump to Phase 5 (SUBMIT ORDER) + │ │ + │ └─ ELSE (manual approval): + │ │ + │ └─ Add to approval_queue in dashboard + │ └─ User sees: symbol, entry, stop, target, risk, share count, sparkline chart + │ └─ User clicks Approve or Reject + │ └─ Dashboard emits OrderApprovedEvent or OrderRejectedEvent + │ + ├─ 5. ON APPROVAL (if approved) + │ │ + │ └─ TradingExecutionService detects OrderApprovedEvent + │ │ + │ ├─ Create ExecutionRequest + │ │ └─ {symbol, entry, stop, target, shares, order_type: "limit"/"market", ...} + │ │ + │ └─ Call AlpacaPaperExecutor.submit(request) + │ │ + │ └─ AlpacaPaperExecutor.submit() [PHASE 5] + │ + └─ END for each signal_ready + + ┌─────────────────────────────────────────────────────────────────────────┐ + │ PHASE 5: BROKER ORDER SUBMISSION │ + └─────────────────────────────────────────────────────────────────────────┘ + + AlpacaPaperExecutor.submit(execution_request) + │ + ├─ Create bracket order: + │ │ + │ ├─ ENTRY ORDER + │ │ └─ Type: limit @ entry_price (restful, will fill on touch, can be cancelled) + │ │ OR market (fills immediately at market) + │ │ + │ ├─ PROFIT TARGET + │ │ └─ Type: limit @ target_price + │ │ (target = entry + (entry - stop) * reward_multiple) + │ │ + │ └─ STOP LOSS + │ └─ Type: stop @ stop_price (becomes market order when stop is hit) + │ + ├─ Submit to Alpaca REST API + │ └─ Returns: Alpaca order_id + │ + ├─ Emit OrderSubmittedEvent + │ └─ EventStore.emit(OrderSubmittedEvent(symbol, order_id, entry, stop, target, ...)) + │ + ├─ Track armed entry in TradingExecutionService._armed dict + │ └─ _armed[order_id] = {symbol, entry_price, broker_order_id, armed_at, checks} + │ + └─ Monitor for: + │ + ├─ Entry Fill + │ └─ If entry fills → position open → wait for target or stop + │ + ├─ Entry Timeout (if not filled after entry_timeout_bars minutes) + │ └─ Call Alpaca to cancel unfilled entry order + │ + ├─ Price Break Invalidation + │ └─ If current price < (entry - entry_invalidate_pct * entry): + │ └─ Cancel entry order (price broke back below entry) + │ + └─ Full Fill (target or stop hit) + └─ Position closes → emit OrderFilledEvent + + ┌─────────────────────────────────────────────────────────────────────────┐ + │ PHASE 6: ACCOUNT & POSITION SYNC │ + └─────────────────────────────────────────────────────────────────────────┘ + + AlpacaPaperSync.sync_account() + │ + ├─ 1. GET ACCOUNT SUMMARY + │ └─ Alpaca REST: /account + │ └─ Returns: {equity, buying_power, multiplier, long_market_value, ...} + │ + ├─ 2. EMIT ACCOUNT SUMMARY UPDATED + │ └─ EventStore.emit(AccountSummaryUpdatedEvent(account_data)) + │ + ├─ 3. GET OPEN POSITIONS + │ └─ Alpaca REST: /positions + │ └─ Returns: [{symbol, qty, avg_fill_price, current_price, unrealized_pl, ...}] + │ + ├─ 4. EMIT ACCOUNT POSITIONS UPDATED + │ └─ EventStore.emit(AccountPositionsUpdatedEvent(positions)) + │ + ├─ 5. GET PENDING ORDERS + │ └─ Alpaca REST: /orders?status=open + │ └─ Returns: [{id, symbol, qty, type, status, filled_qty, ...}] + │ + ├─ 6. EMIT ACCOUNT ORDERS UPDATED + │ └─ EventStore.emit(AccountOrdersUpdatedEvent(orders)) + │ + └─ 7. CALCULATE SESSION P&L + └─ Query event store for all fills this session + ├─ For each closed trade: + │ └─ realized_pnl = (exit_price - entry_price) * qty + ├─ Aggregate: + │ └─ total_realized_pnl, total_unrealized_pnl, win_rate, avg_r, trade_count + └─ Used by dashboard P&L strip + + ┌─────────────────────────────────────────────────────────────────────────┐ + │ PHASE 7: DASHBOARD UPDATES (READ-ONLY PROJECTIONS) │ + └─────────────────────────────────────────────────────────────────────────┘ + + DashboardAPI (runs in background thread) + │ + ├─ SSE Stream Handler (/api/events) + │ └─ Pushes new events to all connected clients in real-time + │ └─ Falls back to polling if connection drops + │ + ├─ JSON Endpoints: + │ │ + │ ├─ GET /api/approval-queue + │ │ └─ query_approval_queue(event_store) + │ │ └─ Returns: [{symbol, entry, stop, target, shares, risk_pct, notional, ...}] + │ │ + │ ├─ GET /api/positions + │ │ └─ query_account_positions_snapshot(event_store) + │ │ └─ Returns: [{symbol, qty, entry, current, unrealized_pl, ...}] + │ │ + │ ├─ GET /api/pnl + │ │ └─ query_pnl_snapshot(event_store) + │ │ └─ Returns: {realized, unrealized, total, win_rate, avg_r, open_count, closed_count} + │ │ + │ ├─ GET /api/activity-feed + │ │ └─ query_activity_feed(event_store, limit=100) + │ │ └─ Returns: [{timestamp, event_type, message, symbol, ...}] + │ │ + │ ├─ GET /api/symbol/:symbol/bars + │ │ └─ query_minute_bars(research_db, symbol, session_date) + │ │ └─ Returns: [{timestamp, open, high, low, close, volume}] + │ │ (for sparkline chart rendering) + │ │ + │ └─ POST /api/approval/:order_id/approve|reject + │ └─ Emits OrderApprovedEvent or OrderRejectedEvent + │ + └─ UI Features: + ├─ P&L Strip (realized, unrealized, total, win rate, R multiple, counts) + ├─ Approval Queue (cards with entry/stop/target preview, Approve/Reject buttons) + ├─ Open Positions (list with live sparkline charts, P&L per position) + ├─ Activity Feed (chronological log of all events) + ├─ Symbol Details (expand to see 9 criteria: pass/fail/not-evaluated) + └─ Keyboard Shortcuts: A=Approve, R=Reject, S=Sound Toggle, /=Add Symbol, ?=Help + + ┌─────────────────────────────────────────────────────────────────────────┐ + │ PHASE 8: CLEANUP & REPEAT │ + └─────────────────────────────────────────────────────────────────────────┘ + + After all phases complete: + ├─ Log tick summary (discovered, ready, blocked, evaluated, errors) + ├─ IF --once: exit + └─ ELSE: sleep(60 seconds) and go back to PHASE 1 + +================================================================================ +KEY CONCEPTS & INVARIANTS +================================================================================ + +EVENT SOURCING + • All state changes are recorded as immutable events in the append-only event store + • The current state is reconstructed by replaying events (projections) + • Every read operation queries the event store; no other state is authoritative + • Correlation ID = session_id (all events from one trading session have the same correlation_id) + +SIGNAL DEBOUNCING + • Once a symbol emits signal_ready, it cannot emit another signal_ready until: + → It falls back to signal_blocked (criteria fail), which resets the debounce + • Prevents duplicate signals from the same symbol in one session + +STATE MACHINE (Watcher) + • discovered → watching → { ready | blocked } (can toggle between ready ↔ blocked) + • States are per-session; new session = fresh symbol states + +BRACKET ORDER + • Entry (limit) + Profit Target (limit) + Stop Loss (stop) submitted as a single order + • If entry fills: + → Two child orders activate (target and stop) + → First one to fill = exit (other cancels automatically) + • If entry never fills: + → Entry can timeout (entry_timeout_bars) or be cancelled on price break + +RISK CONTROLS + • Max concurrent positions: once N positions open, no more entry orders until one closes + • Daily loss circuit breaker: once losses hit threshold, halt trading for the session + • Entry timeout: if entry rests >N minutes, cancel to avoid stale orders + • Price break invalidation: if price falls back >X% below entry, cancel entry (setup failed) + +APPROVAL WORKFLOW + • Manual: user clicks Approve in dashboard → OrderApprovedEvent + • Auto: --auto-approve flag or TRADING_AUTO_APPROVE=1 env var → immediate approval + • Rejection: user clicks Reject or risk rule triggers → OrderRejectedEvent (queued signal removed) + +================================================================================ +TROUBLESHOOTING BY PHASE +================================================================================ + +Phase 1 (Ingest): + • No new bars? Check if Alpaca API keys are valid. Run --once in dry mode. + • Bars not in research DB? Check data/research/market.duckdb exists and is writable. + +Phase 2 (Discover): + • No symbols in watchlist? Check --symbols / WATCHER_SYMBOLS env / most-actives screener. + +Phase 3 (Evaluate): + • Symbols stuck in "watching"? Criteria too strict. Lower ready_score_pct or min_quality. + • Signals appear then disappear? They're being blocked. Check criteria_evaluated events. + • Check event store: SELECT * FROM events WHERE event_type = 'criteria_evaluated' + +Phase 4 (Execution): + • Signal ready but no approval request? Risk rule triggered. Check RiskRuleTriggeredEvent. + • Approval request visible but user forgets to click? Use --auto-approve for testing. + +Phase 5 (Broker): + • Order submitted but no fill? Entry price may be stale. Use market orders (entry_order_type). + • Entry stuck for >N minutes? Timeout will cancel it. Check entry_timeout_bars setting. + +Phase 6 (Sync): + • Account sync fails? Alpaca API likely down. Dry mode still works. + • Positions don't match Alpaca? Event store is source of truth. Refresh browser. + +Phase 7 (Dashboard): + • Dashboard not updating? Check browser console for errors. Try hard-refresh (Ctrl+Shift+R). + • SSE stream failing? Falls back to polling (/api/approval-queue, etc.). Check /api/events. + +================================================================================ +END +================================================================================ diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..1d858f5 --- /dev/null +++ b/__init__.py @@ -0,0 +1,9 @@ +"""Momentum trading system. + +Milestone 2: Event model and storage. +Milestone 3: Schwab integration + MOMENTUM_OS UI. +Milestone 4: Scanner execution loop + paper execution. +Milestone 5: Trading control room dashboard + live paper orchestrator. +""" + +__version__ = "0.4.0" diff --git a/alpaca_paper/__init__.py b/alpaca_paper/__init__.py new file mode 100644 index 0000000..01bb80c --- /dev/null +++ b/alpaca_paper/__init__.py @@ -0,0 +1,20 @@ +"""Alpaca paper-trading integration (execution + account sync).""" + +from alpaca_paper.settings import AlpacaPaperSettings +from alpaca_paper.client import AlpacaPaperClient, AlpacaApiError +from alpaca_paper.execution import ( + AlpacaPaperExecutor, + ExecutionRequest, + ExecutionResult, +) +from alpaca_paper.sync import AlpacaPaperSync + +__all__ = [ + "AlpacaPaperSettings", + "AlpacaPaperClient", + "AlpacaApiError", + "AlpacaPaperExecutor", + "ExecutionRequest", + "ExecutionResult", + "AlpacaPaperSync", +] diff --git a/alpaca_paper/client.py b/alpaca_paper/client.py new file mode 100644 index 0000000..3f36cc1 --- /dev/null +++ b/alpaca_paper/client.py @@ -0,0 +1,184 @@ +"""Minimal Alpaca REST client (stdlib only — no SDK dependency). + +Covers what the system needs: account, positions, orders, order submission +and cancellation, minute bars, latest trades, and the most-actives screener. +""" + +from __future__ import annotations + +import json +import logging +import urllib.error +import urllib.parse +import urllib.request + +from alpaca_paper.settings import AlpacaPaperSettings + +logger = logging.getLogger(__name__) + + +class AlpacaApiError(RuntimeError): + def __init__(self, status: int, body: str): + super().__init__(f"Alpaca API error {status}: {body[:300]}") + self.status = status + self.body = body + + +class AlpacaPaperClient: + def __init__(self, settings: AlpacaPaperSettings | None = None, timeout: int = 15): + self.settings = settings or AlpacaPaperSettings.from_env() + self.timeout = timeout + + # -- low level ------------------------------------------------------- + + def _headers(self) -> dict[str, str]: + return { + "APCA-API-KEY-ID": self.settings.api_key, + "APCA-API-SECRET-KEY": self.settings.secret_key, + "Content-Type": "application/json", + } + + def _request( + self, method: str, url: str, params: dict | None = None, body: dict | None = None + ): + if params: + qs = urllib.parse.urlencode( + {k: v for k, v in params.items() if v is not None} + ) + url = f"{url}?{qs}" + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request( + url, data=data, headers=self._headers(), method=method + ) + try: + with urllib.request.urlopen(req, timeout=self.timeout) as resp: + raw = resp.read().decode() + return json.loads(raw) if raw else {} + except urllib.error.HTTPError as exc: + raise AlpacaApiError(exc.code, exc.read().decode(errors="replace")) from exc + + def _trading(self, method: str, path: str, **kw): + return self._request( + method, f"{self.settings.trading_base_url}/v2{path}", **kw + ) + + def _data(self, method: str, path: str, **kw): + return self._request(method, f"{self.settings.data_base_url}{path}", **kw) + + # -- trading API ------------------------------------------------------- + + def get_account(self) -> dict: + return self._trading("GET", "/account") + + def get_positions(self) -> list[dict]: + return self._trading("GET", "/positions") + + def get_orders(self, status: str = "all", limit: int = 100) -> list[dict]: + return self._trading( + "GET", "/orders", params={"status": status, "limit": limit} + ) + + def submit_order( + self, + symbol: str, + qty: int, + side: str = "buy", + order_type: str = "market", + time_in_force: str = "day", + limit_price: float | None = None, + stop_loss_price: float | None = None, + take_profit_price: float | None = None, + client_order_id: str | None = None, + ) -> dict: + body: dict = { + "symbol": symbol, + "qty": str(qty), + "side": side, + "type": order_type, + "time_in_force": time_in_force, + } + if limit_price is not None: + body["limit_price"] = str(round(limit_price, 2)) + if client_order_id: + body["client_order_id"] = client_order_id + if stop_loss_price is not None or take_profit_price is not None: + body["order_class"] = "bracket" if ( + stop_loss_price and take_profit_price + ) else "oto" + if stop_loss_price is not None: + body["stop_loss"] = {"stop_price": str(round(stop_loss_price, 2))} + if take_profit_price is not None: + body["take_profit"] = {"limit_price": str(round(take_profit_price, 2))} + return self._trading("POST", "/orders", body=body) + + def cancel_order(self, order_id: str) -> None: + self._trading("DELETE", f"/orders/{order_id}") + + def close_position(self, symbol: str) -> dict: + return self._trading("DELETE", f"/positions/{symbol}") + + # -- market data API ---------------------------------------------------- + + def get_minute_bars( + self, + symbols: list[str], + start_iso: str, + end_iso: str | None = None, + limit: int = 10_000, + feed: str | None = None, + ) -> dict[str, list[dict]]: + """Fetch 1-minute bars for symbols, following pagination tokens.""" + all_bars: dict[str, list[dict]] = {s: [] for s in symbols} + page_token = None + while True: + params = { + "symbols": ",".join(symbols), + "timeframe": "1Min", + "start": start_iso, + "end": end_iso, + "limit": limit, + "feed": feed or self.settings.feed, + "adjustment": "raw", + "page_token": page_token, + } + payload = self._data("GET", "/v2/stocks/bars", params=params) + for symbol, bars in (payload.get("bars") or {}).items(): + all_bars.setdefault(symbol, []).extend(bars or []) + page_token = payload.get("next_page_token") + if not page_token: + break + return all_bars + + def get_latest_trades(self, symbols: list[str]) -> dict[str, dict]: + payload = self._data( + "GET", + "/v2/stocks/trades/latest", + params={"symbols": ",".join(symbols), "feed": self.settings.feed}, + ) + return payload.get("trades") or {} + + def get_most_actives(self, top: int = 20, by: str = "volume") -> list[dict]: + payload = self._data( + "GET", + "/v1beta1/screener/stocks/most-actives", + params={"top": top, "by": by}, + ) + return payload.get("most_actives") or [] + + def get_daily_bars( + self, symbols: list[str], start_iso: str, end_iso: str | None = None + ) -> dict[str, list[dict]]: + payload = self._data( + "GET", + "/v2/stocks/bars", + params={ + "symbols": ",".join(symbols), + "timeframe": "1Day", + "start": start_iso, + "end": end_iso, + "limit": 10_000, + "feed": self.settings.feed, + "adjustment": "raw", + }, + ) + return payload.get("bars") or {} diff --git a/alpaca_paper/execution.py b/alpaca_paper/execution.py new file mode 100644 index 0000000..a48d048 --- /dev/null +++ b/alpaca_paper/execution.py @@ -0,0 +1,223 @@ +"""Alpaca paper execution bridge. + +Turns an approved execution request into a real Alpaca paper order and +emits order lifecycle events into the event store. +""" + +from __future__ import annotations + +import logging +import uuid +from dataclasses import dataclass +from datetime import datetime + +from alpaca_paper.client import AlpacaApiError, AlpacaPaperClient +from storage.event_schema import ( + EventMode, + OrderCancelledEvent, + OrderFilledEvent, + OrderSubmittedEvent, +) +from storage.event_store import EventStore + +logger = logging.getLogger(__name__) + + +@dataclass +class ExecutionRequest: + """Broker-agnostic order request derived from a signal.""" + + symbol: str + side: str + quantity: int + entry_price: float | None = None + stop_loss_price: float | None = None + take_profit_price: float | None = None + order_type: str = "market" + time_in_force: str = "day" + order_id: str = "" + + def __post_init__(self): + if not self.order_id: + self.order_id = str(uuid.uuid4()) + + def to_payload(self) -> dict: + return { + "symbol": self.symbol, + "side": self.side, + "quantity": self.quantity, + "entry_price": self.entry_price, + "stop_loss_price": self.stop_loss_price, + "take_profit_price": self.take_profit_price, + "order_type": self.order_type, + "time_in_force": self.time_in_force, + "order_id": self.order_id, + } + + @classmethod + def from_payload(cls, payload: dict) -> "ExecutionRequest": + return cls( + symbol=payload["symbol"], + side=payload.get("side", "buy"), + quantity=int(payload.get("quantity", 0)), + entry_price=payload.get("entry_price"), + stop_loss_price=payload.get("stop_loss_price"), + take_profit_price=payload.get("take_profit_price"), + order_type=payload.get("order_type", "market"), + time_in_force=payload.get("time_in_force", "day"), + order_id=str(payload.get("order_id") or ""), + ) + + +@dataclass +class ExecutionResult: + ok: bool + order_id: str + broker_order_id: str | None = None + status: str | None = None + error: str | None = None + raw: dict | None = None + + +class AlpacaPaperExecutor: + """Submits execution requests to the Alpaca paper account.""" + + broker_name = "alpaca_paper" + + def __init__( + self, + store: EventStore, + client: AlpacaPaperClient | None = None, + session_id: str | None = None, + mode: EventMode = EventMode.PAPER, + ): + self.store = store + self.client = client or AlpacaPaperClient() + self.session_id = session_id + self.mode = mode + + def execute(self, request: ExecutionRequest) -> ExecutionResult: + if request.quantity <= 0: + return ExecutionResult( + ok=False, order_id=request.order_id, error="quantity must be > 0" + ) + try: + raw = self.client.submit_order( + symbol=request.symbol, + qty=request.quantity, + side=request.side, + order_type=request.order_type, + time_in_force=request.time_in_force, + limit_price=request.entry_price + if request.order_type == "limit" + else None, + stop_loss_price=request.stop_loss_price, + take_profit_price=request.take_profit_price, + client_order_id=request.order_id[:48], + ) + except AlpacaApiError as exc: + logger.error("alpaca order failed: %s", exc) + self._emit_cancelled(request, f"broker rejected: {exc}") + return ExecutionResult( + ok=False, order_id=request.order_id, error=str(exc) + ) + except Exception as exc: + logger.exception("alpaca order failed") + self._emit_cancelled(request, f"submit error: {exc}") + return ExecutionResult( + ok=False, order_id=request.order_id, error=str(exc) + ) + + broker_order_id = raw.get("id") + status = raw.get("status") + self.store.emit( + OrderSubmittedEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=( + f"Submitted {request.side} {request.quantity} {request.symbol} " + f"to alpaca_paper ({status})" + ), + symbol=request.symbol, + order_id=request.order_id, + side=request.side, + quantity=request.quantity, + price=request.entry_price or 0.0, + payload={ + "symbol": request.symbol, + "order_id": request.order_id, + "side": request.side, + "quantity": request.quantity, + "price": request.entry_price, + "broker_order_id": broker_order_id, + "broker_status": status, + }, + ) + ) + + if status == "filled": + fill_price = float(raw.get("filled_avg_price") or 0) or None + self.store.emit( + OrderFilledEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=f"Filled {request.symbol} @ {fill_price}", + symbol=request.symbol, + order_id=request.order_id, + fill_price=fill_price or 0.0, + fill_quantity=int(float(raw.get("filled_qty") or 0)), + payload={ + "symbol": request.symbol, + "order_id": request.order_id, + "fill_price": fill_price, + "fill_quantity": int(float(raw.get("filled_qty") or 0)), + }, + ) + ) + + return ExecutionResult( + ok=True, + order_id=request.order_id, + broker_order_id=broker_order_id, + status=status, + raw=raw, + ) + + def cancel_entry( + self, order_id: str, broker_order_id: str | None, symbol: str, reason: str + ) -> ExecutionResult: + """Cancel a resting (unfilled) entry order at the broker and record it.""" + try: + if broker_order_id: + self.client.cancel_order(broker_order_id) + except AlpacaApiError as exc: + # already filled/cancelled at the broker, or transient — log and + # still emit the cancel so our state stops tracking it + logger.warning("cancel of %s returned: %s", order_id, exc) + except Exception as exc: # noqa: BLE001 + logger.warning("cancel of %s errored: %s", order_id, exc) + self._emit_cancelled( + ExecutionRequest(symbol=symbol, side="buy", quantity=0, order_id=order_id), + reason, + ) + return ExecutionResult(ok=True, order_id=order_id, status="cancelled") + + def _emit_cancelled(self, request: ExecutionRequest, reason: str) -> None: + self.store.emit( + OrderCancelledEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=f"Order for {request.symbol} cancelled: {reason}", + symbol=request.symbol, + order_id=request.order_id, + cancel_reason=reason, + payload={ + "symbol": request.symbol, + "order_id": request.order_id, + "cancel_reason": reason, + }, + ) + ) diff --git a/alpaca_paper/settings.py b/alpaca_paper/settings.py new file mode 100644 index 0000000..86ed340 --- /dev/null +++ b/alpaca_paper/settings.py @@ -0,0 +1,39 @@ +"""Alpaca paper-trading settings.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + + +@dataclass(frozen=True) +class AlpacaPaperSettings: + api_key: str = "" + secret_key: str = "" + trading_base_url: str = "https://paper-api.alpaca.markets" + data_base_url: str = "https://data.alpaca.markets" + feed: str = "iex" + + @classmethod + def from_env(cls, env: dict[str, str] | None = None) -> "AlpacaPaperSettings": + values = dict(os.environ) + if env is not None: + values.update(env) + api_key = values.get("ALPACA_API_KEY") or values.get("APCA_API_KEY_ID") or "" + secret = ( + values.get("ALPACA_SECRET_KEY") + or values.get("APCA_API_SECRET_KEY") + or "" + ) + base = values.get("APCA_API_BASE_URL", "https://paper-api.alpaca.markets") + feed = values.get("ALPACA_DATA_FEED", "iex") + return cls( + api_key=api_key, + secret_key=secret, + trading_base_url=base.rstrip("/"), + feed=feed, + ) + + @property + def is_configured(self) -> bool: + return bool(self.api_key and self.secret_key) diff --git a/alpaca_paper/sync.py b/alpaca_paper/sync.py new file mode 100644 index 0000000..cf3a37a --- /dev/null +++ b/alpaca_paper/sync.py @@ -0,0 +1,135 @@ +"""Sync the Alpaca paper account into the event store. + +Emits account_summary_updated / account_positions_updated / +account_orders_updated events that the dashboard projections read. +""" + +from __future__ import annotations + +import logging +from datetime import datetime + +from alpaca_paper.client import AlpacaPaperClient +from storage.event_schema import ( + AccountOrdersUpdatedEvent, + AccountPositionsUpdatedEvent, + AccountSummaryUpdatedEvent, + EventMode, +) +from storage.event_store import EventStore + +logger = logging.getLogger(__name__) + +BROKER_NAME = "alpaca_paper" + + +class AlpacaPaperSync: + def __init__( + self, + store: EventStore, + client: AlpacaPaperClient | None = None, + session_id: str | None = None, + mode: EventMode = EventMode.PAPER, + ): + self.store = store + self.client = client or AlpacaPaperClient() + self.session_id = session_id + self.mode = mode + + def sync_account(self) -> dict | None: + try: + account = self.client.get_account() + except Exception: + logger.exception("alpaca account sync failed") + return None + account_id = str(account.get("account_number") or account.get("id") or "paper") + self.store.emit( + AccountSummaryUpdatedEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=f"Alpaca paper equity {account.get('equity')}", + broker_name=BROKER_NAME, + account_id=account_id, + account_desc="Alpaca Paper", + total_equity=float(account.get("equity") or 0), + cash_balance=float(account.get("cash") or 0), + buying_power=float(account.get("buying_power") or 0), + net_liquidating_value=float(account.get("equity") or 0), + ) + ) + return account + + def sync_positions(self) -> list[dict] | None: + try: + raw = self.client.get_positions() + except Exception: + logger.exception("alpaca positions sync failed") + return None + positions = [ + { + "symbol": p.get("symbol"), + "quantity": float(p.get("qty") or 0), + "avg_entry_price": float(p.get("avg_entry_price") or 0), + "current_price": float(p.get("current_price") or 0), + "market_value": float(p.get("market_value") or 0), + "unrealized_pnl": float(p.get("unrealized_pl") or 0), + "unrealized_pnl_pct": float(p.get("unrealized_plpc") or 0), + "side": p.get("side"), + } + for p in raw + ] + self.store.emit( + AccountPositionsUpdatedEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=f"Alpaca paper positions: {len(positions)}", + broker_name=BROKER_NAME, + account_id="paper", + positions=positions, + ) + ) + return positions + + def sync_orders(self, status: str = "all", limit: int = 50) -> list[dict] | None: + try: + raw = self.client.get_orders(status=status, limit=limit) + except Exception: + logger.exception("alpaca orders sync failed") + return None + orders = [ + { + "broker_order_id": o.get("id"), + "client_order_id": o.get("client_order_id"), + "symbol": o.get("symbol"), + "side": o.get("side"), + "quantity": float(o.get("qty") or 0), + "filled_quantity": float(o.get("filled_qty") or 0), + "type": o.get("type"), + "status": o.get("status"), + "limit_price": o.get("limit_price"), + "filled_avg_price": o.get("filled_avg_price"), + "submitted_at": o.get("submitted_at"), + } + for o in raw + ] + self.store.emit( + AccountOrdersUpdatedEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=f"Alpaca paper orders: {len(orders)}", + broker_name=BROKER_NAME, + account_id="paper", + orders=orders, + ) + ) + return orders + + def sync_all(self) -> dict: + return { + "account": self.sync_account(), + "positions": self.sync_positions(), + "orders": self.sync_orders(), + } diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/main.py b/api/main.py new file mode 100644 index 0000000..0f0237f --- /dev/null +++ b/api/main.py @@ -0,0 +1,387 @@ +"""Dashboard API (Milestone 5) — stdlib ThreadingHTTPServer, no FastAPI. + +Endpoints +--------- +GET / dashboard HTML +GET /api/snapshots everything the dashboard needs (one shot) +GET /api/stream Server-Sent Events: pushes snapshots on change +GET /api/approval-queue pending approvals only +GET /api/criteria?symbol=XYZ per-criterion pass/fail breakdown +GET /api/bars?symbol=XYZ[&minutes=60] recent 1-min bars for a sparkline +POST /api/trading/approvals/approve {"order_id": ..., "approved_by"?: ...} +POST /api/trading/approvals/reject {"order_id": ..., "reason"?: ...} +POST /api/trading/exit-order {"symbol": ...} +POST /api/watch/add {"symbol": ...} (manual watchlist inject) + +The server holds an EventStore for reads; trade actions route through a +TradingExecutionService when one is attached (the orchestrator does this). +Without one, action endpoints return 503 instead of crashing, so the +dashboard still works read-only. +""" + +from __future__ import annotations + +import json +import threading +import time +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +from storage.event_store import EventStore +from storage.projections import ( + query_account_orders_snapshot, + query_account_positions_snapshot, + query_account_summary_snapshot, + query_approval_queue, + query_fills_feed, + query_order_lifecycle_snapshot, + query_ready_signals_snapshot, + query_session_pnl, + query_session_summary, + query_symbol_criteria, + query_watch_states_snapshot, +) + +STATIC_DIR = Path(__file__).parent / "static" + + +def _eastern_session_date(): + """Today's date in US/Eastern — matches the ingestion pipeline's flags.""" + from zoneinfo import ZoneInfo + + return datetime.now(ZoneInfo("America/New_York")).date() + + +class DashboardState: + """Shared state: event store path + optional live execution service.""" + + def __init__( + self, + event_db_path: str, + execution_service=None, + execution_mode: str = "alpaca_paper", + research_con=None, + watch_inject=None, + ) -> None: + self.event_db_path = event_db_path + self.execution_service = execution_service + self.execution_mode = execution_mode + # optional research DB connection + callback for charts / manual inject + self.research_con = research_con + self.watch_inject = watch_inject + self._lock = threading.Lock() + + def open_store(self) -> EventStore: + return EventStore(self.event_db_path) + + def snapshots(self) -> dict: + with self._lock: + store = self.open_store() + try: + return { + "generated_at": datetime.now(timezone.utc).isoformat(), + "execution_mode": self.execution_mode, + "has_execution": self.execution_service is not None, + "pnl": query_session_pnl(store), + "approval_queue": query_approval_queue(store), + "ready_signals": query_ready_signals_snapshot(store), + "watch_states": query_watch_states_snapshot(store), + "accounts": query_account_summary_snapshot(store), + "positions": query_account_positions_snapshot(store), + "orders": query_account_orders_snapshot(store), + "order_lifecycle": query_order_lifecycle_snapshot(store), + "fills": query_fills_feed(store, limit=30), + "sessions": query_session_summary(store), + } + finally: + store.close() + + def event_count(self) -> int: + with self._lock: + store = self.open_store() + try: + return store.count_events() + finally: + store.close() + + def approval_queue(self) -> list[dict]: + with self._lock: + store = self.open_store() + try: + return query_approval_queue(store) + finally: + store.close() + + def criteria(self, symbol: str) -> dict: + with self._lock: + store = self.open_store() + try: + return query_symbol_criteria(store, symbol) + finally: + store.close() + + def bars(self, symbol: str, minutes: int = 60) -> dict: + """Recent 1-min bars for a sparkline, from the research DB if present. + + Uses the current US/Eastern session date (matching the ingestion + pipeline). If today has no bars yet (pre-market, weekend, or replay of + an older session), falls back to the most recent session that does. + """ + if self.research_con is None: + return {"symbol": symbol, "bars": [], "reason": "no research db attached"} + from research.query import query_minute_bars + + with self._lock: + try: + session_date = _eastern_session_date() + df = query_minute_bars(self.research_con, symbol, session_date) + if df is None or df.empty: + # fall back to the latest session with bars for this symbol + row = self.research_con.execute( + "SELECT MAX(session_date) FROM minute_bars WHERE symbol = ?", + [symbol], + ).fetchone() + if row and row[0] is not None: + df = query_minute_bars(self.research_con, symbol, row[0]) + except Exception as exc: # noqa: BLE001 + return {"symbol": symbol, "bars": [], "reason": str(exc)} + if df is None or df.empty: + return {"symbol": symbol, "bars": []} + tail = df.tail(minutes) + bars = [ + { + "t": str(row["timestamp"]), + "c": float(row["close"]), + "h": float(row["high"]), + "l": float(row["low"]), + "v": int(row["volume"]), + "vwap": float(row["vwap"]) if row.get("vwap") is not None else None, + } + for _, row in tail.iterrows() + ] + return {"symbol": symbol, "bars": bars} + + def _with_service(self, fn): + if self.execution_service is None: + return None + with self._lock: + return fn(self.execution_service) + + def approve(self, order_id: str, approved_by: str, notes: str | None) -> dict | None: + return self._with_service( + lambda svc: svc.approve_order(order_id, approved_by=approved_by, notes=notes) + ) + + def reject(self, order_id: str, rejected_by: str, reason: str) -> dict | None: + return self._with_service( + lambda svc: svc.reject_order(order_id, rejected_by=rejected_by, reason=reason) + ) + + def exit_position(self, symbol: str) -> dict | None: + return self._with_service(lambda svc: svc.submit_exit_order(symbol)) + + def add_watch(self, symbol: str) -> dict | None: + """Manually inject a symbol onto the watchlist (if supported).""" + if not self.watch_inject or not symbol: + return None + try: + self.watch_inject(symbol.upper()) + return {"ok": True, "symbol": symbol.upper()} + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def _json_default(obj): + if isinstance(obj, datetime): + return obj.isoformat() + return str(obj) + + +def _dump(payload) -> bytes: + return json.dumps(payload, default=_json_default).encode() + + +class DashboardHandler(BaseHTTPRequestHandler): + state: DashboardState # injected via make_handler + protocol_version = "HTTP/1.1" + + # -- plumbing ----------------------------------------------------------- + + def log_message(self, fmt, *args): # noqa: A003 - quiet by default + pass + + def _send_json(self, payload, status: int = 200) -> None: + body = _dump(payload) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(body) + + def _send_html(self, path: Path) -> None: + if not path.exists(): + self._send_json({"error": "dashboard.html not found"}, 404) + return + body = path.read_bytes() + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _read_body(self) -> dict: + length = int(self.headers.get("Content-Length") or 0) + if not length: + return {} + try: + return json.loads(self.rfile.read(length).decode() or "{}") + except json.JSONDecodeError: + return {} + + def _query(self) -> dict: + return parse_qs(urlparse(self.path).query) + + # -- SSE ---------------------------------------------------------------- + + def _stream(self) -> None: + """Server-Sent Events: push a fresh snapshot whenever events change. + + Falls back gracefully — a heartbeat every ~15s keeps proxies open even + when nothing is happening, and the loop exits when the client drops or + the server signals shutdown (so the DuckDB handle is released cleanly). + """ + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "keep-alive") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + + shutdown = getattr(type(self), "shutdown_event", None) + last_count = -1 + last_heartbeat = 0.0 + try: + while shutdown is None or not shutdown.is_set(): + count = self.state.event_count() + now = time.monotonic() + if count != last_count: + last_count = count + payload = _dump(self.state.snapshots()).decode() + self.wfile.write(f"event: snapshot\ndata: {payload}\n\n".encode()) + self.wfile.flush() + last_heartbeat = now + elif now - last_heartbeat > 15: + self.wfile.write(b": heartbeat\n\n") + self.wfile.flush() + last_heartbeat = now + # sleep in short slices so shutdown is responsive + for _ in range(10): + if shutdown is not None and shutdown.is_set(): + break + time.sleep(0.1) + except (BrokenPipeError, ConnectionResetError, OSError): + return # client disconnected + + # -- routes --------------------------------------------------------------- + + def do_GET(self): # noqa: N802 + try: + route = urlparse(self.path).path + if route in ("/", "/index.html", "/dashboard"): + self._send_html(STATIC_DIR / "dashboard.html") + elif route == "/api/stream": + self._stream() + elif route == "/api/snapshots": + self._send_json(self.state.snapshots()) + elif route == "/api/approval-queue": + self._send_json({"approval_queue": self.state.approval_queue()}) + elif route == "/api/criteria": + symbol = (self._query().get("symbol") or [""])[0] + self._send_json(self.state.criteria(symbol)) + elif route == "/api/bars": + q = self._query() + symbol = (q.get("symbol") or [""])[0] + minutes = int((q.get("minutes") or ["60"])[0]) + self._send_json(self.state.bars(symbol, minutes)) + elif route == "/api/health": + self._send_json({"ok": True, "mode": self.state.execution_mode}) + else: + self._send_json({"error": "not found"}, 404) + except Exception as exc: # noqa: BLE001 + self._send_json({"error": str(exc)}, 500) + + def do_POST(self): # noqa: N802 + try: + route = urlparse(self.path).path + body = self._read_body() + if route == "/api/trading/approvals/approve": + result = self.state.approve( + str(body.get("order_id") or ""), + approved_by=str(body.get("approved_by") or "dashboard"), + notes=body.get("notes"), + ) + elif route == "/api/trading/approvals/reject": + result = self.state.reject( + str(body.get("order_id") or ""), + rejected_by=str(body.get("rejected_by") or "dashboard"), + reason=str(body.get("reason") or "manual"), + ) + elif route == "/api/trading/exit-order": + result = self.state.exit_position(str(body.get("symbol") or "")) + elif route == "/api/watch/add": + result = self.state.add_watch(str(body.get("symbol") or "")) + else: + self._send_json({"error": "not found"}, 404) + return + if result is None: + self._send_json( + {"ok": False, "error": "not available in read-only mode"}, 503 + ) + else: + self._send_json(result, 200 if result.get("ok") else 400) + except Exception as exc: # noqa: BLE001 + self._send_json({"error": str(exc)}, 500) + + def do_OPTIONS(self): # noqa: N802 + self.send_response(204) + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + self.end_headers() + + +def make_handler(state: DashboardState, shutdown_event=None): + return type( + "BoundDashboardHandler", + (DashboardHandler,), + {"state": state, "shutdown_event": shutdown_event}, + ) + + +def create_server( + state: DashboardState, host: str = "127.0.0.1", port: int = 8010 +) -> ThreadingHTTPServer: + shutdown_event = threading.Event() + server = ThreadingHTTPServer((host, port), make_handler(state, shutdown_event)) + # let SSE loops notice shutdown and release their DuckDB handles cleanly + server._sse_shutdown = shutdown_event # type: ignore[attr-defined] + _orig_shutdown = server.shutdown + + def _shutdown(): + shutdown_event.set() + _orig_shutdown() + + server.shutdown = _shutdown # type: ignore[method-assign] + return server + + +def serve_in_background( + state: DashboardState, host: str = "127.0.0.1", port: int = 8010 +) -> tuple[ThreadingHTTPServer, threading.Thread]: + server = create_server(state, host, port) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread diff --git a/api/static/dashboard.html b/api/static/dashboard.html new file mode 100644 index 0000000..62f9b1c --- /dev/null +++ b/api/static/dashboard.html @@ -0,0 +1,598 @@ + + + + + +Momentum — Trading Control Room + + + +
+

MOMENTUM / control room

+ PAPER +
+ + +
connecting...
+
+ +
+
Total P&L
—
+
Realized
—
+
Unrealized
—
+
Win Rate
—
+
Avg R
—
+
Open / Closed
—
+
+ +
+
+

Account 0

+
No account snapshot yet — waiting for broker sync.
+
+ +
+

Approval Queue 0A approve top · R reject top

+
+
+ +
+

Ready Signals 0

+
+
+ +
+

Open Positions 0

+
+
+ +
+

Watch States 0click a row for criteria

+
+ + +
+
+
+ +
+

Activity Feed 0

+
+
+ +
+

Order Lifecycle 0

+
+
+
+ +
+ +
+
+

Keyboard shortcuts

+
Approve top of queueA
+
Reject top of queueR
+
Toggle alert soundS
+
Focus add-symbol box/
+
Show / hide this help?
+
Close helpEsc
+
+
+ + + + diff --git a/backtest_cli.py b/backtest_cli.py new file mode 100644 index 0000000..4834e83 --- /dev/null +++ b/backtest_cli.py @@ -0,0 +1,68 @@ +"""Run intraday backtests against bars stored in the research database. + + python backtest_cli.py --symbol SNDL --date 2026-06-10 + python backtest_cli.py --date 2026-06-10 # all in-band symbols +""" + +from __future__ import annotations + +import argparse +import sys +from datetime import date + +from research import query as rq +from research.multi_schema import open_research_db +from strategy.backtest.engine import BacktestEngine + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--symbol", help="single symbol (default: scan session)") + parser.add_argument("--date", required=True, help="session date YYYY-MM-DD") + parser.add_argument("--price-min", type=float, default=1.0) + parser.add_argument("--price-max", type=float, default=20.0) + parser.add_argument("--equity", type=float, default=100_000.0) + args = parser.parse_args(argv) + + session_date = date.fromisoformat(args.date) + con = open_research_db("market") + engine = BacktestEngine(equity=args.equity) + + if args.symbol: + symbols = [args.symbol.upper()] + else: + rows = rq.query_session_symbols( + con, session_date, price_min=args.price_min, price_max=args.price_max + ) + symbols = [r["symbol"] for r in rows] + if not symbols: + print(f"no symbols with minute bars on {session_date} in " + f"[{args.price_min}, {args.price_max}] — run fetch_minute_bars.py first") + return 1 + + total_pnl = 0.0 + total_trades = 0 + for symbol in symbols: + bars = rq.query_minute_bars(con, symbol, session_date) + if bars.empty: + print(f"{symbol}: no bars") + continue + result = engine.run( + bars, + symbol, + previous_close=rq.query_previous_close(con, symbol, session_date), + avg_daily_volume=rq.query_avg_daily_volume(con, symbol, session_date), + ) + s = result.summary() + total_pnl += s["total_pnl"] + total_trades += s["trades"] + print(f"{symbol:>6} trades={s['trades']:<3} win_rate={s['win_rate']:<6} " + f"pnl={s['total_pnl']:>9.2f} signals={s['signals']} evals={s['evaluations']}") + + print(f"\nTOTAL trades={total_trades} pnl={total_pnl:.2f}") + con.close() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/comprehensive_backtest.py b/comprehensive_backtest.py new file mode 100644 index 0000000..be109b0 --- /dev/null +++ b/comprehensive_backtest.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Multi-week comprehensive backtest with detailed statistical analysis.""" + +from datetime import datetime, timedelta, date +import statistics + +import pandas as pd +from rich.console import Console +from rich.table import Table + +from config import Config +from research.connection import ResearchConnection +from research.query import query_minute_bars +from strategy.backtest.engine import BacktestEngine + +console = Console() + +def get_trading_days(start: date, end: date) -> list[date]: + """Get all weekdays between start and end (trading days only).""" + days = [] + current = start + while current <= end: + # Only include weekdays (0=Monday, 6=Sunday) + if current.weekday() < 5: + days.append(current) + current += timedelta(days=1) + return days + + +def run_backtest_multi_day(start_date: date, end_date: date, symbols: list[str] | None = None): + """Run backtest across multiple days and aggregate results.""" + + config = Config() + con = ResearchConnection() + + if symbols is None: + symbols = ["SNDL", "MARA", "RIOT", "BBAI", "SOUN", "PLUG"] + + console.print(f"\n[bold blue]Comprehensive Backtest: {start_date} to {end_date}[/bold blue]") + console.print(f"[dim]Testing {len(symbols)} symbols across trading days[/dim]\n") + + trading_days = get_trading_days(start_date, end_date) + console.print(f"[cyan]{len(trading_days)} trading days in range[/cyan]\n") + + # Aggregate results + all_trades: list[dict] = [] + daily_results: dict[date, dict] = {} + symbol_stats: dict[str, dict] = {} + + engine = BacktestEngine(config) + + # Progress bar + processed = 0 + for trading_day in trading_days: + # Query minute bars for all symbols that day + day_symbols_with_data = [] + day_bars = {} + + for symbol in symbols: + try: + bars = query_minute_bars(con, symbol, trading_day) + if len(bars) > 0: + day_symbols_with_data.append(symbol) + day_bars[symbol] = bars + except Exception as e: + pass # No data for this day + + if not day_symbols_with_data: + processed += 1 + continue + + # Initialize daily results if needed + if trading_day not in daily_results: + daily_results[trading_day] = {"trades": [], "pnl": 0.0} + + # Run backtest for each symbol with data + for symbol in day_symbols_with_data: + bars = day_bars[symbol] + + try: + result = engine.run(bars, symbol) + + # Aggregate + daily_results[trading_day]["pnl"] += result.total_pnl + + # Initialize symbol stats if needed + if symbol not in symbol_stats: + symbol_stats[symbol] = {"trades": [], "wins": 0, "losses": 0, "pnl": 0.0} + + for trade in result.trades: + trade_data = { + "date": trading_day.isoformat(), + "symbol": symbol, + "entry_time": trade.entry_time, + "entry_price": trade.entry_price, + "stop_price": trade.stop_price, + "target_price": trade.target_price, + "quantity": trade.quantity, + "exit_price": trade.exit_price, + "exit_reason": trade.exit_reason, + "pnl": trade.realized_pnl or 0.0, + "r_multiple": trade.r_multiple or 0.0, + } + all_trades.append(trade_data) + daily_results[trading_day]["trades"].append(trade_data) + + # Symbol stats + symbol_stats[symbol]["trades"].append(trade_data) + symbol_stats[symbol]["pnl"] += trade.realized_pnl or 0.0 + if (trade.realized_pnl or 0) > 0: + symbol_stats[symbol]["wins"] += 1 + else: + symbol_stats[symbol]["losses"] += 1 + except Exception as e: + console.print(f"[red]Error backtesting {symbol} on {trading_day}: {e}[/red]") + + processed += 1 + console.print(f"[dim]Processed {processed}/{len(trading_days)} days[/dim]", end="\r") + + console.print() + + # === SUMMARY STATISTICS === + if not all_trades: + console.print("[yellow]No trades found in backtest range[/yellow]") + return + + # Daily summary + console.print("\n[bold cyan]=== DAILY SUMMARY ===[/bold cyan]") + daily_table = Table(title="P&L by Day") + daily_table.add_column("Date", style="cyan") + daily_table.add_column("Trades", justify="right") + daily_table.add_column("P&L", justify="right") + + daily_pnls = [] + for day in sorted(daily_results.keys()): + daily_pnl = daily_results[day]["pnl"] + daily_pnls.append(daily_pnl) + color = "green" if daily_pnl > 0 else "red" + daily_table.add_row( + str(day), + str(len(daily_results[day]["trades"])), + f"[{color}]${daily_pnl:,.2f}[/{color}]" + ) + console.print(daily_table) + + # Symbol summary + console.print("\n[bold cyan]=== SYMBOL SUMMARY ===[/bold cyan]") + symbol_table = Table(title="Results by Symbol") + symbol_table.add_column("Symbol", style="cyan") + symbol_table.add_column("Trades", justify="right") + symbol_table.add_column("Wins", justify="right") + symbol_table.add_column("Win %", justify="right") + symbol_table.add_column("P&L", justify="right") + symbol_table.add_column("Avg R", justify="right") + + for symbol in sorted(symbol_stats.keys()): + stats = symbol_stats[symbol] + trades = stats["trades"] + if not trades: + continue + + win_pct = (stats["wins"] / len(trades)) * 100 if trades else 0 + avg_r = statistics.mean([t["r_multiple"] for t in trades]) if trades else 0 + + color = "green" if stats["pnl"] > 0 else "red" + symbol_table.add_row( + symbol, + str(len(trades)), + str(stats["wins"]), + f"{win_pct:.1f}%", + f"[{color}]${stats['pnl']:,.2f}[/{color}]", + f"{avg_r:.2f}R" + ) + console.print(symbol_table) + + # Overall stats + total_trades = len(all_trades) + total_pnl = sum(t["pnl"] for t in all_trades) + wins = len([t for t in all_trades if t["pnl"] > 0]) + losses = len([t for t in all_trades if t["pnl"] < 0]) + win_rate = (wins / total_trades) * 100 if total_trades else 0 + + r_multiples = [t["r_multiple"] for t in all_trades] + avg_r = statistics.mean(r_multiples) if r_multiples else 0 + + # Separate wins and losses for analysis + win_trades = [t for t in all_trades if t["pnl"] > 0] + loss_trades = [t for t in all_trades if t["pnl"] < 0] + + avg_win = statistics.mean([t["pnl"] for t in win_trades]) if win_trades else 0 + avg_loss = statistics.mean([t["pnl"] for t in loss_trades]) if loss_trades else 0 + max_loss = min([t["pnl"] for t in loss_trades]) if loss_trades else 0 + + profit_factor = (sum([t["pnl"] for t in win_trades]) / abs(sum([t["pnl"] for t in loss_trades]))) if loss_trades and sum([t["pnl"] for t in loss_trades]) < 0 else 0 + + console.print("\n[bold cyan]=== OVERALL STATISTICS ===[/bold cyan]") + console.print(f"Total Trades: {total_trades}") + console.print(f"Wins: {wins} | Losses: {losses}") + console.print(f"Win Rate: {win_rate:.1f}%") + console.print(f"Total P&L: [{'green' if total_pnl > 0 else 'red'}]${total_pnl:,.2f}[/{'green' if total_pnl > 0 else 'red'}]") + console.print(f"Avg Win: ${avg_win:,.2f} | Avg Loss: ${avg_loss:,.2f}") + console.print(f"Max Loss: ${max_loss:,.2f}") + if profit_factor > 0: + console.print(f"Profit Factor: {profit_factor:.2f}x") + console.print(f"Avg R-Multiple: {avg_r:.2f}R") + + # === LOSS ANALYSIS === + if loss_trades: + console.print("\n[bold yellow]=== LOSS ANALYSIS ===[/bold yellow]") + console.print(f"Total Losing Trades: {len(loss_trades)}") + + loss_table = Table(title="Top 10 Losses (for pattern analysis)") + loss_table.add_column("Date", style="cyan") + loss_table.add_column("Symbol", style="cyan") + loss_table.add_column("Entry", justify="right") + loss_table.add_column("Stop", justify="right") + loss_table.add_column("Exit", justify="right") + loss_table.add_column("P&L", justify="right") + loss_table.add_column("Reason") + + sorted_losses = sorted(loss_trades, key=lambda t: t["pnl"]) + for trade in sorted_losses[:10]: + loss_table.add_row( + trade["date"], + trade["symbol"], + f"${trade['entry_price']:.2f}", + f"${trade['stop_price']:.2f}", + f"${trade['exit_price']:.2f}" if trade["exit_price"] else "-", + f"[red]${trade['pnl']:,.2f}[/red]", + trade["exit_reason"] or "?" + ) + console.print(loss_table) + + # Loss metrics + stop_hits = len([t for t in loss_trades if "stop" in (t["exit_reason"] or "").lower()]) + console.print(f"\n[dim]Losses by reason:[/dim]") + console.print(f" Hit stop: {stop_hits}/{len(loss_trades)}") + + # Gap-through analysis + for trade in sorted_losses[:5]: + entry = trade["entry_price"] + stop = trade["stop_price"] + exit_price = trade["exit_price"] + if exit_price and exit_price < stop: + gap_pct = ((stop - exit_price) / stop) * 100 + console.print(f" [yellow]{trade['symbol']} {trade['date']}: gap-through {gap_pct:.2f}% ({stop:.2f} → {exit_price:.2f})[/yellow]") + + # === WIN ANALYSIS === + if win_trades: + console.print("\n[bold green]=== WIN ANALYSIS ===[/bold green]") + console.print(f"Total Winning Trades: {len(win_trades)}") + + win_table = Table(title="Recent Wins") + win_table.add_column("Date", style="cyan") + win_table.add_column("Symbol", style="cyan") + win_table.add_column("Entry", justify="right") + win_table.add_column("Target", justify="right") + win_table.add_column("Exit", justify="right") + win_table.add_column("P&L", justify="right") + win_table.add_column("R") + + sorted_wins = sorted(win_trades, key=lambda t: t["pnl"], reverse=True) + for trade in sorted_wins[:5]: + win_table.add_row( + trade["date"], + trade["symbol"], + f"${trade['entry_price']:.2f}", + f"${trade['target_price']:.2f}", + f"${trade['exit_price']:.2f}" if trade["exit_price"] else "-", + f"[green]${trade['pnl']:,.2f}[/green]", + f"{trade['r_multiple']:.2f}R" + ) + console.print(win_table) + + # === RISK ASSESSMENT === + console.print("\n[bold yellow]=== RISK ASSESSMENT ===[/bold yellow]") + + if avg_loss != 0: + expectancy = (win_rate / 100 * avg_win) + ((1 - win_rate / 100) * avg_loss) + console.print(f"Expected value per trade: ${expectancy:,.2f}") + + if profit_factor > 0: + if profit_factor < 1.5: + console.print("[yellow]⚠️ Profit factor < 1.5 indicates marginal edge or slippage issues[/yellow]") + else: + console.print("[green]✓ Profit factor suggests adequate win/loss ratio[/green]") + + # Check for consistent losses + max_win = max([t["pnl"] for t in win_trades]) if win_trades else 0 + if max_loss < 0 and abs(max_loss) > max_win * 1.5: + console.print(f"[red]⚠️ Max loss (${max_loss:,.2f}) >> max win (${max_win:,.2f}) — stop is too wide or slippage is severe[/red]") + + # Sharpe-ish ratio (simplified) + if len(daily_pnls) > 1: + daily_std = statistics.stdev(daily_pnls) if len(daily_pnls) > 1 else 0 + if daily_std > 0: + daily_mean = statistics.mean(daily_pnls) + sharpe_approx = daily_mean / daily_std if daily_std > 0 else 0 + console.print(f"Daily Sharpe (approx): {sharpe_approx:.2f}") + + console.print("\n[dim]Recommendations:[/dim]") + if win_rate < 40: + console.print("[red]• Win rate < 40% — strategy may lack edge even with profitable trades[/red]") + if profit_factor < 1.3 and profit_factor > 0: + console.print("[yellow]• Profit factor < 1.3 — consider tightening stops or improving entry[/yellow]") + if len(loss_trades) > 0 and max_loss < avg_loss / 2: + console.print("[yellow]• Losses are inconsistent — gap-throughs may be a factor[/yellow]") + + console.print() + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Multi-week comprehensive backtest") + parser.add_argument("--start", type=str, default="2026-06-05", help="Start date (YYYY-MM-DD)") + parser.add_argument("--end", type=str, default="2026-06-12", help="End date (YYYY-MM-DD)") + parser.add_argument("--symbols", type=str, default="SNDL,MARA,RIOT,BBAI,SOUN,PLUG", help="Comma-separated symbols") + + args = parser.parse_args() + + start = datetime.strptime(args.start, "%Y-%m-%d").date() + end = datetime.strptime(args.end, "%Y-%m-%d").date() + symbols = args.symbols.split(",") + + run_backtest_multi_day(start, end, symbols) diff --git a/config.py b/config.py new file mode 100644 index 0000000..3a1d506 --- /dev/null +++ b/config.py @@ -0,0 +1,191 @@ +"""Configuration loading from YAML files.""" + +from pathlib import Path +from typing import Any + +import yaml +from pydantic import BaseModel, ConfigDict + + +class SlippageConfig(BaseModel): + base_spread_pct: float = 0.0015 + volatility_adder_pct: float = 0.0005 + atr_volatility_threshold: float = 2.0 + + +class CommissionConfig(BaseModel): + per_share: float = 0.005 + minimum: float = 1.00 + + +class BacktestConfig(BaseModel): + slippage: SlippageConfig = SlippageConfig() + commission: CommissionConfig = CommissionConfig() + entry_mode: str = "next_bar" + + +class ScannerWeights(BaseModel): + gap_strength: float = 0.30 + relative_volume: float = 0.25 + premarket_participation: float = 0.20 + range_expansion: float = 0.15 + catalyst_freshness: float = 0.10 + + +class ScannerConfig(BaseModel): + gap_pct_min: float = 0.05 + premarket_volume_min: int = 250_000 + relative_volume_min: float = 2.5 + spread_pct_max: float = 0.05 + quality_score_min: float = 0.7 + weights: ScannerWeights = ScannerWeights() + + +class SetupConfig(BaseModel): + impulse_size_min: float = 0.05 + max_pullback_depth_pct: float = 0.40 + breakout_volume_ratio_min: float = 1.5 + pullback_volume_ratio_max: float = 0.6 + timeframes: list[int] = [1, 5] + + +class RiskConfig(BaseModel): + risk_per_trade_pct: float = 0.01 + max_daily_loss_pct: float = 0.03 + max_concurrent_positions: int = 3 + no_new_entries_after: str = "10:30" + no_averaging_down: bool = True + + +class UniverseConfig(BaseModel): + price_min: float = 1.0 + price_max: float = 20.0 + min_avg_volume_20d: int = 500_000 + exclude_etf: bool = True + exclude_otc: bool = True + common_shares_only: bool = True + + +class QualityConfig(BaseModel): + min_regular_hours_coverage: float = 0.95 + min_premarket_coverage: float = 0.80 + max_price_gap_pct: float = 0.20 + min_score: float = 0.7 + + +class DataConfig(BaseModel): + dir: str = "./data" + db_path: str = "./data/momentum.duckdb" + + +class TelemetryConfig(BaseModel): + heartbeat_interval_seconds: int = 60 + log_dir: str = "./data/logs" + + +class ProviderQuotaConfig(BaseModel): + """Per-provider API quota and rate limit settings.""" + + model_config = ConfigDict(extra="ignore") + + fmp_daily_limit: int = 250 + fmp_intraday_limit: int = 100 + twelve_data_daily_limit: int = 800 + twelve_data_intraday_limit: int = 120 + alpha_vantage_daily_limit: int = 5 + newsapi_daily_limit: int = 100 + newsdata_daily_limit: int = 200 + marketaux_daily_limit: int = 100 + mediastack_daily_limit: int = 100 + + +class RSSFeedConfig(BaseModel): + """RSS feed source definitions.""" + + model_config = ConfigDict(extra="ignore") + + feeds: list[str] = [ + "https://feeds.bloomberg.com/markets/news.rss", + "https://feeds.marketwatch.com/marketwatch/topstories/", + "https://www.cnbc.com/id/100003114/device/rss/rss.html", + "https://finance.yahoo.com/news/rssindex", + "https://seekingalpha.com/feed.xml", + ] + + +class NewsSourcesConfig(BaseModel): + """News source configuration and cadence.""" + + model_config = ConfigDict(extra="ignore") + + finnhub_enabled: bool = True + finnhub_refresh_interval_seconds: int = 300 + rss_enabled: bool = True + rss_refresh_interval_seconds: int = 600 + sec_enabled: bool = True + sec_refresh_interval_seconds: int = 3600 + fed_enabled: bool = True + fed_refresh_interval_seconds: int = 3600 + newsapi_enabled: bool = False + newsapi_refresh_interval_seconds: int = 600 + newsdata_enabled: bool = False + newsdata_refresh_interval_seconds: int = 600 + marketaux_enabled: bool = False + marketaux_refresh_interval_seconds: int = 600 + mediastack_enabled: bool = False + mediastack_refresh_interval_seconds: int = 600 + source_cooldown_default_seconds: int = 180 + provider_source_cooldown_seconds: int = 180 + rss_source_cooldown_seconds: int = 180 + regulatory_source_cooldown_seconds: int = 180 + rss_feeds: RSSFeedConfig = RSSFeedConfig() + + +class OllamaConfig(BaseModel): + """Ollama local LLM settings for enrichment.""" + + model_config = ConfigDict(extra="ignore") + + enabled: bool = False + host: str = "http://localhost:11434" + model: str = "mistral" + timeout_seconds: int = 30 + max_tokens: int = 256 + temperature: float = 0.3 + + +class CapabilityRegistryConfig(BaseModel): + """Capability registry configuration.""" + + model_config = ConfigDict(extra="ignore") + + enabled: bool = True + + +class Config(BaseModel): + model_config = ConfigDict(extra="ignore") + + data: DataConfig = DataConfig() + universe: UniverseConfig = UniverseConfig() + scanner: ScannerConfig = ScannerConfig() + setup: SetupConfig = SetupConfig() + risk: RiskConfig = RiskConfig() + backtest: BacktestConfig = BacktestConfig() + quality: QualityConfig = QualityConfig() + telemetry: TelemetryConfig = TelemetryConfig() + providers: ProviderQuotaConfig = ProviderQuotaConfig() + news_sources: NewsSourcesConfig = NewsSourcesConfig() + ollama: OllamaConfig = OllamaConfig() + capability_registry: CapabilityRegistryConfig = CapabilityRegistryConfig() + + +def load_config(path: str | Path | None = None) -> Config: + """Load config from YAML file, falling back to defaults.""" + if path is None: + path = Path("configs/base.yaml") + path = Path(path) + if path.exists(): + with open(path) as f: + raw: dict[str, Any] = yaml.safe_load(f) or {} + return Config(**raw) + return Config() diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/contracts.py b/core/contracts.py new file mode 100644 index 0000000..d8ce52e --- /dev/null +++ b/core/contracts.py @@ -0,0 +1,259 @@ +"""Module contracts and interfaces for the momentum trading system. + +Every module must implement the ModuleContract protocol, defining: +- input_schema: What the module expects +- output_schema: What the module produces +- run(): Execute the module +- validate_input(): Ensure data meets contracts +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime, date +from typing import Any, Protocol, TypeVar + +T_Input = TypeVar("T_Input") +T_Output = TypeVar("T_Output") + + +@dataclass +class ModuleMetrics: + """Standard metrics every module should track.""" + + module_name: str + timestamp: datetime + duration_ms: float + input_count: int = 0 + output_count: int = 0 + error_count: int = 0 + warnings: list[str] = field(default_factory=list) + custom_metrics: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict: + return { + "module_name": self.module_name, + "timestamp": self.timestamp.isoformat(), + "duration_ms": self.duration_ms, + "input_count": self.input_count, + "output_count": self.output_count, + "error_count": self.error_count, + "warnings": self.warnings, + "custom_metrics": self.custom_metrics, + } + + +@dataclass +class ValidationFailure: + """Single validation failure with context.""" + + field: str + value: Any + reason: str + severity: str = "error" # "error" or "warning" + + +@dataclass +class ModuleResult: + """Base class for all module results.""" + + success: bool + timestamp: datetime + metrics: ModuleMetrics + validation_failures: list[ValidationFailure] = field(default_factory=list) + + def has_errors(self) -> bool: + return bool(self.validation_failures or not self.success) + + def error_summary(self) -> str: + """Human-readable error summary.""" + if not self.validation_failures: + return "No errors" + errors = [f for f in self.validation_failures if f.severity == "error"] + if not errors: + return f"{len(self.validation_failures)} warnings only" + return f"{len(errors)} error(s), {len([f for f in self.validation_failures if f.severity == 'warning'])} warning(s)" + + +class ModuleContract(Protocol): + """Contract every module must fulfill.""" + + def run(self, input: Any) -> ModuleResult: + """Execute the module. + + Args: + input: Module-specific input conforming to input_schema + + Returns: + Module-specific result inheriting from ModuleResult + """ + ... + + def validate_input(self, input: Any) -> tuple[bool, list[ValidationFailure]]: + """Validate input before processing. + + Args: + input: Data to validate + + Returns: + (is_valid, failures) + """ + ... + + def get_metrics(self) -> ModuleMetrics: + """Get last execution metrics.""" + ... + + +# ============================================================================ +# INGESTION MODULE CONTRACTS +# ============================================================================ + + +@dataclass +class BarIngestRequest: + """Input contract for ingestion module.""" + + symbols: list[str] + session_date: date + lookback_minutes: int = 120 # how many minutes of history to fetch + source: str = "alpaca" # "alpaca", "test", "file" + + +@dataclass +class BarIngestResult(ModuleResult): + """Output contract for ingestion module.""" + + minute_rows_inserted: int = 0 + symbols_updated: list[str] = field(default_factory=list) + source_health: dict[str, str] = field(default_factory=dict) # {source: "healthy"|"error"} + + +# ============================================================================ +# RESEARCH MODULE CONTRACTS +# ============================================================================ + + +@dataclass +class ResearchRequest: + """Input contract for research module.""" + + symbols: list[str] + session_date: date + lookback_days: int = 30 # for daily history + + +@dataclass +class ResearchResult(ModuleResult): + """Output contract for research module.""" + + symbols_researched: list[str] = field(default_factory=list) + symbols_with_data: list[str] = field(default_factory=list) + symbols_missing_data: list[str] = field(default_factory=list) + total_bars_available: int = 0 + + +# ============================================================================ +# EVALUATION MODULE CONTRACTS +# ============================================================================ + + +@dataclass +class CriterionDetail: + """Result of evaluating one criterion.""" + + name: str + passed: bool + value: float # actual metric value + threshold: float | None = None # minimum to pass + reason: str = "" + confidence: float = 1.0 # 0-1, how confident in this result + + +@dataclass +class EvaluationRequest: + """Input contract for evaluation module.""" + + symbol: str + bars_df: Any # pd.DataFrame with {timestamp, open, high, low, close, volume} + previous_close: float + avg_daily_volume: float + session_date: date + + +@dataclass +class EvaluationResult(ModuleResult): + """Output contract for evaluation module.""" + + symbol: str = "" + status: str = "late" # "ready", "blocked", "late", "error" + score: float = 0.0 # 0-100 + criteria: list[CriterionDetail] = field(default_factory=list) + entry_price: float | None = None + stop_loss: float | None = None + target_price: float | None = None + quality_score: float = 0.0 # 0-1 + blocking_reason: str | None = None + metrics_dict: dict[str, float] = field(default_factory=dict) # gap%, rvol, etc. + + +# ============================================================================ +# EXECUTION MODULE CONTRACTS +# ============================================================================ + + +@dataclass +class ExecutionRequest: + """Input contract for execution module.""" + + symbol: str + entry_price: float + stop_price: float + target_price: float + score: float + criteria_count: int + account_equity: float + current_positions: int + today_realized_pnl: float + + +@dataclass +class ExecutionResult(ModuleResult): + """Output contract for execution module.""" + + symbol: str = "" + status: str = "" # "approved", "rejected", "pending_approval", "error" + reason: str = "" # why it was approved/rejected + order_id: str | None = None + shares: int | None = None + risk_dollars: float | None = None + risk_checks_applied: list[str] = field(default_factory=list) + + +# ============================================================================ +# BROKER MODULE CONTRACTS +# ============================================================================ + + +@dataclass +class BrokerRequest: + """Input contract for broker module.""" + + symbol: str + order_id: str + entry_price: float + stop_price: float + target_price: float + shares: int + order_type: str = "limit" # "limit" or "market" + + +@dataclass +class BrokerResult(ModuleResult): + """Output contract for broker module.""" + + symbol: str = "" + status: str = "" # "submitted", "rejected", "error" + broker_order_id: str | None = None + submitted_at: datetime | None = None diff --git a/core/telemetry.py b/core/telemetry.py new file mode 100644 index 0000000..3ef5382 --- /dev/null +++ b/core/telemetry.py @@ -0,0 +1,274 @@ +"""Telemetry collection and event emission for module observability.""" + +from __future__ import annotations + +import time +import logging +from dataclasses import dataclass, field +from datetime import datetime +from typing import TYPE_CHECKING, Any, Callable + +if TYPE_CHECKING: + from core.contracts import ModuleMetrics + +logger = logging.getLogger(__name__) + + +@dataclass +class TelemetryCollector: + """Collect metrics during module execution.""" + + module_name: str + start_time: float = field(default_factory=time.time) + input_count: int = 0 + output_count: int = 0 + error_count: int = 0 + warnings: list[str] = field(default_factory=list) + custom_metrics: dict[str, Any] = field(default_factory=dict) + + def record_input(self, count: int = 1) -> None: + """Record input items processed.""" + self.input_count += count + + def record_output(self, count: int = 1) -> None: + """Record output items produced.""" + self.output_count += count + + def record_error(self) -> None: + """Record an error occurred.""" + self.error_count += 1 + + def record_warning(self, message: str) -> None: + """Record a warning.""" + self.warnings.append(message) + + def record_metric(self, name: str, value: float | int | str) -> None: + """Record a custom metric.""" + self.custom_metrics[name] = value + + def finalize(self) -> ModuleMetrics: + """Finalize and return metrics.""" + duration_ms = (time.time() - self.start_time) * 1000 + return ModuleMetrics( + module_name=self.module_name, + timestamp=datetime.now(), + duration_ms=duration_ms, + input_count=self.input_count, + output_count=self.output_count, + error_count=self.error_count, + warnings=self.warnings, + custom_metrics=self.custom_metrics, + ) + + +class TelemetryEmitter: + """Emit telemetry events to event store and observability systems.""" + + def __init__(self, event_store=None): + """Initialize emitter. + + Args: + event_store: Optional EventStore for persisting telemetry events + """ + self.event_store = event_store + self.callbacks: list[Callable[[dict], None]] = [] + + def add_callback(self, callback: Callable[[dict], None]) -> None: + """Add a callback to receive telemetry events.""" + self.callbacks.append(callback) + + def emit_module_tick( + self, + module_name: str, + stage: str, + metrics: ModuleMetrics | None = None, + errors: list[dict] | None = None, + correlation_id: str | None = None, + ) -> None: + """Emit a module tick event. + + Args: + module_name: Name of module (e.g., "ingestion", "evaluation") + stage: Stage name (e.g., "started", "processing", "completed") + metrics: Optional ModuleMetrics object + errors: Optional list of error details + correlation_id: Correlation ID linking to signal_ready event + """ + event = { + "type": "module_tick", + "timestamp": datetime.now().isoformat(), + "module": module_name, + "stage": stage, + "metrics": metrics.to_dict() if metrics else None, + "errors": errors or [], + "correlation_id": correlation_id, + } + self._emit(event) + + def emit_data_validation( + self, + module_name: str, + validation_type: str, + valid: bool, + details: dict, + correlation_id: str | None = None, + ) -> None: + """Emit a data validation event. + + Args: + module_name: Name of module performing validation + validation_type: Type of validation ("schema", "checksum", "business_rules") + valid: Whether validation passed + details: Validation details (errors, warnings, statistics) + correlation_id: Optional correlation ID + """ + event = { + "type": "data_validation", + "timestamp": datetime.now().isoformat(), + "module": module_name, + "validation_type": validation_type, + "valid": valid, + "details": details, + "correlation_id": correlation_id, + } + self._emit(event) + + def emit_symbol_evaluation_detail( + self, + symbol: str, + stage: str, + metrics: dict[str, Any], + passed: bool, + blocking_reason: str | None = None, + correlation_id: str | None = None, + ) -> None: + """Emit detailed evaluation event for a symbol. + + Args: + symbol: Symbol being evaluated + stage: Evaluation stage (e.g., "bars_loaded", "gap_checked", "rvol_calculated") + metrics: Metrics at this stage + passed: Whether this stage passed + blocking_reason: If not passed, why + correlation_id: Optional correlation ID + """ + event = { + "type": "symbol_evaluation_detail", + "timestamp": datetime.now().isoformat(), + "symbol": symbol, + "stage": stage, + "metrics": metrics, + "passed": passed, + "blocking_reason": blocking_reason, + "correlation_id": correlation_id, + } + self._emit(event) + + def _emit(self, event: dict) -> None: + """Emit event to all registered callbacks and event store.""" + logger.debug(f"Telemetry: {event['type']} - {event.get('module', event.get('symbol', 'n/a'))}") + + # Call all registered callbacks + for callback in self.callbacks: + try: + callback(event) + except Exception as e: + logger.error(f"Telemetry callback failed: {e}") + + # Emit to event store if available + if self.event_store: + try: + # Import here to avoid circular dependency + from storage.event_schema import ModuleTickEvent, DataValidationEvent + + if event["type"] == "module_tick": + self.event_store.emit( + ModuleTickEvent( + timestamp=datetime.now(), + module=event["module"], + stage=event["stage"], + duration_ms=event["metrics"]["duration_ms"] + if event["metrics"] + else 0, + input_count=event["metrics"]["input_count"] + if event["metrics"] + else 0, + output_count=event["metrics"]["output_count"] + if event["metrics"] + else 0, + metrics=event["metrics"]["custom_metrics"] + if event["metrics"] + else {}, + errors=event["errors"], + correlation_id=event.get("correlation_id"), + ) + ) + elif event["type"] == "data_validation": + self.event_store.emit( + DataValidationEvent( + timestamp=datetime.now(), + module=event["module"], + validation_type=event["validation_type"], + valid=event["valid"], + details=event["details"], + correlation_id=event.get("correlation_id"), + ) + ) + except Exception as e: + logger.error(f"Failed to emit telemetry event to store: {e}") + + +# Global instance (can be overridden in tests) +_telemetry_emitter: TelemetryEmitter | None = None + + +def set_telemetry_emitter(emitter: TelemetryEmitter) -> None: + """Set the global telemetry emitter.""" + global _telemetry_emitter + _telemetry_emitter = emitter + + +def get_telemetry_emitter() -> TelemetryEmitter: + """Get the global telemetry emitter.""" + global _telemetry_emitter + if _telemetry_emitter is None: + _telemetry_emitter = TelemetryEmitter() + return _telemetry_emitter + + +def emit_module_tick( + module_name: str, + stage: str, + metrics: ModuleMetrics | None = None, + errors: list[dict] | None = None, + correlation_id: str | None = None, +) -> None: + """Convenience function to emit module tick.""" + get_telemetry_emitter().emit_module_tick(module_name, stage, metrics, errors, correlation_id) + + +def emit_data_validation( + module_name: str, + validation_type: str, + valid: bool, + details: dict, + correlation_id: str | None = None, +) -> None: + """Convenience function to emit data validation.""" + get_telemetry_emitter().emit_data_validation( + module_name, validation_type, valid, details, correlation_id + ) + + +def emit_symbol_evaluation_detail( + symbol: str, + stage: str, + metrics: dict[str, Any], + passed: bool, + blocking_reason: str | None = None, + correlation_id: str | None = None, +) -> None: + """Convenience function to emit symbol evaluation detail.""" + get_telemetry_emitter().emit_symbol_evaluation_detail( + symbol, stage, metrics, passed, blocking_reason, correlation_id + ) diff --git a/core/validation.py b/core/validation.py new file mode 100644 index 0000000..8422917 --- /dev/null +++ b/core/validation.py @@ -0,0 +1,284 @@ +"""Data validation utilities for module boundaries.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable + +if TYPE_CHECKING: + from core.contracts import ValidationFailure + + +@dataclass +class SchemaValidator: + """Validate data against a schema.""" + + required_fields: dict[str, type] # {field_name: expected_type} + optional_fields: dict[str, type] | None = None + + def __post_init__(self): + if self.optional_fields is None: + self.optional_fields = {} + + def validate(self, data: dict) -> tuple[bool, list[dict]]: + """Validate data against schema. + + Args: + data: Dictionary to validate + + Returns: + (is_valid, list of failure dicts) + """ + failures = [] + + # Check required fields + for field_name, expected_type in self.required_fields.items(): + if field_name not in data: + failures.append( + { + "field": field_name, + "value": None, + "reason": "Required field missing", + "severity": "error", + } + ) + elif not isinstance(data[field_name], expected_type): + failures.append( + { + "field": field_name, + "value": data[field_name], + "reason": f"Expected {expected_type.__name__}, got {type(data[field_name]).__name__}", + "severity": "error", + } + ) + + # Check optional fields if present + if self.optional_fields: + for field_name, expected_type in self.optional_fields.items(): + if field_name in data and not isinstance(data[field_name], expected_type): + failures.append( + { + "field": field_name, + "value": data[field_name], + "reason": f"Expected {expected_type.__name__}, got {type(data[field_name]).__name__}", + "severity": "warning", + } + ) + + return len([f for f in failures if f.severity == "error"]) == 0, failures + + +class RangeValidator: + """Validate numeric values are within expected ranges.""" + + def __init__(self, min_val: float | None = None, max_val: float | None = None): + self.min_val = min_val + self.max_val = max_val + + def validate(self, value: float, field_name: str = "value") -> tuple[bool, dict | None]: + """Validate value is in range. + + Args: + value: Numeric value to validate + field_name: Name of field for error message + + Returns: + (is_valid, failure dict or None) + """ + if self.min_val is not None and value < self.min_val: + return False, { + "field": field_name, + "value": value, + "reason": f"Value {value} below minimum {self.min_val}", + "severity": "error", + } + + if self.max_val is not None and value > self.max_val: + return False, { + "field": field_name, + "value": value, + "reason": f"Value {value} above maximum {self.max_val}", + "severity": "error", + } + + return True, None + + +class ChecksumValidator: + """Validate data integrity using checksums.""" + + @staticmethod + def compute_checksum(data: dict | list) -> str: + """Compute simple checksum for data. + + Args: + data: Dictionary or list to checksum + + Returns: + Hex checksum string + """ + import hashlib + import json + + data_str = json.dumps(data, sort_keys=True, default=str) + return hashlib.md5(data_str.encode()).hexdigest() + + @staticmethod + def validate_checksum( + data: dict | list, expected_checksum: str + ) -> tuple[bool, dict | None]: + """Validate data matches checksum. + + Args: + data: Data to validate + expected_checksum: Expected checksum value + + Returns: + (is_valid, failure dict or None) + """ + actual_checksum = ChecksumValidator.compute_checksum(data) + if actual_checksum != expected_checksum: + return False, { + "field": "checksum", + "value": actual_checksum, + "reason": f"Checksum mismatch: expected {expected_checksum}, got {actual_checksum}", + "severity": "error", + } + return True, None + + +class BusinessRuleValidator: + """Validate data against business logic rules.""" + + def __init__(self): + self.rules: dict[str, Callable[[Any], tuple[bool, str]]] = {} + + def add_rule(self, rule_name: str, rule_fn: Callable[[Any], tuple[bool, str]]) -> None: + """Add a business rule. + + Args: + rule_name: Name of the rule + rule_fn: Function that takes data and returns (is_valid, reason) + """ + self.rules[rule_name] = rule_fn + + def validate(self, data: Any) -> tuple[bool, list[dict]]: + """Apply all business rules. + + Args: + data: Data to validate + + Returns: + (is_valid, list of failure dicts) + """ + failures = [] + for rule_name, rule_fn in self.rules.items(): + try: + is_valid, reason = rule_fn(data) + if not is_valid: + failures.append( + { + "field": "business_rule", + "value": None, + "reason": f"{rule_name}: {reason}", + "severity": "error", + } + ) + except Exception as e: + failures.append( + { + "field": "business_rule", + "value": None, + "reason": f"{rule_name} raised exception: {e}", + "severity": "error", + } + ) + + return len(failures) == 0, failures + + +# Concrete validators for common cases + +class BarDataValidator: + """Validate OHLCV bar data.""" + + @staticmethod + def validate_bar(bar: dict) -> tuple[bool, list[dict]]: + """Validate a single bar. + + Args: + bar: Bar dictionary with {timestamp, open, high, low, close, volume} + + Returns: + (is_valid, list of failure dicts) + """ + failures = [] + + # Check required fields + required = {"timestamp", "open", "high", "low", "close", "volume"} + for field in required: + if field not in bar: + failures.append( + { + "field": field, + "value": None, + "reason": "Missing required field", + "severity": "error", + } + ) + + if failures: + return False, failures + + # Check value consistency + open_val = bar["open"] + high_val = bar["high"] + low_val = bar["low"] + close_val = bar["close"] + volume_val = bar["volume"] + + # High must be >= all others + if high_val < open_val or high_val < close_val or high_val < low_val: + failures.append( + { + "field": "high", + "value": high_val, + "reason": f"High ({high_val}) must be >= open ({open_val}), close ({close_val}), low ({low_val})", + "severity": "error", + } + ) + + # Low must be <= all others + if low_val > open_val or low_val > close_val or low_val > high_val: + failures.append( + { + "field": "low", + "value": low_val, + "reason": f"Low ({low_val}) must be <= open ({open_val}), close ({close_val}), high ({high_val})", + "severity": "error", + } + ) + + # Volume should be non-negative + if volume_val < 0: + failures.append( + { + "field": "volume", + "value": volume_val, + "reason": "Volume cannot be negative", + "severity": "error", + } + ) + + # Warn on zero volume + if volume_val == 0: + failures.append( + { + "field": "volume", + "value": volume_val, + "reason": "Zero volume bar", + "severity": "warning", + } + ) + + return len([f for f in failures if f.severity == "error"]) == 0, failures diff --git a/dashboard_api.py b/dashboard_api.py new file mode 100644 index 0000000..323b8ed --- /dev/null +++ b/dashboard_api.py @@ -0,0 +1,35 @@ +"""Run the dashboard API + UI standalone (read-only against the event DB). + + python dashboard_api.py + # open http://127.0.0.1:8010 + +Env: DASHBOARD_HOST, DASHBOARD_PORT, WATCHER_EVENT_DB_PATH, +TRADING_EXECUTION_MODE. For approve/reject/exit buttons to act, run the full +orchestrator (run_live_paper.py) instead, which attaches the execution +service to this same server. +""" + +from __future__ import annotations + +import os + +from api.main import DashboardState, create_server + + +def main() -> None: + host = os.environ.get("DASHBOARD_HOST", "127.0.0.1") + port = int(os.environ.get("DASHBOARD_PORT", "8010")) + db_path = os.environ.get("WATCHER_EVENT_DB_PATH", "./data/momentum.duckdb") + mode = os.environ.get("TRADING_EXECUTION_MODE", "alpaca_paper") + state = DashboardState(db_path, execution_service=None, execution_mode=mode) + server = create_server(state, host, port) + print(f"[dashboard] serving http://{host}:{port} (event db: {db_path}, mode: {mode})") + print("[dashboard] read-only mode — run run_live_paper.py for live actions") + try: + server.serve_forever() + except KeyboardInterrupt: + print("\n[dashboard] stopped") + + +if __name__ == "__main__": + main() diff --git a/fetch_minute_bars.py b/fetch_minute_bars.py new file mode 100644 index 0000000..6f4c5db --- /dev/null +++ b/fetch_minute_bars.py @@ -0,0 +1,51 @@ +"""Fetch 1-minute Alpaca bars into the research market database. + + python fetch_minute_bars.py AAPL TSLA --lookback 390 + python fetch_minute_bars.py SNDL --daily-days 30 +""" + +from __future__ import annotations + +import argparse +import sys + +from dotenv import load_dotenv + +from alpaca_paper.client import AlpacaPaperClient +from alpaca_paper.settings import AlpacaPaperSettings +from research.ingestion.market_data import ingest_daily_history, ingest_live_minute_bars +from research.multi_schema import open_research_db + + +def main(argv: list[str] | None = None) -> int: + load_dotenv() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("symbols", nargs="+", help="ticker symbols") + parser.add_argument("--lookback", type=int, default=390, help="minutes of 1-min bars") + parser.add_argument("--daily-days", type=int, default=30, help="days of daily backfill") + parser.add_argument("--feed", default=None, help="alpaca feed (default: settings/iex)") + args = parser.parse_args(argv) + + settings = AlpacaPaperSettings.from_env() + if not settings.is_configured: + print("error: set ALPACA_API_KEY and ALPACA_SECRET_KEY (see .env.example)") + return 1 + + client = AlpacaPaperClient(settings) + con = open_research_db("market") + symbols = [s.upper() for s in args.symbols] + + daily = ingest_daily_history(con, client, symbols, days=args.daily_days) + print(f"daily bars: {daily.daily_rows} rows" + (f" errors={daily.errors}" if daily.errors else "")) + + minute = ingest_live_minute_bars( + con, client, symbols, lookback_minutes=args.lookback, feed=args.feed + ) + print(f"minute bars: {minute.minute_rows} rows for {sorted(set(minute.symbols))}" + + (f" errors={minute.errors}" if minute.errors else "")) + con.close() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/milestone3_verify.py b/milestone3_verify.py new file mode 100644 index 0000000..6c19a28 --- /dev/null +++ b/milestone3_verify.py @@ -0,0 +1,100 @@ +"""Milestone 3 verification: Schwab integration health check. + + python milestone3_verify.py + +Walks every layer of the Schwab integration and prints PASS/WARN/FAIL. +Designed to be useful both with and without credentials: without them it +verifies graceful degradation (fallback account, empty orders, +unauthenticated health) instead of crashing. +""" + +from __future__ import annotations + +import sys + +from dotenv import load_dotenv + + +def check(label: str, fn): + try: + status, detail = fn() + except Exception as exc: # noqa: BLE001 + status, detail = "FAIL", str(exc) + print(f"[{status:^4}] {label}: {detail}") + return status != "FAIL" + + +def main() -> int: + load_dotenv() + from schwab.auth.lifecycle import TokenLifecycle + from schwab.auth.token_store import TokenStore + from schwab.health.reporter import HealthReporter + from schwab.market.client import SchwabApiError, SchwabMarketClient + from schwab.orders.reader import OrdersReader + from schwab.positions.reader import PositionsReader + from schwab.settings import SchwabSettings + + settings = SchwabSettings.from_env() + ok = True + + def c_settings(): + creds = settings.has_broker_credentials or settings.has_market_data_credentials + return ("PASS" if creds else "WARN", + f"app keys {'present' if creds else 'absent'}; token_path={settings.token_path}") + ok &= check("settings", c_settings) + + def c_token(): + store = TokenStore(settings.token_path) + bundle = store.load() + if bundle is None: + return "WARN", "no token file — run OAuth flow to authenticate" + return ("PASS" if not bundle.is_expired else "WARN", + f"token loaded, expired={bundle.is_expired}") + ok &= check("token store", c_token) + + lifecycle = TokenLifecycle(settings) + + def c_lifecycle(): + status = lifecycle.status() + if status.get("authenticated"): + return "PASS", f"authenticated (expired={status.get('expired')})" + return "WARN", f"unauthenticated: {status.get('reason')}" + ok &= check("token lifecycle", c_lifecycle) + + def c_health(): + report = HealthReporter(settings=settings, lifecycle=lifecycle).check() + d = report.to_dict() + return "PASS", f"overall={d.get('status')} checks={len(d.get('checks') or [])}" + ok &= check("health reporter", c_health) + + def c_positions(): + reader = PositionsReader(settings=settings, lifecycle=lifecycle) + summary = reader.read_account_summary() + if getattr(summary, "is_fallback", False): + return "WARN", f"fallback summary ({summary.account_id}) — authenticate for live data" + return "PASS", f"account {summary.account_id}" + ok &= check("positions reader", c_positions) + + def c_orders(): + reader = OrdersReader(settings=settings, lifecycle=lifecycle) + orders = reader.read_orders() + return "PASS", f"{len(orders)} orders (empty is expected when unauthenticated)" + ok &= check("orders reader", c_orders) + + def c_market(): + client = SchwabMarketClient(settings=settings, lifecycle=lifecycle) + try: + quotes = client.get_quotes(["AAPL"]) + return "PASS", f"quotes returned for {list(quotes)}" + except SchwabApiError as exc: + if exc.status == 401: + return "WARN", "market client correctly raises 401 when unauthenticated" + raise + ok &= check("market client", c_market) + + print("\nmilestone 3:", "VERIFIED (warnings are expected without live Schwab auth)" if ok else "FAILED") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/base.py b/models/base.py new file mode 100644 index 0000000..62448b2 --- /dev/null +++ b/models/base.py @@ -0,0 +1,26 @@ +"""Models for Milestone 2. + +Event and data models for the event-first architecture. +""" + +from pydantic import BaseModel, Field + + +class EventMode(str): + """Runtime modes for events.""" + + REPLAY = "replay" + PAPER = "paper" + LIVE = "live" + + +class BaseEvent(BaseModel): + """Base event with common fields.""" + + timestamp: str = Field(description="Event timestamp (ISO format)") + mode: EventMode = Field(description="Runtime mode (replay/paper/live)") + event_type: str = Field(description="Event type identifier") + correlation_id: str | None = Field( + default=None, description="Correlation ID for causal chain" + ) + message: str = Field(description="Human-readable message") diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..68e2e2a --- /dev/null +++ b/package-lock.json @@ -0,0 +1,18 @@ +{ + "name": "momentum", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "claude": "^0.1.1" + } + }, + "node_modules/claude": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/claude/-/claude-0.1.1.tgz", + "integrity": "sha512-j7oSibqQdIODNhkI1sEJzHMiPsF43L/GqNbcA+eDDyGM10+x2sH9NW/PK6vM3z0J2tLDKMBcc5ZjVaoRinhuCA==", + "license": "ISC" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..1353701 --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "claude": "^0.1.1" + } +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8811e26 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "momentum" +version = "0.4.0" +description = "Small-cap momentum research & paper-trading system" +requires-python = ">=3.11" +dependencies = [ + "duckdb>=1.0.0", + "pandas>=2.0.0", + "pydantic>=2.0.0", + "pydantic-settings>=2.0.0", + "pyyaml>=6.0", + "typer>=0.9.0", + "rich>=13.0.0", + "numpy>=1.24.0", + "python-dotenv>=1.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] + +[tool.uv] +# This is a runnable app (flat layout: scripts at the repo root plus api/, +# storage/, runtime/, ... packages), not a distributable library. Don't try to +# build/install it as a wheel — just provision the dependencies. Code runs via +# `uv run run_live_paper.py` from the repo root (pytest uses pythonpath="." ). +package = false diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..659810a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +duckdb>=1.0.0 +pandas>=2.0.0 +pydantic>=2.0.0 +pytest>=7.0.0 +python-dotenv>=1.0.0 +pyyaml>=6.0 +numpy>=1.24.0 diff --git a/research/__init__.py b/research/__init__.py new file mode 100644 index 0000000..38e438a --- /dev/null +++ b/research/__init__.py @@ -0,0 +1 @@ +"""Research data layer: ingestion, schema, and queries.""" diff --git a/research/ingestion/__init__.py b/research/ingestion/__init__.py new file mode 100644 index 0000000..47934c4 --- /dev/null +++ b/research/ingestion/__init__.py @@ -0,0 +1,16 @@ +"""Research ingestion: bars, news, signal scans, and watcher providers.""" + +from research.ingestion.market_data import ( # noqa: F401 + IngestionResult, + classify_session, + discover_active_symbols, + ingest_daily_history, + ingest_live_minute_bars, +) +from research.ingestion.scheduler import Scheduler, ScheduledTask # noqa: F401 +from research.ingestion.signals import scan_gappers, store_scanner_snapshot # noqa: F401 +from research.ingestion.watcher_task import ( # noqa: F401 + LiveWatchlistProvider, + ResearchWatchlistProvider, + run_watcher_tick, +) diff --git a/research/ingestion/market_data.py b/research/ingestion/market_data.py new file mode 100644 index 0000000..53f8777 --- /dev/null +++ b/research/ingestion/market_data.py @@ -0,0 +1,235 @@ +"""Market data ingestion: Alpaca bars -> research market.duckdb. + +This module fixes the root cause of "live paper trading never worked": +the watcher reads *minute* bars from the research database, but the old +ingestion only ever wrote *daily* bars for a handful of mega-caps and +hardcoded ``is_regular_hours = TRUE`` on every row. Here we: + +* ingest 1-minute bars (IEX feed by default) for a configurable symbol list, +* derive ``is_premarket`` / ``is_regular_hours`` / ``is_afterhours`` from the + bar timestamp in US/Eastern (regular session 09:30-16:00), +* backfill daily bars so the watcher can compute gap %% and relative volume. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta, timezone +from zoneinfo import ZoneInfo + +EASTERN = ZoneInfo("America/New_York") +REGULAR_OPEN = time(9, 30) +REGULAR_CLOSE = time(16, 0) +PREMARKET_OPEN = time(4, 0) +AFTERHOURS_CLOSE = time(20, 0) + + +def classify_session(ts_utc: datetime) -> tuple[date, bool, bool, bool]: + """Return (session_date, is_premarket, is_regular_hours, is_afterhours).""" + if ts_utc.tzinfo is None: + ts_utc = ts_utc.replace(tzinfo=timezone.utc) + local = ts_utc.astimezone(EASTERN) + t = local.time() + is_pre = PREMARKET_OPEN <= t < REGULAR_OPEN + is_reg = REGULAR_OPEN <= t < REGULAR_CLOSE + is_aft = REGULAR_CLOSE <= t < AFTERHOURS_CLOSE + return local.date(), is_pre, is_reg, is_aft + + +def parse_alpaca_timestamp(raw: str) -> datetime: + """Alpaca returns RFC3339 timestamps like '2026-06-11T13:30:00Z'.""" + return datetime.fromisoformat(raw.replace("Z", "+00:00")) + + +@dataclass +class IngestionResult: + minute_rows: int = 0 + daily_rows: int = 0 + symbols: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + +def upsert_minute_bars( + con, + symbol: str, + bars: list[dict], + source_provider: str = "alpaca_iex", +) -> int: + """Insert/replace Alpaca minute bars for one symbol. Returns row count.""" + rows = [] + for bar in bars: + ts = parse_alpaca_timestamp(bar["t"]) + session_date, is_pre, is_reg, is_aft = classify_session(ts) + rows.append( + ( + symbol, + ts.astimezone(timezone.utc).replace(tzinfo=None), + session_date, + is_pre, + is_reg, + is_aft, + float(bar["o"]), + float(bar["h"]), + float(bar["l"]), + float(bar["c"]), + int(bar["v"]), + float(bar.get("vw") or bar["c"]), + None, # spread_pct (not available from bars endpoint) + False, # halt_status + source_provider, + 1.0, # quality_score: real exchange data + ) + ) + if not rows: + return 0 + con.executemany( + """ + INSERT OR REPLACE INTO minute_bars ( + symbol, timestamp, session_date, + is_premarket, is_regular_hours, is_afterhours, + open, high, low, close, volume, vwap, + spread_pct, halt_status, source_provider, quality_score + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + rows, + ) + return len(rows) + + +def upsert_daily_bars(con, symbol: str, bars: list[dict]) -> int: + """Insert/replace Alpaca daily bars, deriving previous_close per row.""" + ordered = sorted(bars, key=lambda b: b["t"]) + rows = [] + prev_close: float | None = None + for bar in ordered: + ts = parse_alpaca_timestamp(bar["t"]).astimezone(EASTERN) + close = float(bar["c"]) + rows.append( + ( + symbol, + ts.date(), + float(bar["o"]), + float(bar["h"]), + float(bar["l"]), + close, + int(bar["v"]), + float(bar.get("vw") or close), + prev_close, + ) + ) + prev_close = close + if not rows: + return 0 + con.executemany( + """ + INSERT OR REPLACE INTO daily_bars ( + symbol, trade_date, open, high, low, close, volume, vwap, + previous_close + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + rows, + ) + return len(rows) + + +def ingest_live_minute_bars( + con, + client, + symbols: list[str], + lookback_minutes: int = 240, + feed: str | None = None, +) -> IngestionResult: + """Pull recent 1-minute bars for `symbols` into the research database. + + `client` is an AlpacaPaperClient (or anything with get_minute_bars). + Free Alpaca keys can only see IEX data, and the SIP feed additionally + embargoes the most recent 15 minutes; we end the window slightly in the + past so the request never 403s. + """ + result = IngestionResult() + if not symbols: + return result + now = datetime.now(timezone.utc) + start = now - timedelta(minutes=lookback_minutes) + end = now - timedelta(seconds=30) + try: + bars_by_symbol = client.get_minute_bars( + symbols, + start_iso=start.isoformat().replace("+00:00", "Z"), + end_iso=end.isoformat().replace("+00:00", "Z"), + feed=feed, + ) + except Exception as exc: # noqa: BLE001 - report, don't crash the loop + result.errors.append(f"minute bars fetch failed: {exc}") + return result + for symbol, bars in bars_by_symbol.items(): + try: + count = upsert_minute_bars(con, symbol, bars) + if count: + result.minute_rows += count + result.symbols.append(symbol) + except Exception as exc: # noqa: BLE001 + result.errors.append(f"{symbol}: minute upsert failed: {exc}") + return result + + +def ingest_daily_history( + con, + client, + symbols: list[str], + days: int = 30, +) -> IngestionResult: + """Backfill daily bars so gap %% / relative volume have a baseline.""" + result = IngestionResult() + if not symbols: + return result + start = (datetime.now(timezone.utc) - timedelta(days=days * 2)).date() + try: + bars_by_symbol = client.get_daily_bars( + symbols, start_iso=f"{start.isoformat()}T00:00:00Z" + ) + except Exception as exc: # noqa: BLE001 + result.errors.append(f"daily bars fetch failed: {exc}") + return result + for symbol, bars in bars_by_symbol.items(): + try: + count = upsert_daily_bars(con, symbol, bars) + if count: + result.daily_rows += count + result.symbols.append(symbol) + except Exception as exc: # noqa: BLE001 + result.errors.append(f"{symbol}: daily upsert failed: {exc}") + return result + + +def discover_active_symbols( + client, + top: int = 20, + price_min: float = 1.0, + price_max: float = 20.0, +) -> list[str]: + """Use Alpaca's most-actives screener to find live momentum candidates. + + Filters the screener output to the strategy's price band using latest + trades. Failures return an empty list so the caller can fall back to a + static symbol list. + """ + try: + actives = client.get_most_actives(top=top * 3, by="volume") + symbols = [a["symbol"] for a in actives if a.get("symbol")] + if not symbols: + return [] + trades = client.get_latest_trades(symbols[:50]) + in_band = [] + for symbol in symbols: + trade = trades.get(symbol) + if not trade: + continue + price = float(trade.get("p") or 0.0) + if price_min <= price <= price_max: + in_band.append(symbol) + if len(in_band) >= top: + break + return in_band + except Exception: # noqa: BLE001 - screener needs a paid feed sometimes + return [] diff --git a/research/ingestion/rss.py b/research/ingestion/rss.py new file mode 100644 index 0000000..5088e30 --- /dev/null +++ b/research/ingestion/rss.py @@ -0,0 +1,158 @@ +"""RSS news ingestion: feed XML -> raw_fetch_attempts + raw_news_items. + +Stdlib-only (urllib + xml.etree). Append-only raw landing tables; downstream +enrichment can dedupe via payload_hash. +""" + +from __future__ import annotations + +import hashlib +import re +import urllib.request +import uuid +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from datetime import datetime, timezone + +PARSER_VERSION = "rss-v2" +TICKER_RE = re.compile(r"\((?:NASDAQ|NYSE|AMEX|OTC)[:\s]+([A-Z]{1,5})\)") + + +@dataclass +class RssIngestResult: + source: str + fetch_attempt_id: str = "" + item_count: int = 0 + http_status: int | None = None + error: str | None = None + new_items: list[str] = field(default_factory=list) + + +def _text(element, tag: str) -> str: + child = element.find(tag) + return (child.text or "").strip() if child is not None else "" + + +def parse_rss(xml_bytes: bytes) -> list[dict]: + """Parse RSS 2.0 / Atom into a list of raw item dicts.""" + root = ET.fromstring(xml_bytes) + items: list[dict] = [] + # RSS 2.0 + for item in root.iter("item"): + items.append( + { + "title": _text(item, "title"), + "url": _text(item, "link"), + "published": _text(item, "pubDate"), + "snippet": _text(item, "description")[:500], + } + ) + # Atom + ns = "{http://www.w3.org/2005/Atom}" + for entry in root.iter(f"{ns}entry"): + link = entry.find(f"{ns}link") + items.append( + { + "title": _text(entry, f"{ns}title"), + "url": link.get("href", "") if link is not None else "", + "published": _text(entry, f"{ns}updated"), + "snippet": _text(entry, f"{ns}summary")[:500], + } + ) + return items + + +def extract_tickers(text: str) -> list[str]: + return sorted(set(TICKER_RE.findall(text or ""))) + + +def ingest_rss_feed( + con, + source: str, + url: str, + timeout: int = 15, + ingest_run_id: str | None = None, +) -> RssIngestResult: + """Fetch one RSS feed and land items into raw_news_items.""" + result = RssIngestResult(source=source) + attempt_id = str(uuid.uuid4()) + result.fetch_attempt_id = attempt_id + fetched_at = datetime.now(timezone.utc).replace(tzinfo=None) + payload: bytes = b"" + try: + request = urllib.request.Request( + url, headers={"User-Agent": "momentum-research/1.0"} + ) + with urllib.request.urlopen(request, timeout=timeout) as resp: + result.http_status = resp.status + payload = resp.read() + items = parse_rss(payload) + except Exception as exc: # noqa: BLE001 + result.error = str(exc) + items = [] + payload_hash = hashlib.sha256(payload).hexdigest() if payload else None + con.execute( + """ + INSERT INTO raw_fetch_attempts ( + id, source, capability, fetched_at, ingest_run_id, + http_status, item_count, error_msg, payload_hash, parser_version + ) VALUES (?, ?, 'news_rss', ?, ?, ?, ?, ?, ?, ?) + """, + [ + attempt_id, + source, + fetched_at, + ingest_run_id, + result.http_status, + len(items), + result.error, + payload_hash, + PARSER_VERSION, + ], + ) + for item in items: + item_hash = hashlib.sha256( + f"{source}|{item['url']}|{item['title']}".encode() + ).hexdigest() + exists = con.execute( + "SELECT 1 FROM raw_news_items WHERE payload_hash = ? LIMIT 1", + [item_hash], + ).fetchone() + if exists: + continue + item_id = str(uuid.uuid4()) + con.execute( + """ + INSERT INTO raw_news_items ( + id, fetch_attempt_id, source, raw_url, raw_title, + raw_published_at, raw_body_snippet, raw_tickers, + payload_hash, parser_version, fetched_at, ingest_run_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + item_id, + attempt_id, + source, + item["url"], + item["title"], + item["published"], + item["snippet"], + ",".join(extract_tickers(f"{item['title']} {item['snippet']}")), + item_hash, + PARSER_VERSION, + fetched_at, + ingest_run_id, + ], + ) + result.new_items.append(item_id) + result.item_count = len(items) + return result + + +def ingest_all_feeds(con, feeds: dict[str, str]) -> list[RssIngestResult]: + """Ingest a {source_name: url} mapping; never raises.""" + run_id = str(uuid.uuid4()) + return [ + ingest_rss_feed(con, source, url, ingest_run_id=run_id) + for source, url in feeds.items() + ] diff --git a/research/ingestion/scheduler.py b/research/ingestion/scheduler.py new file mode 100644 index 0000000..eb4a54a --- /dev/null +++ b/research/ingestion/scheduler.py @@ -0,0 +1,68 @@ +"""Tiny interval scheduler used by the orchestrator and research jobs. + +Each task has its own interval; ``run_pending`` executes whatever is due. +Task failures are recorded and never propagate, so one flaky data source +cannot stall the trading loop. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Callable + + +@dataclass +class ScheduledTask: + name: str + func: Callable[[], object] + interval_seconds: float + enabled: bool = True + last_run: float = 0.0 + last_error: str | None = None + run_count: int = 0 + error_count: int = 0 + + def due(self, now: float) -> bool: + return self.enabled and (now - self.last_run) >= self.interval_seconds + + +@dataclass +class Scheduler: + tasks: list[ScheduledTask] = field(default_factory=list) + + def add( + self, + name: str, + func: Callable[[], object], + interval_seconds: float, + enabled: bool = True, + ) -> ScheduledTask: + task = ScheduledTask(name, func, interval_seconds, enabled=enabled) + self.tasks.append(task) + return task + + def run_pending(self, now: float | None = None) -> dict[str, object]: + """Run all due tasks once. Returns {task_name: result_or_error}.""" + now = time.monotonic() if now is None else now + results: dict[str, object] = {} + for task in self.tasks: + if not task.due(now): + continue + task.last_run = now + task.run_count += 1 + try: + results[task.name] = task.func() + task.last_error = None + except Exception as exc: # noqa: BLE001 + task.error_count += 1 + task.last_error = str(exc) + results[task.name] = exc + return results + + def run_forever(self, tick_seconds: float = 1.0, stop_check=None) -> None: + while True: + if stop_check and stop_check(): + return + self.run_pending() + time.sleep(tick_seconds) diff --git a/research/ingestion/signals.py b/research/ingestion/signals.py new file mode 100644 index 0000000..2da5f28 --- /dev/null +++ b/research/ingestion/signals.py @@ -0,0 +1,136 @@ +"""Signal scanning: derive gapper candidates and scanner snapshots. + +Reads minute/daily bars from the market database, computes gap %% and +relative volume, and writes ranked rows into scanner_snapshots so the +dashboard and research notebooks can replay what the scanner saw. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from datetime import date, datetime, timezone + +SCANNER_VERSION = "gapper-v2" + + +@dataclass +class GapperCandidate: + symbol: str + price: float + gap_pct: float + cumulative_volume: int + relative_volume: float + vwap: float | None + rank: int = 0 + + +def scan_gappers( + con, + session_date: date, + min_gap_pct: float = 5.0, + min_relative_volume: float = 2.0, + price_min: float = 1.0, + price_max: float = 20.0, + limit: int = 25, +) -> list[GapperCandidate]: + """Find symbols gapping up on elevated volume for a session.""" + rows = con.execute( + """ + WITH latest AS ( + SELECT symbol, close, vwap, + SUM(volume) OVER (PARTITION BY symbol) AS cum_volume, + ROW_NUMBER() OVER ( + PARTITION BY symbol ORDER BY timestamp DESC + ) AS rn + FROM minute_bars + WHERE session_date = ? + ), + prev AS ( + SELECT symbol, close AS prev_close, + ROW_NUMBER() OVER ( + PARTITION BY symbol ORDER BY trade_date DESC + ) AS rn + FROM daily_bars + WHERE trade_date < ? + ), + avg_vol AS ( + SELECT symbol, AVG(volume) AS adv + FROM ( + SELECT symbol, volume, + ROW_NUMBER() OVER ( + PARTITION BY symbol ORDER BY trade_date DESC + ) AS rn + FROM daily_bars + WHERE trade_date < ? + ) + WHERE rn <= 20 + GROUP BY symbol + ) + SELECT l.symbol, l.close, l.vwap, l.cum_volume, + p.prev_close, a.adv + FROM latest l + JOIN prev p ON p.symbol = l.symbol AND p.rn = 1 + LEFT JOIN avg_vol a ON a.symbol = l.symbol + WHERE l.rn = 1 AND l.close BETWEEN ? AND ? + """, + [session_date, session_date, session_date, price_min, price_max], + ).fetchall() + + candidates: list[GapperCandidate] = [] + for symbol, close, vwap, cum_volume, prev_close, adv in rows: + if not prev_close or prev_close <= 0: + continue + gap_pct = (float(close) - float(prev_close)) / float(prev_close) * 100.0 + rvol = (float(cum_volume) / float(adv)) if adv and adv > 0 else 0.0 + if gap_pct >= min_gap_pct and rvol >= min_relative_volume: + candidates.append( + GapperCandidate( + symbol=symbol, + price=float(close), + gap_pct=gap_pct, + cumulative_volume=int(cum_volume), + relative_volume=rvol, + vwap=float(vwap) if vwap is not None else None, + ) + ) + candidates.sort(key=lambda c: (c.gap_pct * max(c.relative_volume, 0.1)), reverse=True) + for i, candidate in enumerate(candidates[:limit], start=1): + candidate.rank = i + return candidates[:limit] + + +def store_scanner_snapshot( + con, candidates: list[GapperCandidate], snapshot_time: datetime | None = None +) -> int: + """Persist a scan into scanner_snapshots. Returns rows written.""" + ts = (snapshot_time or datetime.now(timezone.utc)).replace(tzinfo=None) + rows = [ + ( + str(uuid.uuid4()), + ts, + c.symbol, + c.rank, + c.price, + c.gap_pct, + c.cumulative_volume, + c.relative_volume, + c.vwap, + (c.price - c.vwap) / c.vwap * 100.0 if c.vwap else None, + SCANNER_VERSION, + ) + for c in candidates + ] + if not rows: + return 0 + con.executemany( + """ + INSERT INTO scanner_snapshots ( + id, snapshot_time, symbol, rank, price, gap_pct, + cumulative_volume, relative_volume, vwap, distance_from_vwap, + scanner_version + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + rows, + ) + return len(rows) diff --git a/research/ingestion/watcher_task.py b/research/ingestion/watcher_task.py new file mode 100644 index 0000000..4921f0b --- /dev/null +++ b/research/ingestion/watcher_task.py @@ -0,0 +1,186 @@ +"""Watchlist providers and watcher task glue. + +Two providers implement the WatchlistProvider protocol: + +* ResearchWatchlistProvider — reads candidates and bars from the research + market.duckdb (populated by research.ingestion.market_data). The price + band is configurable via WATCHER_PRICE_MIN / WATCHER_PRICE_MAX, fixing + the old hardcoded ``close BETWEEN 1 AND 20`` filter that silently + excluded every ingested symbol. +* LiveWatchlistProvider — takes an explicit symbol list and pulls bars + straight from Alpaca, for running without a research database. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from datetime import date, datetime, timedelta, timezone + +import pandas as pd + +from research import query as research_query +from research.ingestion.market_data import classify_session, parse_alpaca_timestamp +from runtime.watcher import WatchCandidate, Watcher, WatcherTickResult + + +def _env_float(name: str, default: float) -> float: + raw = os.environ.get(name, "").strip() + try: + return float(raw) if raw else default + except ValueError: + return default + + +@dataclass +class ResearchWatchlistProvider: + """Candidates + bars from the research market database.""" + + con: object + price_min: float = field(default_factory=lambda: _env_float("WATCHER_PRICE_MIN", 1.0)) + price_max: float = field(default_factory=lambda: _env_float("WATCHER_PRICE_MAX", 20.0)) + limit: int = 50 + # symbols manually injected from the dashboard; always evaluated regardless + # of the price-band query (e.g. a ticker not yet ingested) + extra_symbols: set = field(default_factory=set) + + def add_symbol(self, symbol: str) -> None: + self.extra_symbols.add(symbol.upper()) + + def get_candidates(self, session_date: date) -> list[WatchCandidate]: + rows = research_query.query_session_symbols( + self.con, + session_date, + price_min=self.price_min, + price_max=self.price_max, + limit=self.limit, + ) + seen = set() + candidates = [] + for row in rows: + symbol = row["symbol"] + seen.add(symbol) + candidates.append( + WatchCandidate( + symbol=symbol, + last_price=row.get("last_price"), + previous_close=research_query.query_previous_close( + self.con, symbol, session_date + ), + avg_daily_volume=research_query.query_avg_daily_volume( + self.con, symbol, session_date + ), + source="research", + ) + ) + # always include manually injected symbols + for symbol in sorted(self.extra_symbols - seen): + candidates.append( + WatchCandidate( + symbol=symbol, + previous_close=research_query.query_previous_close( + self.con, symbol, session_date + ), + avg_daily_volume=research_query.query_avg_daily_volume( + self.con, symbol, session_date + ), + source="manual", + ) + ) + return candidates + + def get_bars(self, symbol: str, session_date: date) -> pd.DataFrame: + return research_query.query_minute_bars(self.con, symbol, session_date) + + +@dataclass +class LiveWatchlistProvider: + """Explicit symbols, bars fetched live from Alpaca (no database needed).""" + + client: object + symbols: list[str] + lookback_minutes: int = 420 + _daily_cache: dict[str, list[dict]] = field(default_factory=dict) + + def _daily_bars(self, symbol: str) -> list[dict]: + if symbol not in self._daily_cache: + start = (datetime.now(timezone.utc) - timedelta(days=60)).date() + try: + payload = self.client.get_daily_bars( + [symbol], start_iso=f"{start.isoformat()}T00:00:00Z" + ) + self._daily_cache[symbol] = payload.get(symbol, []) + except Exception: # noqa: BLE001 + self._daily_cache[symbol] = [] + return self._daily_cache[symbol] + + def get_candidates(self, session_date: date) -> list[WatchCandidate]: + candidates = [] + for symbol in self.symbols: + daily = self._daily_bars(symbol) + prev_close = None + adv = None + if daily: + # last bar strictly before the session date + prior = [ + b for b in daily + if parse_alpaca_timestamp(b["t"]).date() < session_date + ] + if prior: + prev_close = float(prior[-1]["c"]) + recent = prior[-20:] + adv = sum(float(b["v"]) for b in recent) / len(recent) + candidates.append( + WatchCandidate( + symbol=symbol, + previous_close=prev_close, + avg_daily_volume=adv, + source="live", + ) + ) + return candidates + + def get_bars(self, symbol: str, session_date: date) -> pd.DataFrame: + now = datetime.now(timezone.utc) + start = now - timedelta(minutes=self.lookback_minutes) + try: + payload = self.client.get_minute_bars( + [symbol], + start_iso=start.isoformat().replace("+00:00", "Z"), + end_iso=(now - timedelta(seconds=30)).isoformat().replace("+00:00", "Z"), + ) + bars = payload.get(symbol, []) + except Exception: # noqa: BLE001 + bars = [] + rows = [] + for bar in bars: + ts = parse_alpaca_timestamp(bar["t"]) + bar_date, is_pre, is_reg, _ = classify_session(ts) + if bar_date != session_date: + continue + rows.append( + { + "symbol": symbol, + "timestamp": ts.astimezone(timezone.utc).replace(tzinfo=None), + "session_date": bar_date, + "is_premarket": is_pre, + "is_regular_hours": is_reg, + "open": float(bar["o"]), + "high": float(bar["h"]), + "low": float(bar["l"]), + "close": float(bar["c"]), + "volume": int(bar["v"]), + "vwap": float(bar.get("vw") or bar["c"]), + "quality_score": 1.0, + } + ) + return pd.DataFrame(rows) + + +def run_watcher_tick( + watcher: Watcher, session_date: date | None = None +) -> WatcherTickResult: + """One watcher tick with a safe default session date (US/Eastern today).""" + if session_date is None: + session_date, _, _, _ = classify_session(datetime.now(timezone.utc)) + return watcher.tick(session_date) diff --git a/research/multi_schema.py b/research/multi_schema.py new file mode 100644 index 0000000..10da77d --- /dev/null +++ b/research/multi_schema.py @@ -0,0 +1,34 @@ +"""Multi-database layout for research data. + +market.duckdb — bars, halts, market context +news.duckdb — raw news items and fetch attempts +signals.duckdb — scanner snapshots and engineered features + +All share the same core schema (tables are cheap when empty), which keeps +cross-database queries simple. +""" + +from __future__ import annotations + +from pathlib import Path + +from storage.db import get_connection + +RESEARCH_DIR = Path("data/research") + +DB_FILES = { + "market": "market.duckdb", + "news": "news.duckdb", + "signals": "signals.duckdb", +} + + +def research_db_path(name: str, base_dir: str | Path | None = None) -> Path: + base = Path(base_dir) if base_dir else RESEARCH_DIR + filename = DB_FILES.get(name, f"{name}.duckdb") + return base / filename + + +def open_research_db(name: str, base_dir: str | Path | None = None): + """Open one of the research databases, creating schema if needed.""" + return get_connection(research_db_path(name, base_dir)) diff --git a/research/query.py b/research/query.py new file mode 100644 index 0000000..f625455 --- /dev/null +++ b/research/query.py @@ -0,0 +1,83 @@ +"""Read-side queries over the research market database.""" + +from __future__ import annotations + +from datetime import date + +import pandas as pd + + +def query_session_symbols( + con, + session_date: date, + price_min: float = 1.0, + price_max: float = 20.0, + limit: int = 50, +) -> list[dict]: + """Symbols with minute bars today whose latest close is inside the band. + + Pass price_min=0 / price_max=inf to disable the band. + """ + rows = con.execute( + """ + WITH latest AS ( + SELECT symbol, close, volume, + ROW_NUMBER() OVER ( + PARTITION BY symbol ORDER BY timestamp DESC + ) AS rn, + COUNT(*) OVER (PARTITION BY symbol) AS bar_count + FROM minute_bars + WHERE session_date = ? + ) + SELECT symbol, close AS last_price, bar_count + FROM latest + WHERE rn = 1 AND close BETWEEN ? AND ? + ORDER BY bar_count DESC + LIMIT ? + """, + [session_date, price_min, price_max, limit], + ).fetchall() + return [ + {"symbol": r[0], "last_price": float(r[1]), "bar_count": int(r[2])} + for r in rows + ] + + +def query_minute_bars(con, symbol: str, session_date: date) -> pd.DataFrame: + """All of today's minute bars for a symbol, chronological.""" + return con.execute( + """ + SELECT symbol, timestamp, session_date, is_premarket, is_regular_hours, + open, high, low, close, volume, vwap, quality_score + FROM minute_bars + WHERE symbol = ? AND session_date = ? + ORDER BY timestamp ASC + """, + [symbol, session_date], + ).df() + + +def query_previous_close(con, symbol: str, session_date: date) -> float | None: + row = con.execute( + """ + SELECT close FROM daily_bars + WHERE symbol = ? AND trade_date < ? + ORDER BY trade_date DESC LIMIT 1 + """, + [symbol, session_date], + ).fetchone() + return float(row[0]) if row else None + + +def query_avg_daily_volume(con, symbol: str, session_date: date, days: int = 20) -> float | None: + row = con.execute( + """ + SELECT AVG(volume) FROM ( + SELECT volume FROM daily_bars + WHERE symbol = ? AND trade_date < ? + ORDER BY trade_date DESC LIMIT ? + ) + """, + [symbol, session_date, days], + ).fetchone() + return float(row[0]) if row and row[0] is not None else None diff --git a/research/schema.py b/research/schema.py new file mode 100644 index 0000000..4d56fd0 --- /dev/null +++ b/research/schema.py @@ -0,0 +1,14 @@ +"""Research database schema — reuses the core market schema.""" + +from __future__ import annotations + +from pathlib import Path + +from storage.db import get_connection + +DEFAULT_RESEARCH_DB = "data/research/market.duckdb" + + +def get_research_connection(db_path: str | Path = DEFAULT_RESEARCH_DB): + """Open (and initialize) the research market database.""" + return get_connection(db_path) diff --git a/research_cli.py b/research_cli.py new file mode 100644 index 0000000..ddde484 --- /dev/null +++ b/research_cli.py @@ -0,0 +1,102 @@ +"""Research utilities CLI. + + python research_cli.py symbols --date 2026-06-11 + python research_cli.py gappers --date 2026-06-11 + python research_cli.py news +""" + +from __future__ import annotations + +import argparse +import os +import sys +from datetime import date + +from dotenv import load_dotenv + +from research import query as rq +from research.ingestion.rss import ingest_all_feeds +from research.ingestion.signals import scan_gappers, store_scanner_snapshot +from research.multi_schema import open_research_db + +DEFAULT_FEEDS = { + "globenewswire_public": "https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies", + "prnewswire_news": "https://www.prnewswire.com/rss/news-releases-list.rss", +} + + +def cmd_symbols(args) -> int: + con = open_research_db("market") + rows = rq.query_session_symbols( + con, args.session_date, price_min=args.price_min, + price_max=args.price_max, limit=args.limit, + ) + if not rows: + print("no symbols — ingest minute bars first (fetch_minute_bars.py)") + return 1 + for r in rows: + print(f"{r['symbol']:>6} last={r['last_price']:<8.2f} bars={r['bar_count']}") + return 0 + + +def cmd_gappers(args) -> int: + con = open_research_db("market") + candidates = scan_gappers( + con, args.session_date, + min_gap_pct=args.min_gap, min_relative_volume=args.min_rvol, + price_min=args.price_min, price_max=args.price_max, + ) + if not candidates: + print("no gappers found for this session") + return 0 + stored = store_scanner_snapshot(con, candidates) + for c in candidates: + print(f"#{c.rank:<2} {c.symbol:>6} gap={c.gap_pct:>6.1f}% rvol={c.relative_volume:>5.1f}x " + f"price={c.price:<8.2f} vol={c.cumulative_volume:,}") + print(f"\nstored {stored} rows in scanner_snapshots") + return 0 + + +def cmd_news(args) -> int: + con = open_research_db("news") + feeds = dict(DEFAULT_FEEDS) + extra = os.environ.get("NEWS_RSS_FEEDS", "") + for pair in extra.split(";"): + if "=" in pair: + name, url = pair.split("=", 1) + feeds[name.strip()] = url.strip() + for result in ingest_all_feeds(con, feeds): + status = result.error or f"http {result.http_status}" + print(f"{result.source:<28} items={result.item_count:<4} new={len(result.new_items):<4} ({status})") + return 0 + + +def main(argv: list[str] | None = None) -> int: + load_dotenv() + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + def add_common(p): + p.add_argument("--date", default=date.today().isoformat()) + p.add_argument("--price-min", type=float, default=1.0) + p.add_argument("--price-max", type=float, default=20.0) + + p_sym = sub.add_parser("symbols", help="list symbols with bars for a session") + add_common(p_sym) + p_sym.add_argument("--limit", type=int, default=50) + + p_gap = sub.add_parser("gappers", help="scan for gappers and store snapshot") + add_common(p_gap) + p_gap.add_argument("--min-gap", type=float, default=5.0) + p_gap.add_argument("--min-rvol", type=float, default=2.0) + + sub.add_parser("news", help="ingest RSS feeds into news.duckdb") + + args = parser.parse_args(argv) + if hasattr(args, "date"): + args.session_date = date.fromisoformat(args.date) + return {"symbols": cmd_symbols, "gappers": cmd_gappers, "news": cmd_news}[args.command](args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/run_live_paper.py b/run_live_paper.py new file mode 100644 index 0000000..a3d15b2 --- /dev/null +++ b/run_live_paper.py @@ -0,0 +1,356 @@ +"""Live paper-trading orchestrator (Milestone 4 end-to-end). + +This is the piece that was missing: a single loop tying together + + 1. ingest — latest 1-minute Alpaca IEX bars into research market.duckdb + (plus a daily-bar backfill on startup so gap %% / RVOL work) + 2. watch — Watcher.tick over the ingested bars, emitting signal_ready + 3. sync — Alpaca paper account / positions / orders into the event DB + 4. execute — TradingExecutionService.tick: approval requests, and order + submission (auto or via dashboard approval) + 5. dashboard — optional embedded API + UI at http://127.0.0.1:8010 + +Usage +----- + python run_live_paper.py # continuous loop + python run_live_paper.py --once # single pass, then exit + python run_live_paper.py --symbols AAPL,SNDL --auto-approve + python run_live_paper.py --no-dashboard + +Without ALPACA_API_KEY / ALPACA_SECRET_KEY the loop runs in dry mode: +network steps are skipped with clear messages, but watcher + dashboard +still work against whatever data already exists. +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time +import uuid +from datetime import datetime, timezone + +from dotenv import load_dotenv + +from alpaca_paper.client import AlpacaPaperClient +from alpaca_paper.execution import AlpacaPaperExecutor +from alpaca_paper.settings import AlpacaPaperSettings +from alpaca_paper.sync import AlpacaPaperSync +from api.main import DashboardState, serve_in_background +from research.ingestion.market_data import ( + classify_session, + discover_active_symbols, + ingest_daily_history, + ingest_live_minute_bars, +) +from research.ingestion.scheduler import Scheduler +from research.ingestion.watcher_task import ResearchWatchlistProvider +from research.multi_schema import open_research_db +from runtime.watcher import Watcher, WatcherConfig +from storage.event_schema import EventMode +from storage.event_store import EventStore +from trading_execution import ExecutionSettings, TradingExecutionService +from trading_mode import TradingModeSettings + +DEFAULT_SYMBOLS = ["AAPL", "TSLA", "AMD", "NVDA", "SNDL"] + + +def _flag(name: str, default: str = "1") -> bool: + return os.environ.get(name, default).strip().lower() in {"1", "true", "yes", "on"} + + +def _now_session_date(): + session_date, _, _, _ = classify_session(datetime.now(timezone.utc)) + return session_date + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Momentum live paper-trading loop") + parser.add_argument("--once", action="store_true", help="run one pass and exit") + parser.add_argument( + "--symbols", + default=os.environ.get("WATCHER_SYMBOLS", ""), + help="comma-separated symbols to watch (default: WATCHER_SYMBOLS env or built-ins)", + ) + parser.add_argument( + "--auto-approve", + action="store_true", + help="auto-approve and submit ready signals (otherwise approve from the dashboard)", + ) + parser.add_argument( + "--discover", + action="store_true", + help="also pull symbols from Alpaca's most-actives screener", + ) + parser.add_argument("--no-dashboard", action="store_true", help="skip the embedded dashboard") + return parser.parse_args(argv) + + +def build_runtime(args: argparse.Namespace) -> dict: + """Wire up every component; safe without API keys.""" + load_dotenv() + + symbols = [s.strip().upper() for s in args.symbols.split(",") if s.strip()] + if not symbols: + symbols = DEFAULT_SYMBOLS + + alpaca_settings = AlpacaPaperSettings.from_env() + has_keys = alpaca_settings.is_configured + client = AlpacaPaperClient(alpaca_settings) if has_keys else None + + event_db = os.environ.get("WATCHER_EVENT_DB_PATH", "./data/momentum.duckdb") + os.makedirs(os.path.dirname(event_db) or ".", exist_ok=True) + store = EventStore(event_db) + research_con = open_research_db("market") + + session_id = f"paper-{_now_session_date().isoformat()}-{uuid.uuid4().hex[:6]}" + + watcher = Watcher( + store, + ResearchWatchlistProvider( + research_con, + limit=int(os.environ.get("WATCHER_MAX_SYMBOLS", "25")), + ), + WatcherConfig( + session_id=session_id, + mode=EventMode.PAPER, + max_symbols=int(os.environ.get("WATCHER_MAX_SYMBOLS", "25")), + min_bars=int(os.environ.get("WATCHER_MIN_BARS", "10")), + min_quality=float(os.environ.get("WATCHER_MIN_QUALITY", "0.30")), + ), + ) + + exec_settings = ExecutionSettings.from_env() + if args.auto_approve: + exec_settings.auto_approve = True + executor = AlpacaPaperExecutor(store, client=client, session_id=session_id) if client else None + + # Live last-trade price source for entry invalidation — tick-by-tick, not + # bar close. Pulls the most recent trade from Alpaca's data API on demand; + # a tiny TTL cache collapses redundant lookups within a single guard pass. + # On any failure it returns None so the price-break check is simply skipped + # for that pass (the wall-clock timeout still protects the entry) rather + # than silently falling back to a stale bar close. + _price_cache: dict[str, tuple[float, float]] = {} + _PRICE_TTL = 2.0 + + def latest_price(symbol: str): + if client is None: + return None + import time as _time + + cached = _price_cache.get(symbol) + now = _time.monotonic() + if cached and (now - cached[1]) < _PRICE_TTL: + return cached[0] + try: + trades = client.get_latest_trades([symbol]) + trade = trades.get(symbol) or trades.get(symbol.upper()) + price = float(trade["p"]) if trade and trade.get("p") is not None else None + except Exception: + price = None + if price is not None: + _price_cache[symbol] = (price, now) + return price + + execution = ( + TradingExecutionService( + store, + executor=executor, + settings=exec_settings, + trading_mode=TradingModeSettings.from_env(), + session_id=session_id, + price_provider=latest_price, + ) + if executor + else None + ) + sync = AlpacaPaperSync(store, client=client, session_id=session_id) if client else None + + return { + "symbols": symbols, + "client": client, + "has_keys": has_keys, + "store": store, + "research_con": research_con, + "watcher": watcher, + "execution": execution, + "sync": sync, + "session_id": session_id, + "event_db": event_db, + } + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + rt = build_runtime(args) + symbols: list[str] = rt["symbols"] + client = rt["client"] + + print(f"[boot] session={rt['session_id']} event_db={rt['event_db']}") + print(f"[boot] watching: {', '.join(symbols)}") + if not rt["has_keys"]: + print("[boot] no ALPACA_API_KEY/SECRET — dry mode: ingestion, sync and " + "order submission are skipped; watcher runs on existing data") + + # dashboard + server = None + if not args.no_dashboard and _flag("DASHBOARD_ENABLED", "1"): + host = os.environ.get("DASHBOARD_HOST", "127.0.0.1") + port = int(os.environ.get("DASHBOARD_PORT", "8010")) + + def inject_symbol(symbol: str) -> None: + # add to the watcher's provider so it's evaluated next tick, and + # to the live symbol list so the next ingest pass pulls its bars + provider = rt["watcher"].provider + if hasattr(provider, "add_symbol"): + provider.add_symbol(symbol) + if symbol not in symbols: + symbols.append(symbol) + + state = DashboardState( + rt["event_db"], + execution_service=rt["execution"], + execution_mode=TradingModeSettings.from_env().execution_mode, + research_con=rt["research_con"], + watch_inject=inject_symbol, + ) + try: + server, _ = serve_in_background(state, host, port) + print(f"[boot] dashboard at http://{host}:{port}") + except OSError as exc: + print(f"[boot] dashboard disabled ({exc})") + + # startup: daily history backfill so gap%/RVOL have a baseline + if client: + if args.discover: + discovered = discover_active_symbols( + client, + top=10, + price_min=float(os.environ.get("WATCHER_PRICE_MIN", "1")), + price_max=float(os.environ.get("WATCHER_PRICE_MAX", "20")), + ) + if discovered: + print(f"[boot] screener added: {', '.join(discovered)}") + symbols = sorted(set(symbols) | set(discovered)) + else: + print("[boot] screener unavailable — using static symbol list") + daily = ingest_daily_history(rt["research_con"], client, symbols, days=30) + print(f"[boot] daily backfill: {daily.daily_rows} rows" + + (f" (errors: {daily.errors})" if daily.errors else "")) + + lookback = int(os.environ.get("LIVE_BARS_LOOKBACK_MINUTES", "240")) + + def step_ingest(): + if not client: + return "skipped (no keys)" + res = ingest_live_minute_bars(rt["research_con"], client, symbols, lookback_minutes=lookback) + return f"{res.minute_rows} minute rows" + (f", errors={res.errors}" if res.errors else "") + + def step_watch(): + res = rt["watcher"].tick(_now_session_date()) + return (f"evaluated={res.evaluated} ready={res.ready} blocked={len(res.blocked)}" + + (f" errors={res.errors}" if res.errors else "")) + + def step_sync(): + if not rt["sync"]: + return "skipped (no keys)" + rt["sync"].sync_all() + return "ok" + + def step_execute(): + if not rt["execution"]: + return "skipped (no keys)" + res = rt["execution"].tick() + backed = len(res.get("backed_out") or []) + line = (f"approvals_requested={res['approvals_requested']} " + f"auto_executed={len(res['auto_executed'])}") + if backed: + syms = ", ".join(b["symbol"] for b in res["backed_out"]) + line += f" backed_out={backed} ({syms})" + return line + + def step_guard(): + # Fast invalidation watch: re-check armed (unfilled) entries against the + # live last-trade price and the wall-clock timeout. Runs far more often + # than `execute` so a break of the entry trigger is caught tick-by-tick + # instead of waiting for the next execute pass. + if not rt["execution"]: + return None + backed = rt["execution"].expire_stale_entries() + if not backed: + return None + return "backed_out " + ", ".join( + f"{b['symbol']} ({b['reason']})" for b in backed + ) + + scheduler = Scheduler() + scheduler.add("ingest", step_ingest, float(os.environ.get("LIVE_BARS_INTERVAL_SECONDS", "60")), + enabled=_flag("LIVE_BARS_ENABLED", "1")) + scheduler.add("watch", step_watch, float(os.environ.get("WATCHER_INTERVAL_SECONDS", "30")), + enabled=_flag("WATCHER_ENABLED", "1")) + scheduler.add("sync", step_sync, float(os.environ.get("ALPACA_PAPER_SYNC_INTERVAL_SECONDS", "60")), + enabled=_flag("ALPACA_PAPER_SYNC_ENABLED", "1")) + scheduler.add("execute", step_execute, float(os.environ.get("TRADING_EXECUTION_INTERVAL_SECONDS", "30")), + enabled=_flag("TRADING_EXECUTION_ENABLED", "1")) + scheduler.add("guard", step_guard, float(os.environ.get("TRADING_ENTRY_GUARD_INTERVAL_SECONDS", "5")), + enabled=_flag("TRADING_EXECUTION_ENABLED", "1")) + + def run_pass(): + # force every task due, in pipeline order + for task in scheduler.tasks: + task.last_run = -10**9 + results = {} + for name in ("ingest", "watch", "sync", "execute"): + for task in scheduler.tasks: + if task.name == name and task.enabled: + results.update(scheduler_run_one(task)) + stamp = datetime.now().strftime("%H:%M:%S") + line = " | ".join(f"{k}: {v}" for k, v in results.items()) + print(f"[{stamp}] {line}") + + def scheduler_run_one(task): + task.last_run = time.monotonic() + task.run_count += 1 + try: + out = task.func() + task.last_error = None + return {task.name: out} + except Exception as exc: # noqa: BLE001 + task.error_count += 1 + task.last_error = str(exc) + return {task.name: f"ERROR {exc}"} + + if args.once: + run_pass() + if server: + server.shutdown() + return 0 + + print("[boot] entering live loop (Ctrl-C to stop)") + try: + # one immediate full pass, then interval-driven + run_pass() + while True: + results = scheduler.run_pending() + if results: + stamp = datetime.now().strftime("%H:%M:%S") + line = " | ".join( + f"{k}: {v if not isinstance(v, Exception) else f'ERROR {v}'}" + for k, v in results.items() + ) + print(f"[{stamp}] {line}") + time.sleep(1.0) + except KeyboardInterrupt: + print("\n[stop] shutting down") + finally: + if server: + server.shutdown() + rt["store"].close() + rt["research_con"].close() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/runtime/__init__.py b/runtime/__init__.py new file mode 100644 index 0000000..daa16f1 --- /dev/null +++ b/runtime/__init__.py @@ -0,0 +1 @@ +"""Runtime package: intraday watcher and provider registry (Milestone 4).""" diff --git a/runtime/providers/__init__.py b/runtime/providers/__init__.py new file mode 100644 index 0000000..d6eeb5f --- /dev/null +++ b/runtime/providers/__init__.py @@ -0,0 +1,10 @@ +"""Market-data provider registry.""" + +from runtime.providers.registry import ( + ProviderKind, + ProviderConfig, + ProviderRegistry, + default_registry, +) + +__all__ = ["ProviderKind", "ProviderConfig", "ProviderRegistry", "default_registry"] diff --git a/runtime/providers/registry.py b/runtime/providers/registry.py new file mode 100644 index 0000000..9365c1c --- /dev/null +++ b/runtime/providers/registry.py @@ -0,0 +1,112 @@ +"""Provider registry for market-data sources. + +(Repaired: ProviderConfig is a proper top-level dataclass and the registry +initializes its mapping correctly — the original nested the dataclass inside +the enum and used an invalid `self.providers: {}` annotation-as-assignment.) +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from enum import Enum + + +class ProviderKind(str, Enum): + """Supported market-data providers.""" + + ALPACA = "alpaca" + SCHWAB = "schwab" + POLYGON = "polygon" + FINNHUB = "finnhub" + SYNTHETIC = "synthetic" + + +@dataclass +class ProviderConfig: + """Configuration for one provider.""" + + kind: ProviderKind + enabled: bool = True + priority: int = 100 # lower = preferred + rate_limit_per_minute: int = 200 + supports_minute_bars: bool = True + supports_quotes: bool = True + env_keys: tuple[str, ...] = field(default_factory=tuple) + + def is_configured(self, env: dict[str, str] | None = None) -> bool: + """A provider is configured when all of its env keys are present.""" + values = dict(os.environ) + if env is not None: + values.update(env) + return all(values.get(key) for key in self.env_keys) + + +class ProviderRegistry: + """Registry of provider configs with priority-based selection.""" + + def __init__(self) -> None: + self.providers: dict[ProviderKind, ProviderConfig] = {} + + def register(self, config: ProviderConfig) -> None: + self.providers[config.kind] = config + + def get(self, kind: ProviderKind | str) -> ProviderConfig | None: + if isinstance(kind, str): + try: + kind = ProviderKind(kind) + except ValueError: + return None + return self.providers.get(kind) + + def enabled_providers(self) -> list[ProviderConfig]: + return sorted( + (p for p in self.providers.values() if p.enabled), + key=lambda p: p.priority, + ) + + def best_configured( + self, env: dict[str, str] | None = None + ) -> ProviderConfig | None: + for provider in self.enabled_providers(): + if provider.is_configured(env): + return provider + return None + + +def default_registry() -> ProviderRegistry: + """Registry with the standard provider stack.""" + registry = ProviderRegistry() + registry.register( + ProviderConfig( + kind=ProviderKind.ALPACA, + priority=10, + env_keys=("ALPACA_API_KEY", "ALPACA_SECRET_KEY"), + ) + ) + registry.register( + ProviderConfig( + kind=ProviderKind.SCHWAB, + priority=20, + env_keys=("SCHWAB_MARKET_DATA_APP_KEY", "SCHWAB_MARKET_DATA_APP_SECRET"), + ) + ) + registry.register( + ProviderConfig( + kind=ProviderKind.POLYGON, priority=30, env_keys=("POLYGON_API_KEY",) + ) + ) + registry.register( + ProviderConfig( + kind=ProviderKind.FINNHUB, priority=40, env_keys=("FINNHUB_API_KEY",) + ) + ) + registry.register( + ProviderConfig( + kind=ProviderKind.SYNTHETIC, + priority=1000, + env_keys=(), + supports_quotes=False, + ) + ) + return registry diff --git a/runtime/watcher.py b/runtime/watcher.py new file mode 100644 index 0000000..957b84a --- /dev/null +++ b/runtime/watcher.py @@ -0,0 +1,274 @@ +"""Intraday watcher (Milestone 4). + +Pulls candidates from a watchlist provider, evaluates each with the pure +strategy engine, drives a per-symbol state machine, and emits canonical +events into the event store: + + discovered -> watching -> ready | blocked (re-evaluated each tick) + +Signals are debounced: a symbol emits signal_ready at most once per session +unless it falls back to blocked first. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from datetime import datetime, date +from typing import Protocol + +import pandas as pd + +from storage.event_schema import ( + CriteriaEvaluatedEvent, + EventMode, + SignalBlockedEvent, + SignalReadyEvent, + SymbolDiscoveredEvent, + SymbolStateChangedEvent, +) +from storage.event_store import EventStore +from strategy.evaluation.setup_evaluator import evaluate_setup + +logger = logging.getLogger(__name__) + + +@dataclass +class WatchCandidate: + """One symbol the watcher should evaluate.""" + + symbol: str + last_price: float | None = None + previous_close: float | None = None + avg_daily_volume: float | None = None + source: str = "research" + + +class WatchlistProvider(Protocol): + """Source of candidates and their session bars.""" + + def get_candidates(self, session_date: date) -> list[WatchCandidate]: ... + + def get_bars(self, symbol: str, session_date: date) -> pd.DataFrame: ... + + +@dataclass +class WatcherConfig: + session_id: str = "" + mode: EventMode = EventMode.PAPER + max_symbols: int = 25 + min_bars: int = 10 + ready_score_pct: float = 60.0 + min_quality: float = 0.30 + + +@dataclass +class WatcherTickResult: + session_date: str + evaluated: int = 0 + ready: list[str] = field(default_factory=list) + blocked: list[str] = field(default_factory=list) + discovered: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + +class Watcher: + """Event-emitting intraday watcher.""" + + def __init__( + self, + store: EventStore, + provider: WatchlistProvider, + config: WatcherConfig | None = None, + ): + self.store = store + self.provider = provider + self.config = config or WatcherConfig() + if not self.config.session_id: + self.config.session_id = f"session-{date.today().isoformat()}" + # symbol -> current state for this session + self._states: dict[str, str] = {} + + # -- state helpers ------------------------------------------------- + + def _transition(self, symbol: str, new_state: str, reason: str | None) -> None: + previous = self._states.get(symbol) + if previous == new_state: + return + self._states[symbol] = new_state + self.store.emit( + SymbolStateChangedEvent( + timestamp=datetime.now(), + mode=self.config.mode, + correlation_id=self.config.session_id, + message=f"{symbol}: {previous or 'new'} -> {new_state}" + + (f" ({reason})" if reason else ""), + symbol=symbol, + previous_state=previous, + new_state=new_state, + state_reason=reason, + payload={"symbol": symbol, "previous_state": previous, + "new_state": new_state, "state_reason": reason}, + ) + ) + + def _discover(self, candidate: WatchCandidate) -> None: + if candidate.symbol in self._states: + return + self.store.emit( + SymbolDiscoveredEvent( + timestamp=datetime.now(), + mode=self.config.mode, + correlation_id=self.config.session_id, + message=f"Discovered {candidate.symbol} via {candidate.source}", + symbol=candidate.symbol, + symbol_data={ + "last_price": candidate.last_price, + "previous_close": candidate.previous_close, + "avg_daily_volume": candidate.avg_daily_volume, + "source": candidate.source, + }, + payload={"symbol": candidate.symbol, "source": candidate.source}, + ) + ) + self._transition(candidate.symbol, "watching", "added to watchlist") + + # -- main loop body ------------------------------------------------ + + def tick(self, session_date: date | None = None) -> WatcherTickResult: + """One evaluation pass over all candidates.""" + session_date = session_date or date.today() + result = WatcherTickResult(session_date=session_date.isoformat()) + + try: + candidates = self.provider.get_candidates(session_date) + except Exception as exc: # provider failure should not kill the loop + logger.exception("watchlist provider failed") + result.errors.append(f"provider: {exc}") + return result + + for candidate in candidates[: self.config.max_symbols]: + symbol = candidate.symbol + try: + if symbol not in self._states: + self._discover(candidate) + result.discovered.append(symbol) + + bars = self.provider.get_bars(symbol, session_date) + if bars is None or len(bars) < self.config.min_bars: + self._transition( + symbol, "watching", + f"waiting for data ({0 if bars is None else len(bars)} bars)", + ) + continue + + # Evaluate the setup as of the latest bar's timestamp, not the + # wall clock. Minute bars are stored UTC-naive, but the entry + # cutoff reasons in US/Eastern wall-clock, so convert. This + # keeps replay and backfilled data honest (a 09:40 ET bar is + # judged against the 09:40 cutoff regardless of run time). + eval_time = None + try: + from datetime import timezone as _tz + from zoneinfo import ZoneInfo as _ZI + + last_ts = pd.Timestamp(bars["timestamp"].iloc[-1]).to_pydatetime() + if last_ts.tzinfo is None: + last_ts = last_ts.replace(tzinfo=_tz.utc) + eval_time = last_ts.astimezone(_ZI("America/New_York")).replace( + tzinfo=None + ) + except Exception: # noqa: BLE001 - fall back to now() + eval_time = None + + evaluation = evaluate_setup( + bars, + previous_close=candidate.previous_close, + avg_daily_volume=candidate.avg_daily_volume, + ready_score_pct=self.config.ready_score_pct, + evaluation_time=eval_time, + ) + result.evaluated += 1 + + self.store.emit( + CriteriaEvaluatedEvent( + timestamp=datetime.now(), + mode=self.config.mode, + correlation_id=self.config.session_id, + message=( + f"{symbol} criteria {evaluation.criteria_passed}/" + f"{evaluation.criteria_total} " + f"({evaluation.success_score_pct:.0f}%)" + ), + symbol=symbol, + criteria_results={ + "passed": evaluation.criteria_names_passed, + "failed": evaluation.criteria_names_failed, + }, + total_criteria=evaluation.criteria_total, + passed_criteria=evaluation.criteria_passed, + success_score_pct=evaluation.success_score_pct, + payload={ + "symbol": symbol, + "success_score_pct": evaluation.success_score_pct, + "status": evaluation.status, + }, + ) + ) + + if evaluation.status == "ready" and evaluation.setups: + result.ready.append(symbol) + if self._states.get(symbol) != "ready": + setup = evaluation.setups[0] + self._transition(symbol, "ready", setup["setup_type"]) + self.store.emit( + SignalReadyEvent( + timestamp=datetime.now(), + mode=self.config.mode, + correlation_id=self.config.session_id, + message=( + f"{symbol} READY {setup['setup_type']} " + f"entry {setup['entry_price']} stop " + f"{setup['stop_loss_price']}" + ), + symbol=symbol, + signal_type=setup["setup_type"], + confidence=setup["confidence"], + signal_data=setup, + payload={ + "symbol": symbol, + "signal_type": setup["setup_type"], + "confidence": setup["confidence"], + "signal_data": setup, + }, + ) + ) + else: + status = "blocked" if evaluation.status != "late" else "late" + result.blocked.append(symbol) + if self._states.get(symbol) == "ready": + # fell out of ready — allow re-signal later + self._transition(symbol, status, evaluation.reason) + elif self._states.get(symbol) != status: + self._transition(symbol, status, evaluation.reason) + self.store.emit( + SignalBlockedEvent( + timestamp=datetime.now(), + mode=self.config.mode, + correlation_id=self.config.session_id, + message=f"{symbol} blocked: {evaluation.reason}", + symbol=symbol, + blocking_reason=evaluation.reason or "unknown", + unmet_criteria=evaluation.criteria_names_failed, + payload={ + "symbol": symbol, + "blocking_reason": evaluation.reason, + "unmet_criteria": evaluation.criteria_names_failed, + }, + ) + ) + except Exception as exc: + logger.exception("watcher tick failed for %s", symbol) + result.errors.append(f"{symbol}: {exc}") + + return result diff --git a/schwab/__init__.py b/schwab/__init__.py new file mode 100644 index 0000000..8186d05 --- /dev/null +++ b/schwab/__init__.py @@ -0,0 +1,5 @@ +"""Schwab broker integration (Milestone 3): auth, market data, account sync.""" + +from schwab.settings import SchwabSettings + +__all__ = ["SchwabSettings"] diff --git a/schwab/auth/__init__.py b/schwab/auth/__init__.py new file mode 100644 index 0000000..c6ebf6c --- /dev/null +++ b/schwab/auth/__init__.py @@ -0,0 +1,7 @@ +"""Schwab OAuth2 authentication.""" + +from schwab.auth.token_store import TokenBundle, TokenStore +from schwab.auth.oauth2_flow_manager import OAuth2FlowManager +from schwab.auth.lifecycle import TokenLifecycle + +__all__ = ["TokenBundle", "TokenStore", "OAuth2FlowManager", "TokenLifecycle"] diff --git a/schwab/auth/lifecycle.py b/schwab/auth/lifecycle.py new file mode 100644 index 0000000..ff6f71d --- /dev/null +++ b/schwab/auth/lifecycle.py @@ -0,0 +1,50 @@ +"""Token lifecycle: load, refresh when stale, surface auth status.""" + +from __future__ import annotations + +import logging + +from schwab.auth.oauth2_flow_manager import OAuth2FlowManager +from schwab.auth.token_store import TokenBundle, TokenStore +from schwab.settings import SchwabSettings + +logger = logging.getLogger(__name__) + + +class TokenLifecycle: + def __init__( + self, + settings: SchwabSettings | None = None, + token_store: TokenStore | None = None, + ): + self.settings = settings or SchwabSettings.from_env() + self.token_store = token_store or TokenStore(self.settings.token_path) + self.flow = OAuth2FlowManager(self.settings, self.token_store) + + def status(self) -> dict: + bundle = self.token_store.load() + if bundle is None: + return {"authenticated": False, "reason": "no token file"} + if bundle.is_expired and not bundle.refresh_token: + return {"authenticated": False, "reason": "token expired, no refresh token"} + return { + "authenticated": True, + "expired": bundle.is_expired, + "expires_at": bundle.expires_at, + } + + def get_access_token(self) -> str | None: + """Return a valid access token, refreshing if needed. None if unauth.""" + bundle = self.token_store.load() + if bundle is None: + return None + if not bundle.is_expired: + return bundle.access_token + if not bundle.refresh_token: + return None + try: + refreshed: TokenBundle = self.flow.refresh(bundle.refresh_token) + return refreshed.access_token + except Exception: + logger.exception("schwab token refresh failed") + return None diff --git a/schwab/auth/oauth2_flow_manager.py b/schwab/auth/oauth2_flow_manager.py new file mode 100644 index 0000000..456a7dc --- /dev/null +++ b/schwab/auth/oauth2_flow_manager.py @@ -0,0 +1,81 @@ +"""Schwab OAuth2 authorization-code flow.""" + +from __future__ import annotations + +import base64 +import json +import logging +import urllib.parse +import urllib.request + +from schwab.auth.token_store import TokenBundle, TokenStore +from schwab.settings import SchwabSettings + +logger = logging.getLogger(__name__) + +AUTH_URL = "https://api.schwabapi.com/v1/oauth/authorize" +TOKEN_URL = "https://api.schwabapi.com/v1/oauth/token" + + +class OAuth2FlowManager: + def __init__( + self, + settings: SchwabSettings | None = None, + token_store: TokenStore | None = None, + use_broker_app: bool = True, + ): + self.settings = settings or SchwabSettings.from_env() + self.token_store = token_store or TokenStore(self.settings.token_path) + if use_broker_app and self.settings.has_broker_credentials: + self.app_key = self.settings.broker_app_key + self.app_secret = self.settings.broker_app_secret + else: + self.app_key = self.settings.market_data_app_key + self.app_secret = self.settings.market_data_app_secret + + def build_authorization_url(self, state: str = "momentum") -> str: + params = urllib.parse.urlencode( + { + "client_id": self.app_key, + "redirect_uri": self.settings.redirect_uri, + "response_type": "code", + "state": state, + } + ) + return f"{AUTH_URL}?{params}" + + def _basic_auth_header(self) -> str: + raw = f"{self.app_key}:{self.app_secret}".encode() + return "Basic " + base64.b64encode(raw).decode() + + def _token_request(self, form: dict) -> TokenBundle: + data = urllib.parse.urlencode(form).encode() + req = urllib.request.Request( + TOKEN_URL, + data=data, + headers={ + "Authorization": self._basic_auth_header(), + "Content-Type": "application/x-www-form-urlencoded", + }, + method="POST", + ) + with urllib.request.urlopen(req, timeout=20) as resp: + payload = json.loads(resp.read().decode()) + bundle = TokenBundle.from_oauth_response(payload) + self.token_store.save(bundle) + return bundle + + def exchange_code(self, authorization_code: str) -> TokenBundle: + """Exchange the redirect ?code= for access + refresh tokens.""" + return self._token_request( + { + "grant_type": "authorization_code", + "code": urllib.parse.unquote(authorization_code), + "redirect_uri": self.settings.redirect_uri, + } + ) + + def refresh(self, refresh_token: str) -> TokenBundle: + return self._token_request( + {"grant_type": "refresh_token", "refresh_token": refresh_token} + ) diff --git a/schwab/auth/token_store.py b/schwab/auth/token_store.py new file mode 100644 index 0000000..28c40de --- /dev/null +++ b/schwab/auth/token_store.py @@ -0,0 +1,71 @@ +"""Token persistence for Schwab OAuth2 tokens. + +SECURITY: token files contain live credentials. They are written 0600 and +data/schwab_tokens.json is git-ignored. Never commit token files. +""" + +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, asdict +from pathlib import Path + + +@dataclass +class TokenBundle: + access_token: str = "" + refresh_token: str = "" + expires_at: float = 0.0 # unix epoch seconds + token_type: str = "Bearer" + scope: str = "" + + @property + def is_expired(self) -> bool: + return not self.access_token or time.time() >= self.expires_at - 60 + + @classmethod + def from_oauth_response(cls, payload: dict) -> "TokenBundle": + return cls( + access_token=payload.get("access_token", ""), + refresh_token=payload.get("refresh_token", ""), + expires_at=time.time() + float(payload.get("expires_in", 1800)), + token_type=payload.get("token_type", "Bearer"), + scope=payload.get("scope", ""), + ) + + +class TokenStore: + def __init__(self, path: str | Path = "data/schwab_tokens.json"): + self.path = Path(path) + + def exists(self) -> bool: + return self.path.exists() + + def load(self) -> TokenBundle | None: + if not self.path.exists(): + return None + try: + data = json.loads(self.path.read_text()) + except (json.JSONDecodeError, OSError): + return None + return TokenBundle( + access_token=data.get("access_token", ""), + refresh_token=data.get("refresh_token", ""), + expires_at=float(data.get("expires_at", 0)), + token_type=data.get("token_type", "Bearer"), + scope=data.get("scope", ""), + ) + + def save(self, bundle: TokenBundle) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(json.dumps(asdict(bundle), indent=2)) + try: + os.chmod(self.path, 0o600) + except OSError: + pass + + def clear(self) -> None: + if self.path.exists(): + self.path.unlink() diff --git a/schwab/health/__init__.py b/schwab/health/__init__.py new file mode 100644 index 0000000..9aeb0a7 --- /dev/null +++ b/schwab/health/__init__.py @@ -0,0 +1,6 @@ +"""Schwab broker health.""" + +from schwab.health.models import HealthStatus, HealthCheck, HealthReport +from schwab.health.reporter import HealthReporter + +__all__ = ["HealthStatus", "HealthCheck", "HealthReport", "HealthReporter"] diff --git a/schwab/health/models.py b/schwab/health/models.py new file mode 100644 index 0000000..9607fe4 --- /dev/null +++ b/schwab/health/models.py @@ -0,0 +1,37 @@ +"""Broker health models.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + + +class HealthStatus(str, Enum): + HEALTHY = "healthy" + DEGRADED = "degraded" + DOWN = "down" + UNAUTHENTICATED = "unauthenticated" + + +@dataclass +class HealthCheck: + name: str + status: HealthStatus + detail: str | None = None + + +@dataclass +class HealthReport: + broker_name: str + status: HealthStatus + checks: list[HealthCheck] = field(default_factory=list) + + def to_dict(self) -> dict: + return { + "broker_name": self.broker_name, + "status": self.status.value, + "checks": [ + {"name": c.name, "status": c.status.value, "detail": c.detail} + for c in self.checks + ], + } diff --git a/schwab/health/reporter.py b/schwab/health/reporter.py new file mode 100644 index 0000000..c4c1e68 --- /dev/null +++ b/schwab/health/reporter.py @@ -0,0 +1,96 @@ +"""Schwab broker health reporter — emits broker_health_changed events.""" + +from __future__ import annotations + +import logging +from datetime import datetime + +from schwab.auth.lifecycle import TokenLifecycle +from schwab.health.models import HealthCheck, HealthReport, HealthStatus +from schwab.settings import SchwabSettings +from storage.event_schema import BrokerHealthChangedEvent, EventMode +from storage.event_store import EventStore + +logger = logging.getLogger(__name__) + + +class HealthReporter: + def __init__( + self, + store: EventStore | None = None, + settings: SchwabSettings | None = None, + lifecycle: TokenLifecycle | None = None, + session_id: str | None = None, + ): + self.store = store + self.settings = settings or SchwabSettings.from_env() + self.lifecycle = lifecycle or TokenLifecycle(self.settings) + self.session_id = session_id + self._last_status: HealthStatus | None = None + + def check(self) -> HealthReport: + checks: list[HealthCheck] = [] + if self.settings.has_broker_credentials or self.settings.has_market_data_credentials: + checks.append(HealthCheck("credentials", HealthStatus.HEALTHY)) + else: + checks.append( + HealthCheck( + "credentials", + HealthStatus.DOWN, + "no SCHWAB_* app keys configured", + ) + ) + auth = self.lifecycle.status() + if auth.get("authenticated") and not auth.get("expired"): + checks.append(HealthCheck("token", HealthStatus.HEALTHY)) + elif auth.get("authenticated"): + checks.append( + HealthCheck("token", HealthStatus.DEGRADED, "token expired; refresh pending") + ) + else: + checks.append( + HealthCheck( + "token", HealthStatus.UNAUTHENTICATED, + str(auth.get("reason") or "not authenticated"), + ) + ) + + statuses = {c.status for c in checks} + if statuses == {HealthStatus.HEALTHY}: + overall = HealthStatus.HEALTHY + elif HealthStatus.DOWN in statuses: + overall = HealthStatus.DOWN + elif HealthStatus.UNAUTHENTICATED in statuses: + overall = HealthStatus.UNAUTHENTICATED + else: + overall = HealthStatus.DEGRADED + + report = HealthReport(broker_name="schwab", status=overall, checks=checks) + self._maybe_emit(report) + return report + + def _maybe_emit(self, report: HealthReport) -> None: + if self.store is None or report.status == self._last_status: + self._last_status = report.status + return + try: + self.store.emit( + BrokerHealthChangedEvent( + timestamp=datetime.now(), + mode=EventMode.LIVE, + correlation_id=self.session_id, + message=f"Schwab health: {report.status.value}", + broker_name="schwab", + previous_health=self._last_status.value + if self._last_status + else "unknown", + new_health=report.status.value, + health_reason="; ".join( + f"{c.name}={c.status.value}" for c in report.checks + ), + payload=report.to_dict(), + ) + ) + except Exception: + logger.exception("failed to emit broker health event") + self._last_status = report.status diff --git a/schwab/market/__init__.py b/schwab/market/__init__.py new file mode 100644 index 0000000..ce33f84 --- /dev/null +++ b/schwab/market/__init__.py @@ -0,0 +1,24 @@ +"""Schwab market data.""" + +from schwab.market.models import Quote, Candle, PriceHistory, OptionQuote +from schwab.market.client import SchwabMarketClient, SchwabApiError +from schwab.market.adapter import ( + quotes_from_payload, + price_history_to_dataframe, + payload_to_dataframe, +) +from schwab.market.option_chain_service import OptionChainService, OptionChainResult + +__all__ = [ + "Quote", + "Candle", + "PriceHistory", + "OptionQuote", + "SchwabMarketClient", + "SchwabApiError", + "quotes_from_payload", + "price_history_to_dataframe", + "payload_to_dataframe", + "OptionChainService", + "OptionChainResult", +] diff --git a/schwab/market/adapter.py b/schwab/market/adapter.py new file mode 100644 index 0000000..f4b2f43 --- /dev/null +++ b/schwab/market/adapter.py @@ -0,0 +1,34 @@ +"""Adapter: Schwab market payloads -> typed models / DataFrames.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pandas as pd + +from schwab.market.models import PriceHistory, Quote + + +def quotes_from_payload(payload: dict) -> list[Quote]: + return [Quote.from_api(symbol, body) for symbol, body in payload.items()] + + +def price_history_to_dataframe(history: PriceHistory) -> pd.DataFrame: + rows = [ + { + "timestamp": datetime.fromtimestamp( + c.datetime_ms / 1000, tz=timezone.utc + ).replace(tzinfo=None), + "open": c.open, + "high": c.high, + "low": c.low, + "close": c.close, + "volume": c.volume, + } + for c in history.candles + ] + return pd.DataFrame(rows) + + +def payload_to_dataframe(payload: dict) -> pd.DataFrame: + return price_history_to_dataframe(PriceHistory.from_api(payload)) diff --git a/schwab/market/client.py b/schwab/market/client.py new file mode 100644 index 0000000..1767c1b --- /dev/null +++ b/schwab/market/client.py @@ -0,0 +1,83 @@ +"""Schwab market-data HTTP client (quotes + price history).""" + +from __future__ import annotations + +import json +import logging +import urllib.error +import urllib.parse +import urllib.request + +from schwab.auth.lifecycle import TokenLifecycle +from schwab.settings import SchwabSettings + +logger = logging.getLogger(__name__) + + +class SchwabApiError(RuntimeError): + def __init__(self, status: int, body: str): + super().__init__(f"Schwab API error {status}: {body[:300]}") + self.status = status + self.body = body + + +class SchwabMarketClient: + def __init__( + self, + settings: SchwabSettings | None = None, + lifecycle: TokenLifecycle | None = None, + timeout: int = 15, + ): + self.settings = settings or SchwabSettings.from_env() + self.lifecycle = lifecycle or TokenLifecycle(self.settings) + self.timeout = timeout + + def _get(self, path: str, params: dict | None = None) -> dict: + token = self.lifecycle.get_access_token() + if not token: + raise SchwabApiError(401, "no valid Schwab access token") + url = f"{self.settings.api_base_url}{path}" + if params: + url += "?" + urllib.parse.urlencode( + {k: v for k, v in params.items() if v is not None} + ) + req = urllib.request.Request( + url, headers={"Authorization": f"Bearer {token}"} + ) + try: + with urllib.request.urlopen(req, timeout=self.timeout) as resp: + return json.loads(resp.read().decode() or "{}") + except urllib.error.HTTPError as exc: + raise SchwabApiError(exc.code, exc.read().decode(errors="replace")) from exc + + def get_quotes(self, symbols: list[str]) -> dict: + return self._get( + "/marketdata/v1/quotes", params={"symbols": ",".join(symbols)} + ) + + def get_price_history( + self, + symbol: str, + period_type: str = "day", + period: int = 1, + frequency_type: str = "minute", + frequency: int = 1, + need_extended_hours: bool = True, + ) -> dict: + return self._get( + "/marketdata/v1/pricehistory", + params={ + "symbol": symbol, + "periodType": period_type, + "period": period, + "frequencyType": frequency_type, + "frequency": frequency, + "needExtendedHoursData": str(need_extended_hours).lower(), + }, + ) + + def get_option_chain(self, symbol: str, contract_type: str = "ALL") -> dict: + return self._get( + "/marketdata/v1/chains", + params={"symbol": symbol, "contractType": contract_type}, + ) diff --git a/schwab/market/models.py b/schwab/market/models.py new file mode 100644 index 0000000..4631e22 --- /dev/null +++ b/schwab/market/models.py @@ -0,0 +1,86 @@ +"""Schwab market-data models. + +(Repaired: the original had broken dataclass indentation, a `string` type +annotation, and an invalid `int = None = None` default chain.) +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class Quote: + symbol: str + bid_price: float | None = None + ask_price: float | None = None + last_price: float | None = None + bid_size: int | None = None + ask_size: int | None = None + total_volume: int | None = None + quote_time: str | None = None + + @classmethod + def from_api(cls, symbol: str, payload: dict) -> "Quote": + quote = payload.get("quote") or payload + return cls( + symbol=symbol, + bid_price=quote.get("bidPrice"), + ask_price=quote.get("askPrice"), + last_price=quote.get("lastPrice"), + bid_size=quote.get("bidSize"), + ask_size=quote.get("askSize"), + total_volume=quote.get("totalVolume"), + quote_time=str(quote.get("quoteTime")) if quote.get("quoteTime") else None, + ) + + +@dataclass +class Candle: + open: float + high: float + low: float + close: float + volume: int + datetime_ms: int + + @classmethod + def from_api(cls, payload: dict) -> "Candle": + return cls( + open=float(payload.get("open") or 0), + high=float(payload.get("high") or 0), + low=float(payload.get("low") or 0), + close=float(payload.get("close") or 0), + volume=int(payload.get("volume") or 0), + datetime_ms=int(payload.get("datetime") or 0), + ) + + +@dataclass +class PriceHistory: + symbol: str + candles: list[Candle] = field(default_factory=list) + empty: bool = True + + @classmethod + def from_api(cls, payload: dict) -> "PriceHistory": + candles = [Candle.from_api(c) for c in payload.get("candles") or []] + return cls( + symbol=payload.get("symbol", ""), + candles=candles, + empty=payload.get("empty", not candles), + ) + + +@dataclass +class OptionQuote: + symbol: str + strike: float | None = None + expiration: str | None = None + put_call: str | None = None + bid: float | None = None + ask: float | None = None + last: float | None = None + delta: float | None = None + open_interest: int | None = None + volume: int | None = None diff --git a/schwab/market/option_chain_service.py b/schwab/market/option_chain_service.py new file mode 100644 index 0000000..2253bc0 --- /dev/null +++ b/schwab/market/option_chain_service.py @@ -0,0 +1,63 @@ +"""Option chain retrieval service. + +(Repaired: original had dataclass default-ordering errors and unbalanced +parentheses.) +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field + +from schwab.market.client import SchwabApiError, SchwabMarketClient +from schwab.market.models import OptionQuote + +logger = logging.getLogger(__name__) + + +@dataclass +class OptionChainResult: + symbol: str + underlying_price: float | None = None + calls: list[OptionQuote] = field(default_factory=list) + puts: list[OptionQuote] = field(default_factory=list) + error: str | None = None + + +class OptionChainService: + def __init__(self, client: SchwabMarketClient | None = None): + self.client = client or SchwabMarketClient() + + def get_chain(self, symbol: str) -> OptionChainResult: + try: + payload = self.client.get_option_chain(symbol) + except SchwabApiError as exc: + return OptionChainResult(symbol=symbol, error=str(exc)) + result = OptionChainResult( + symbol=symbol, underlying_price=payload.get("underlyingPrice") + ) + result.calls = self._parse_side(payload.get("callExpDateMap") or {}, "CALL") + result.puts = self._parse_side(payload.get("putExpDateMap") or {}, "PUT") + return result + + @staticmethod + def _parse_side(exp_map: dict, put_call: str) -> list[OptionQuote]: + quotes: list[OptionQuote] = [] + for expiration, strikes in exp_map.items(): + for strike, contracts in (strikes or {}).items(): + for contract in contracts or []: + quotes.append( + OptionQuote( + symbol=contract.get("symbol", ""), + strike=float(strike), + expiration=expiration.split(":")[0], + put_call=put_call, + bid=contract.get("bid"), + ask=contract.get("ask"), + last=contract.get("last"), + delta=contract.get("delta"), + open_interest=contract.get("openInterest"), + volume=contract.get("totalVolume"), + ) + ) + return quotes diff --git a/schwab/orders/__init__.py b/schwab/orders/__init__.py new file mode 100644 index 0000000..1b0149d --- /dev/null +++ b/schwab/orders/__init__.py @@ -0,0 +1,6 @@ +"""Schwab orders (read-only).""" + +from schwab.orders.reader import OrdersReader +from schwab.orders.sync import OrderSync + +__all__ = ["OrdersReader", "OrderSync"] diff --git a/schwab/orders/reader.py b/schwab/orders/reader.py new file mode 100644 index 0000000..a233217 --- /dev/null +++ b/schwab/orders/reader.py @@ -0,0 +1,67 @@ +"""Read Schwab orders (read-only, with unauthenticated fallback).""" + +from __future__ import annotations + +import json +import logging +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timedelta, timezone + +from schwab.auth.lifecycle import TokenLifecycle +from schwab.settings import SchwabSettings + +logger = logging.getLogger(__name__) + + +class OrdersReader: + def __init__( + self, + settings: SchwabSettings | None = None, + lifecycle: TokenLifecycle | None = None, + timeout: int = 15, + ): + self.settings = settings or SchwabSettings.from_env() + self.lifecycle = lifecycle or TokenLifecycle(self.settings) + self.timeout = timeout + + def read_orders(self, days_back: int = 2) -> list[dict]: + token = self.lifecycle.get_access_token() + if not token: + return [] + now = datetime.now(timezone.utc) + params = urllib.parse.urlencode( + { + "fromEnteredTime": (now - timedelta(days=days_back)).strftime( + "%Y-%m-%dT%H:%M:%S.000Z" + ), + "toEnteredTime": now.strftime("%Y-%m-%dT%H:%M:%S.000Z"), + } + ) + url = f"{self.settings.api_base_url}/trader/v1/orders?{params}" + req = urllib.request.Request( + url, headers={"Authorization": f"Bearer {token}"} + ) + try: + with urllib.request.urlopen(req, timeout=self.timeout) as resp: + raw = json.loads(resp.read().decode() or "[]") + except (urllib.error.URLError, OSError, json.JSONDecodeError): + logger.exception("schwab orders read failed") + return [] + return [ + { + "broker_order_id": str(o.get("orderId") or ""), + "symbol": (o.get("orderLegCollection") or [{}])[0] + .get("instrument", {}) + .get("symbol"), + "side": (o.get("orderLegCollection") or [{}])[0].get("instruction"), + "quantity": o.get("quantity"), + "filled_quantity": o.get("filledQuantity"), + "status": o.get("status"), + "type": o.get("orderType"), + "price": o.get("price"), + "submitted_at": o.get("enteredTime"), + } + for o in raw + ] diff --git a/schwab/orders/sync.py b/schwab/orders/sync.py new file mode 100644 index 0000000..c7ce496 --- /dev/null +++ b/schwab/orders/sync.py @@ -0,0 +1,45 @@ +"""Sync Schwab orders into the event store.""" + +from __future__ import annotations + +import logging +from datetime import datetime + +from schwab.orders.reader import OrdersReader +from storage.event_schema import AccountOrdersUpdatedEvent, EventMode +from storage.event_store import EventStore + +logger = logging.getLogger(__name__) + +BROKER_NAME = "schwab" + + +class OrderSync: + def __init__( + self, + store: EventStore, + reader: OrdersReader | None = None, + session_id: str | None = None, + mode: EventMode = EventMode.LIVE, + account_id: str = "schwab", + ): + self.store = store + self.reader = reader or OrdersReader() + self.session_id = session_id + self.mode = mode + self.account_id = account_id + + def sync_orders(self) -> list[dict]: + orders = self.reader.read_orders() + self.store.emit( + AccountOrdersUpdatedEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=f"Schwab orders: {len(orders)}", + broker_name=BROKER_NAME, + account_id=self.account_id, + orders=orders, + ) + ) + return orders diff --git a/schwab/positions/__init__.py b/schwab/positions/__init__.py new file mode 100644 index 0000000..8a22a04 --- /dev/null +++ b/schwab/positions/__init__.py @@ -0,0 +1,13 @@ +"""Schwab account & positions (read-only).""" + +from schwab.positions.models import AccountSummary, AccountPositions, Position +from schwab.positions.reader import PositionsReader +from schwab.positions.sync import AccountSync + +__all__ = [ + "AccountSummary", + "AccountPositions", + "Position", + "PositionsReader", + "AccountSync", +] diff --git a/schwab/positions/models.py b/schwab/positions/models.py new file mode 100644 index 0000000..26d6570 --- /dev/null +++ b/schwab/positions/models.py @@ -0,0 +1,33 @@ +"""Schwab account/position models.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class AccountSummary: + account_id: str + account_desc: str = "Schwab" + total_equity: float = 0.0 + cash_balance: float = 0.0 + buying_power: float = 0.0 + net_liquidating_value: float = 0.0 + is_fallback: bool = False + + +@dataclass +class Position: + symbol: str + quantity: float = 0.0 + avg_entry_price: float = 0.0 + market_value: float = 0.0 + unrealized_pnl: float = 0.0 + asset_type: str = "EQUITY" + + +@dataclass +class AccountPositions: + account_id: str + positions: list[Position] = field(default_factory=list) + is_fallback: bool = False diff --git a/schwab/positions/reader.py b/schwab/positions/reader.py new file mode 100644 index 0000000..acd93b7 --- /dev/null +++ b/schwab/positions/reader.py @@ -0,0 +1,111 @@ +"""Read Schwab account summary and positions. + +When no valid token exists (the common case before OAuth setup), the reader +returns clearly-labelled fallback data rather than crashing, so the rest of +the system — and its tests — keep working. +""" + +from __future__ import annotations + +import json +import logging +import urllib.error +import urllib.parse +import urllib.request + +from schwab.auth.lifecycle import TokenLifecycle +from schwab.positions.models import AccountPositions, AccountSummary, Position +from schwab.settings import SchwabSettings + +logger = logging.getLogger(__name__) + + +class PositionsReader: + def __init__( + self, + settings: SchwabSettings | None = None, + lifecycle: TokenLifecycle | None = None, + timeout: int = 15, + ): + self.settings = settings or SchwabSettings.from_env() + self.lifecycle = lifecycle or TokenLifecycle(self.settings) + self.timeout = timeout + + def _get(self, path: str, token: str, params: dict | None = None): + url = f"{self.settings.api_base_url}{path}" + if params: + url += "?" + urllib.parse.urlencode(params) + req = urllib.request.Request( + url, headers={"Authorization": f"Bearer {token}"} + ) + with urllib.request.urlopen(req, timeout=self.timeout) as resp: + return json.loads(resp.read().decode() or "{}") + + # -- fallbacks ------------------------------------------------------- + + @staticmethod + def _fallback_account_summary() -> AccountSummary: + return AccountSummary( + account_id="SCHWAB-UNAUTH", + account_desc="Schwab (not authenticated — fallback data)", + is_fallback=True, + ) + + @staticmethod + def _fallback_positions() -> AccountPositions: + return AccountPositions(account_id="SCHWAB-UNAUTH", is_fallback=True) + + # -- public ---------------------------------------------------------- + + def read_account_summary(self) -> AccountSummary: + token = self.lifecycle.get_access_token() + if not token: + return self._fallback_account_summary() + try: + accounts = self._get("/trader/v1/accounts", token) + except (urllib.error.URLError, OSError, json.JSONDecodeError): + logger.exception("schwab account read failed") + return self._fallback_account_summary() + if not accounts: + return self._fallback_account_summary() + account = accounts[0].get("securitiesAccount", accounts[0]) + balances = account.get("currentBalances", {}) + return AccountSummary( + account_id=str(account.get("accountNumber") or "schwab"), + account_desc=account.get("type", "Schwab"), + total_equity=float(balances.get("equity") or 0), + cash_balance=float(balances.get("cashBalance") or 0), + buying_power=float(balances.get("buyingPower") or 0), + net_liquidating_value=float(balances.get("liquidationValue") or 0), + ) + + def read_positions(self) -> AccountPositions: + token = self.lifecycle.get_access_token() + if not token: + return self._fallback_positions() + try: + accounts = self._get( + "/trader/v1/accounts", token, params={"fields": "positions"} + ) + except (urllib.error.URLError, OSError, json.JSONDecodeError): + logger.exception("schwab positions read failed") + return self._fallback_positions() + if not accounts: + return self._fallback_positions() + account = accounts[0].get("securitiesAccount", accounts[0]) + positions = [ + Position( + symbol=p.get("instrument", {}).get("symbol", ""), + quantity=float(p.get("longQuantity") or 0) + - float(p.get("shortQuantity") or 0), + avg_entry_price=float(p.get("averagePrice") or 0), + market_value=float(p.get("marketValue") or 0), + unrealized_pnl=float(p.get("longOpenProfitLoss") or 0), + asset_type=p.get("instrument", {}).get("assetType", "EQUITY"), + ) + for p in account.get("positions") or [] + ] + return AccountPositions( + account_id=str(account.get("accountNumber") or "schwab"), + positions=positions, + ) diff --git a/schwab/positions/sync.py b/schwab/positions/sync.py new file mode 100644 index 0000000..75fa6e9 --- /dev/null +++ b/schwab/positions/sync.py @@ -0,0 +1,70 @@ +"""Sync Schwab account state into the event store.""" + +from __future__ import annotations + +import logging +from dataclasses import asdict +from datetime import datetime + +from schwab.positions.reader import PositionsReader +from storage.event_schema import ( + AccountPositionsUpdatedEvent, + AccountSummaryUpdatedEvent, + EventMode, +) +from storage.event_store import EventStore + +logger = logging.getLogger(__name__) + +BROKER_NAME = "schwab" + + +class AccountSync: + def __init__( + self, + store: EventStore, + reader: PositionsReader | None = None, + session_id: str | None = None, + mode: EventMode = EventMode.LIVE, + ): + self.store = store + self.reader = reader or PositionsReader() + self.session_id = session_id + self.mode = mode + + def sync_summary(self) -> dict: + summary = self.reader.read_account_summary() + self.store.emit( + AccountSummaryUpdatedEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=f"Schwab account {summary.account_id} equity {summary.total_equity}", + broker_name=BROKER_NAME, + account_id=summary.account_id, + account_desc=summary.account_desc, + total_equity=summary.total_equity, + cash_balance=summary.cash_balance, + buying_power=summary.buying_power, + net_liquidating_value=summary.net_liquidating_value, + ) + ) + return asdict(summary) + + def sync_positions(self) -> dict: + snapshot = self.reader.read_positions() + self.store.emit( + AccountPositionsUpdatedEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=f"Schwab positions: {len(snapshot.positions)}", + broker_name=BROKER_NAME, + account_id=snapshot.account_id, + positions=[asdict(p) for p in snapshot.positions], + ) + ) + return asdict(snapshot) + + def sync_all(self) -> dict: + return {"summary": self.sync_summary(), "positions": self.sync_positions()} diff --git a/schwab/settings.py b/schwab/settings.py new file mode 100644 index 0000000..ac2e250 --- /dev/null +++ b/schwab/settings.py @@ -0,0 +1,41 @@ +"""Schwab API settings (Milestone 3).""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SchwabSettings: + market_data_app_key: str = "" + market_data_app_secret: str = "" + broker_app_key: str = "" + broker_app_secret: str = "" + redirect_uri: str = "https://127.0.0.1:8182/callback" + token_path: str = "data/schwab_tokens.json" + api_base_url: str = "https://api.schwabapi.com" + + @classmethod + def from_env(cls, env: dict[str, str] | None = None) -> "SchwabSettings": + values = dict(os.environ) + if env is not None: + values.update(env) + return cls( + market_data_app_key=values.get("SCHWAB_MARKET_DATA_APP_KEY", ""), + market_data_app_secret=values.get("SCHWAB_MARKET_DATA_APP_SECRET", ""), + broker_app_key=values.get("SCHWAB_BROKER_APP_KEY", ""), + broker_app_secret=values.get("SCHWAB_BROKER_APP_SECRET", ""), + redirect_uri=values.get( + "SCHWAB_REDIRECT_URI", "https://127.0.0.1:8182/callback" + ), + token_path=values.get("SCHWAB_TOKEN_PATH", "data/schwab_tokens.json"), + ) + + @property + def has_market_data_credentials(self) -> bool: + return bool(self.market_data_app_key and self.market_data_app_secret) + + @property + def has_broker_credentials(self) -> bool: + return bool(self.broker_app_key and self.broker_app_secret) diff --git a/schwab/streaming/__init__.py b/schwab/streaming/__init__.py new file mode 100644 index 0000000..815ccd7 --- /dev/null +++ b/schwab/streaming/__init__.py @@ -0,0 +1,5 @@ +"""Schwab streaming (scaffold).""" + +from schwab.streaming.client import StreamingClient, StreamerInfo + +__all__ = ["StreamingClient", "StreamerInfo"] diff --git a/schwab/streaming/client.py b/schwab/streaming/client.py new file mode 100644 index 0000000..f6e2cd1 --- /dev/null +++ b/schwab/streaming/client.py @@ -0,0 +1,87 @@ +"""Schwab streaming client scaffold. + +(Repaired: original had dataclass default-ordering errors and unbalanced +parentheses.) Real-time streaming requires the streamer-info handshake from +user preferences; this scaffold builds the login payload and exposes a +callback interface, while the live system polls REST minute bars instead. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from typing import Callable + +logger = logging.getLogger(__name__) + + +@dataclass +class StreamerInfo: + streamer_socket_url: str = "" + schwab_client_customer_id: str = "" + schwab_client_correl_id: str = "" + schwab_client_channel: str = "N9" + schwab_client_function_id: str = "APIAPP" + + @classmethod + def from_user_preferences(cls, payload: dict) -> "StreamerInfo": + info = (payload.get("streamerInfo") or [{}])[0] + return cls( + streamer_socket_url=info.get("streamerSocketUrl", ""), + schwab_client_customer_id=info.get("schwabClientCustomerId", ""), + schwab_client_correl_id=info.get("schwabClientCorrelId", ""), + schwab_client_channel=info.get("schwabClientChannel", "N9"), + schwab_client_function_id=info.get("schwabClientFunctionId", "APIAPP"), + ) + + +@dataclass +class StreamingClient: + streamer_info: StreamerInfo + access_token: str + request_id: int = 0 + on_message: Callable[[dict], None] | None = None + subscriptions: list[dict] = field(default_factory=list) + + def _next_request_id(self) -> int: + self.request_id += 1 + return self.request_id + + def build_login_request(self) -> dict: + return { + "service": "ADMIN", + "command": "LOGIN", + "requestid": self._next_request_id(), + "SchwabClientCustomerId": self.streamer_info.schwab_client_customer_id, + "SchwabClientCorrelId": self.streamer_info.schwab_client_correl_id, + "parameters": { + "Authorization": self.access_token, + "SchwabClientChannel": self.streamer_info.schwab_client_channel, + "SchwabClientFunctionId": self.streamer_info.schwab_client_function_id, + }, + } + + def build_quote_subscription(self, symbols: list[str]) -> dict: + request = { + "service": "LEVELONE_EQUITIES", + "command": "SUBS", + "requestid": self._next_request_id(), + "SchwabClientCustomerId": self.streamer_info.schwab_client_customer_id, + "SchwabClientCorrelId": self.streamer_info.schwab_client_correl_id, + "parameters": { + "keys": ",".join(symbols), + "fields": "0,1,2,3,4,5,8", + }, + } + self.subscriptions.append(request) + return request + + def handle_raw_message(self, raw: str) -> None: + try: + payload = json.loads(raw) + except json.JSONDecodeError: + logger.warning("unparseable streaming message: %.120s", raw) + return + if self.on_message: + self.on_message(payload) diff --git a/storage/__init__.py b/storage/__init__.py new file mode 100644 index 0000000..3f30d0f --- /dev/null +++ b/storage/__init__.py @@ -0,0 +1,47 @@ +"""Event model for Milestone 2. + +Append-only event store with canonical event types. +""" + +from .event_schema import ( + EventType, + BaseEvent, + SymbolDiscoveredEvent, + SymbolStateChangedEvent, + CriteriaEvaluatedEvent, + SignalReadyEvent, + SignalBlockedEvent, + OrderSubmittedEvent, + OrderApprovalRequestedEvent, + OrderApprovedEvent, + OrderRejectedEvent, + OrderFilledEvent, + PositionOpenedEvent, + PositionClosedEvent, + RiskRuleTriggeredEvent, + BrokerHealthChangedEvent, + SourceHealthChangedEvent, +) + +from .event_store import EventStore + +__all__ = [ + "EventType", + "BaseEvent", + "SymbolDiscoveredEvent", + "SymbolStateChangedEvent", + "CriteriaEvaluatedEvent", + "SignalReadyEvent", + "SignalBlockedEvent", + "OrderSubmittedEvent", + "OrderApprovalRequestedEvent", + "OrderApprovedEvent", + "OrderRejectedEvent", + "OrderFilledEvent", + "PositionOpenedEvent", + "PositionClosedEvent", + "RiskRuleTriggeredEvent", + "BrokerHealthChangedEvent", + "SourceHealthChangedEvent", + "EventStore", +] diff --git a/storage/db.py b/storage/db.py new file mode 100644 index 0000000..8a017a3 --- /dev/null +++ b/storage/db.py @@ -0,0 +1,34 @@ +"""DuckDB connection management.""" + +import importlib +from pathlib import Path +from typing import Any + +from .schema import create_schema + + +def get_connection( + db_path: str | Path = "./data/momentum.duckdb", +) -> Any: + """Get a DuckDB connection, creating schema if needed.""" + duckdb = importlib.import_module("duckdb") + + if str(db_path) == ":memory:": + con = duckdb.connect(":memory:") + create_schema(con) + return con + + db_path = Path(db_path) + db_path.parent.mkdir(parents=True, exist_ok=True) + con = duckdb.connect(str(db_path)) + create_schema(con) + return con + + +def get_memory_connection() -> Any: + """Get an in-memory DuckDB connection for testing.""" + duckdb = importlib.import_module("duckdb") + + con = duckdb.connect(":memory:") + create_schema(con) + return con diff --git a/storage/event_schema.py b/storage/event_schema.py new file mode 100644 index 0000000..ad4abb9 --- /dev/null +++ b/storage/event_schema.py @@ -0,0 +1,344 @@ +"""Event schema definitions for Milestone 2. + +All canonical event types that can be emitted by replay, paper, and live modes. +""" + +from enum import Enum +from datetime import datetime +from pydantic import BaseModel, Field + + +class EventMode(str, Enum): + """Runtime modes for events.""" + + REPLAY = "replay" + PAPER = "paper" + LIVE = "live" + + +class EventType(str, Enum): + """Canonical event types.""" + + SYMBOL_DISCOVERED = "symbol_discovered" + SYMBOL_STATE_CHANGED = "symbol_state_changed" + CRITERIA_EVALUATED = "criteria_evaluated" + SIGNAL_READY = "signal_ready" + SIGNAL_BLOCKED = "signal_blocked" + ORDER_SUBMITTED = "order_submitted" + ORDER_APPROVAL_REQUESTED = "order_approval_requested" + ORDER_APPROVED = "order_approved" + ORDER_REJECTED = "order_rejected" + ORDER_FILLED = "order_filled" + ORDER_CANCELLED = "order_cancelled" + POSITION_OPENED = "position_opened" + POSITION_CLOSED = "position_closed" + ACCOUNT_SUMMARY_UPDATED = "account_summary_updated" + ACCOUNT_POSITIONS_UPDATED = "account_positions_updated" + ACCOUNT_ORDERS_UPDATED = "account_orders_updated" + RISK_RULE_TRIGGERED = "risk_rule_triggered" + BROKER_HEALTH_CHANGED = "broker_health_changed" + SOURCE_HEALTH_CHANGED = "source_health_changed" + SESSION_SUMMARY = "session_summary" + MODULE_TICK = "module_tick" + DATA_VALIDATION = "data_validation" + SYMBOL_EVALUATION_DETAIL = "symbol_evaluation_detail" + + +class BaseEvent(BaseModel): + """Base event with common fields.""" + + timestamp: datetime = Field(description="Event timestamp") + mode: EventMode = Field(description="Runtime mode (replay/paper/live)") + event_type: EventType = Field(description="Event type identifier") + correlation_id: str | None = Field( + default=None, description="Correlation ID for causal chain" + ) + message: str = Field(description="Human-readable message") + payload: dict = Field(default_factory=dict, description="Structured event data") + + +class SymbolDiscoveredEvent(BaseEvent): + """Event when a symbol is discovered.""" + + event_type: EventType = Field( + default=EventType.SYMBOL_DISCOVERED, description="Event type identifier" + ) + symbol: str = Field(description="Ticker symbol") + symbol_data: dict = Field(default_factory=dict, description="Symbol metadata") + + +class SymbolStateChangedEvent(BaseEvent): + """Event when symbol state changes.""" + + event_type: EventType = Field( + default=EventType.SYMBOL_STATE_CHANGED, description="Event type identifier" + ) + symbol: str = Field(description="Ticker symbol") + previous_state: str | None = Field(default=None, description="Previous state") + new_state: str = Field(description="New state") + state_reason: str | None = Field( + default=None, description="Reason for state change" + ) + + +class CriteriaEvaluatedEvent(BaseEvent): + """Event when criteria are evaluated.""" + + event_type: EventType = Field( + default=EventType.CRITERIA_EVALUATED, description="Event type identifier" + ) + symbol: str = Field(description="Ticker symbol") + criteria_results: dict = Field(description="Criteria evaluation results") + total_criteria: int = Field(description="Total number of criteria") + passed_criteria: int = Field(description="Number of criteria passed") + success_score_pct: float = Field(description="Success score percentage") + + +class SignalReadyEvent(BaseEvent): + """Event when signal is ready.""" + + event_type: EventType = Field( + default=EventType.SIGNAL_READY, description="Event type identifier" + ) + symbol: str = Field(description="Ticker symbol") + signal_type: str = Field(description="Type of signal (e.g., first_pullback)") + confidence: float = Field(description="Signal confidence score") + signal_data: dict = Field(default_factory=dict, description="Execution metadata") + + +class SignalBlockedEvent(BaseEvent): + """Event when signal is blocked.""" + + event_type: EventType = Field( + default=EventType.SIGNAL_BLOCKED, description="Event type identifier" + ) + symbol: str = Field(description="Ticker symbol") + blocking_reason: str = Field(description="Reason signal is blocked") + unmet_criteria: list[str] = Field( + default_factory=list, description="List of unmet criteria" + ) + + +class OrderSubmittedEvent(BaseEvent): + """Event when order is submitted.""" + + event_type: EventType = Field( + default=EventType.ORDER_SUBMITTED, description="Event type identifier" + ) + order_id: str = Field(description="Order ID") + symbol: str = Field(description="Ticker symbol") + side: str = Field(description="Buy or sell") + quantity: int = Field(description="Order quantity") + price: float = Field(description="Order price") + + +class OrderApprovalRequestedEvent(BaseEvent): + """Event when order approval is requested.""" + + event_type: EventType = Field( + default=EventType.ORDER_APPROVAL_REQUESTED, description="Event type identifier" + ) + order_id: str = Field(description="Order ID") + symbol: str = Field(description="Ticker symbol") + requested_by: str = Field(description="Who requested approval") + approval_mode: str = Field(description="Approval mode (auto/manual)") + execution_mode: str = Field(description="Target execution mode") + execution_request: dict = Field( + default_factory=dict, description="Pending execution payload" + ) + + +class OrderApprovedEvent(BaseEvent): + """Event when order is approved.""" + + event_type: EventType = Field( + default=EventType.ORDER_APPROVED, description="Event type identifier" + ) + order_id: str = Field(description="Order ID") + symbol: str = Field(description="Ticker symbol") + approved_by: str = Field(description="Who approved the order") + approval_notes: str | None = Field(default=None, description="Approval notes") + + +class OrderRejectedEvent(BaseEvent): + """Event when order is rejected.""" + + event_type: EventType = Field( + default=EventType.ORDER_REJECTED, description="Event type identifier" + ) + order_id: str = Field(description="Order ID") + symbol: str = Field(description="Ticker symbol") + rejected_by: str = Field(description="Who rejected the order") + rejection_reason: str = Field(description="Reason for rejection") + + +class OrderFilledEvent(BaseEvent): + """Event when order is filled.""" + + event_type: EventType = Field( + default=EventType.ORDER_FILLED, description="Event type identifier" + ) + order_id: str = Field(description="Order ID") + symbol: str = Field(description="Ticker symbol") + fill_price: float = Field(description="Fill price") + fill_quantity: int = Field(description="Filled quantity") + slippage: float | None = Field(default=None, description="Slippage amount") + + +class OrderCancelledEvent(BaseEvent): + event_type: EventType = Field( + default=EventType.ORDER_CANCELLED, description="Event type identifier" + ) + order_id: str = Field(description="Order ID") + symbol: str = Field(description="Ticker symbol") + cancel_reason: str | None = Field(default=None, description="Cancel reason") + + +class PositionOpenedEvent(BaseEvent): + """Event when position is opened.""" + + event_type: EventType = Field( + default=EventType.POSITION_OPENED, description="Event type identifier" + ) + position_id: str = Field(description="Position ID") + symbol: str = Field(description="Ticker symbol") + entry_price: float = Field(description="Entry price") + quantity: int = Field(description="Position quantity") + stop_loss_price: float = Field(description="Stop loss price") + + +class PositionClosedEvent(BaseEvent): + """Event when position is closed.""" + + event_type: EventType = Field( + default=EventType.POSITION_CLOSED, description="Event type identifier" + ) + position_id: str = Field(description="Position ID") + symbol: str = Field(description="Ticker symbol") + exit_price: float = Field(description="Exit price") + exit_reason: str = Field(description="Reason for position close") + realized_pnl: float = Field(description="Realized profit/loss") + + +class AccountSummaryUpdatedEvent(BaseEvent): + event_type: EventType = Field( + default=EventType.ACCOUNT_SUMMARY_UPDATED, + description="Event type identifier", + ) + broker_name: str = Field(description="Broker identifier") + account_id: str = Field(description="Broker account identifier") + account_desc: str = Field(description="Display name for account") + total_equity: float = Field(description="Total account equity") + cash_balance: float = Field(description="Available cash balance") + buying_power: float = Field(description="Buying power") + net_liquidating_value: float = Field(description="Net liquidation value") + + +class AccountPositionsUpdatedEvent(BaseEvent): + event_type: EventType = Field( + default=EventType.ACCOUNT_POSITIONS_UPDATED, + description="Event type identifier", + ) + broker_name: str = Field(description="Broker identifier") + account_id: str = Field(description="Broker account identifier") + positions: list[dict] = Field(description="Serialized account positions") + + +class AccountOrdersUpdatedEvent(BaseEvent): + event_type: EventType = Field( + default=EventType.ACCOUNT_ORDERS_UPDATED, + description="Event type identifier", + ) + broker_name: str = Field(description="Broker identifier") + account_id: str = Field(description="Broker account identifier") + orders: list[dict] = Field(description="Serialized account orders") + + +class RiskRuleTriggeredEvent(BaseEvent): + """Event when risk rule is triggered.""" + + event_type: EventType = EventType.RISK_RULE_TRIGGERED + rule_type: str = Field(description="Type of risk rule") + rule_value: float = Field(description="Value that triggered the rule") + current_state: dict = Field(description="Current system state when triggered") + action_taken: str = Field(description="Action taken in response") + + +class BrokerHealthChangedEvent(BaseEvent): + """Event when broker health changes.""" + + event_type: EventType = Field( + default=EventType.BROKER_HEALTH_CHANGED, description="Event type identifier" + ) + broker_name: str = Field(description="Broker identifier") + previous_health: str = Field(description="Previous health status") + new_health: str = Field(description="New health status") + health_reason: str | None = Field( + default=None, description="Reason for health change" + ) + + +class SourceHealthChangedEvent(BaseEvent): + """Event when source health changes.""" + + event_type: EventType = EventType.SOURCE_HEALTH_CHANGED + source_name: str = Field(description="Source identifier") + source_type: str = Field(description="Source type (broker/market data/news)") + previous_health: str = Field(description="Previous health status") + new_health: str = Field(description="New health status") + health_reason: str | None = Field( + default=None, description="Reason for health change" + ) + + +class SessionSummaryEvent(BaseEvent): + event_type: EventType = Field(default=EventType.SESSION_SUMMARY) + session_id: str = Field(description="Unique session identifier") + status: str = Field(description="Session status") + summary_data: dict = Field(default_factory=dict, description="Session summary data") + + +class ModuleTickEvent(BaseEvent): + """Event for module execution telemetry.""" + + event_type: EventType = Field( + default=EventType.MODULE_TICK, description="Event type identifier" + ) + module: str = Field(description="Module name (ingestion, research, evaluation, etc.)") + stage: str = Field(description="Stage (started, processing, completed, failed)") + duration_ms: float = Field(description="Execution duration in milliseconds") + input_count: int = Field(description="Number of inputs processed") + output_count: int = Field(description="Number of outputs produced") + metrics: dict = Field(default_factory=dict, description="Custom module metrics") + errors: list[dict] = Field(default_factory=list, description="Any errors that occurred") + + +class DataValidationEvent(BaseEvent): + """Event for data quality checkpoints.""" + + event_type: EventType = Field( + default=EventType.DATA_VALIDATION, description="Event type identifier" + ) + module: str = Field(description="Module performing validation") + validation_type: str = Field( + description="Type of validation (schema, checksum, business_rules)" + ) + valid: bool = Field(description="Whether validation passed") + details: dict = Field(default_factory=dict, description="Validation details") + + +class SymbolEvaluationDetailEvent(BaseEvent): + """Event for detailed symbol evaluation metrics at each stage.""" + + event_type: EventType = Field( + default=EventType.SYMBOL_EVALUATION_DETAIL, description="Event type identifier" + ) + symbol: str = Field(description="Ticker symbol") + stage: str = Field( + description="Evaluation stage (bars_loaded, gap_checked, rvol_calculated, etc.)" + ) + metrics: dict = Field(description="Metrics at this stage") + passed: bool = Field(description="Whether this stage passed") + blocking_reason: str | None = Field( + default=None, description="If failed, why it failed" + ) diff --git a/storage/event_store.py b/storage/event_store.py new file mode 100644 index 0000000..ddd7703 --- /dev/null +++ b/storage/event_store.py @@ -0,0 +1,223 @@ +"""Event store implementation for Milestone 2. + +Append-only event storage with canonical event types. +""" + +from .event_schema import ( + BaseEvent, + EventType, + EventMode, +) +from .db import get_connection +from uuid import uuid4 +import json +import logging + +logger = logging.getLogger(__name__) + + +class EventStore: + """Append-only event store for canonical events.""" + + def __init__(self, db_path: str = "./data/momentum_events.duckdb"): + """Initialize event store with DuckDB connection.""" + self.con = get_connection(db_path) + self._ensure_schema() + + def _ensure_schema(self): + """Ensure event tables exist in database.""" + cursor = self.con.cursor() + cursor.execute(""" + CREATE TABLE IF NOT EXISTS events ( + id VARCHAR PRIMARY KEY, + timestamp TIMESTAMP, + mode VARCHAR, + event_type VARCHAR, + correlation_id VARCHAR, + message VARCHAR, + payload_json TEXT, + created_at TIMESTAMP DEFAULT current_timestamp + ); + """) + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_events_timestamp + ON events(timestamp); + """) + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_events_session_id + ON events(correlation_id); + """) + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_events_symbol + ON events(payload_json); + """) + self.con.commit() + + def emit(self, event: BaseEvent) -> str: + """Emit an event to the event store. + + Args: + event: Event to emit + + Returns: + Event ID + """ + event_id = str(uuid4()) + payload_json = json.dumps(event.model_dump(mode="json", exclude_unset=True)) + + cursor = self.con.cursor() + cursor.execute( + """ + INSERT INTO events + (id, timestamp, mode, event_type, correlation_id, message, payload_json, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, current_timestamp) + """, + [ + event_id, + event.timestamp, + event.mode.value, + event.event_type.value + if isinstance(event.event_type, EventType) + else str(event.event_type), + event.correlation_id, + event.message, + payload_json, + ], + ) + self.con.commit() + + logger.debug( + f"Emitted event {event_id}: {event.event_type.value} - {event.message}" + ) + return event_id + + def query_events( + self, + event_type: str | None = None, + symbol: str | None = None, + correlation_id: str | None = None, + session_id: str | None = None, + since: str | None = None, + until: str | None = None, + limit: int | None = 1000, + ) -> list[dict]: + """Query events from the store. + + Args: + event_type: Filter by event type + symbol: Filter by symbol + correlation_id: Filter by correlation ID + session_id: Filter by session ID + since: Start timestamp (inclusive) + until: End timestamp (exclusive) + limit: Maximum number of results + + Returns: + List of event dictionaries + """ + conditions = [] + params = [] + + if event_type is not None: + conditions.append("event_type = ?") + params.append(event_type) + + if symbol is not None: + conditions.append("payload_json LIKE ?") + params.append(f"%\"symbol\": \"{symbol}\"%") + + if correlation_id is not None: + conditions.append("correlation_id = ?") + params.append(correlation_id) + + if session_id is not None: + conditions.append("correlation_id = ?") + params.append(session_id) + + if since is not None: + conditions.append("timestamp >= ?") + params.append(since) + + if until is not None: + conditions.append("timestamp < ?") + params.append(until) + + where_clause = " AND ".join(conditions) if conditions else "1=1" + limit_clause = f"LIMIT {limit}" if limit is not None else "" + + query = f""" + SELECT id, timestamp, mode, event_type, correlation_id, message, payload_json, created_at + FROM events + WHERE {where_clause} + ORDER BY timestamp ASC, created_at ASC + {limit_clause} + """ + + cursor = self.con.cursor() + cursor.execute(query, params) + rows = cursor.fetchall() + + results = [] + for row in rows: + results.append( + { + "id": row[0], + "timestamp": str(row[1]), + "mode": row[2], + "event_type": row[3], + "correlation_id": row[4], + "message": row[5], + "payload_json": row[6], + "created_at": str(row[7]), + } + ) + + return results + + def count_events(self) -> int: + """Total number of events. Cheap change-detection signal for SSE.""" + cursor = self.con.cursor() + cursor.execute("SELECT COUNT(*) FROM events") + row = cursor.fetchone() + return int(row[0]) if row else 0 + + def get_symbol_timeline(self, symbol: str) -> list[dict]: + """Get full timeline for a symbol. + + Args: + symbol: Ticker symbol + + Returns: + List of all events for the symbol in chronological order + """ + return self.query_events(symbol=symbol, limit=None) + + def rebuild_state(self, session_id: str) -> dict: + """Rebuild state from event store for a session. + + Args: + session_id: Session ID to rebuild + + Returns: + Dictionary with session summary and latest state + """ + events = self.query_events(session_id=session_id, limit=None) + + summary = { + "session_id": session_id, + "total_events": len(events), + "event_types": {}, + } + + for event in events: + event_type = event["event_type"] + summary["event_types"][event_type] = ( + summary["event_types"].get(event_type, 0) + 1 + ) + + return summary + + def close(self): + """Close event store connection.""" + if self.con: + self.con.close() diff --git a/storage/pg_schema.sql b/storage/pg_schema.sql new file mode 100644 index 0000000..3e9ec4c --- /dev/null +++ b/storage/pg_schema.sql @@ -0,0 +1,451 @@ +-- PostgreSQL schema for the momentum system (single datastore). +-- +-- Port of storage/schema.py (DuckDB) + the event store `events` table that +-- previously lived in storage/event_store.py. Dialect deltas applied: +-- DOUBLE -> DOUBLE PRECISION +-- payload index -> on event_type/correlation_id (a btree on unbounded +-- TEXT payload is invalid in Postgres) +-- everything else -> already Postgres-compatible (DuckDB dialect is +-- Postgres-derived: IF NOT EXISTS, BOOLEAN, VARCHAR, +-- composite PKs, current_timestamp defaults all work). +-- +-- Idempotent: safe to run repeatedly (CREATE ... IF NOT EXISTS throughout). + +-- ===================================================================== +-- Operational event store (append-only source of truth) +-- ===================================================================== +CREATE TABLE IF NOT EXISTS events ( + id VARCHAR PRIMARY KEY, + timestamp TIMESTAMP, + mode VARCHAR, + event_type VARCHAR, + correlation_id VARCHAR, + message VARCHAR, + payload_json TEXT, + created_at TIMESTAMP DEFAULT current_timestamp +); +CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events (timestamp); +CREATE INDEX IF NOT EXISTS idx_events_correlation ON events (correlation_id); +CREATE INDEX IF NOT EXISTS idx_events_type ON events (event_type); + +-- ===================================================================== +-- Universe metadata +-- ===================================================================== +CREATE TABLE IF NOT EXISTS symbols ( + symbol VARCHAR PRIMARY KEY, + security_name VARCHAR, + exchange VARCHAR, + asset_type VARCHAR DEFAULT 'common_stock', + sector VARCHAR, + industry VARCHAR, + shares_outstanding BIGINT, + float_shares BIGINT, + market_cap DOUBLE PRECISION, + is_etf BOOLEAN DEFAULT FALSE, + is_otc BOOLEAN DEFAULT FALSE, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT current_timestamp, + updated_at TIMESTAMP DEFAULT current_timestamp +); + +CREATE TABLE IF NOT EXISTS trading_days ( + trade_date DATE PRIMARY KEY, + session_open TIMESTAMP, + session_close TIMESTAMP, + premarket_open TIMESTAMP, + afterhours_close TIMESTAMP, + is_half_day BOOLEAN DEFAULT FALSE, + is_holiday_adjacent BOOLEAN DEFAULT FALSE, + tape_regime VARCHAR, + market_regime VARCHAR, + volatility_regime VARCHAR, + gap_activity_level INTEGER, + momentum_density INTEGER +); + +-- ===================================================================== +-- Bars +-- ===================================================================== +CREATE TABLE IF NOT EXISTS daily_bars ( + symbol VARCHAR, + trade_date DATE, + open DOUBLE PRECISION, + high DOUBLE PRECISION, + low DOUBLE PRECISION, + close DOUBLE PRECISION, + volume BIGINT, + vwap DOUBLE PRECISION, + previous_close DOUBLE PRECISION, + true_range DOUBLE PRECISION, + atr_14 DOUBLE PRECISION, + rolling_avg_volume_20d DOUBLE PRECISION, + rolling_avg_volume_50d DOUBLE PRECISION, + prior_day_gain_pct DOUBLE PRECISION, + multi_day_runner_flag BOOLEAN DEFAULT FALSE, + PRIMARY KEY (symbol, trade_date) +); + +CREATE TABLE IF NOT EXISTS minute_bars ( + symbol VARCHAR, + timestamp TIMESTAMP, + session_date DATE, + is_premarket BOOLEAN, + is_regular_hours BOOLEAN, + is_afterhours BOOLEAN, + open DOUBLE PRECISION, + high DOUBLE PRECISION, + low DOUBLE PRECISION, + close DOUBLE PRECISION, + volume BIGINT, + vwap DOUBLE PRECISION, + spread_pct DOUBLE PRECISION, + halt_status BOOLEAN DEFAULT FALSE, + source_provider VARCHAR DEFAULT 'synthetic', + quality_score DOUBLE PRECISION, + PRIMARY KEY (symbol, timestamp) +); +-- Fast per-symbol/session bar lookups (the `inspect bars` path). +CREATE INDEX IF NOT EXISTS idx_minute_bars_symbol_session + ON minute_bars (symbol, session_date); + +-- ===================================================================== +-- News / catalysts / halts / context +-- ===================================================================== +CREATE TABLE IF NOT EXISTS news_events ( + id VARCHAR, + symbol VARCHAR, + headline VARCHAR, + source VARCHAR, + published_at TIMESTAMP, + catalyst_freshness_minutes DOUBLE PRECISION, + category VARCHAR, + sentiment DOUBLE PRECISION, + is_earnings BOOLEAN DEFAULT FALSE, + is_offering BOOLEAN DEFAULT FALSE, + is_halt_related BOOLEAN DEFAULT FALSE, + PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS halt_events ( + id VARCHAR, + symbol VARCHAR, + halt_start TIMESTAMP, + halt_end TIMESTAMP, + halt_type VARCHAR, + price_at_halt DOUBLE PRECISION, + resumed_price DOUBLE PRECISION, + gap_on_resume_pct DOUBLE PRECISION, + PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS market_context ( + timestamp TIMESTAMP, + trade_date DATE, + spy_open DOUBLE PRECISION, + spy_high DOUBLE PRECISION, + spy_low DOUBLE PRECISION, + spy_close DOUBLE PRECISION, + spy_volume BIGINT, + qqq_open DOUBLE PRECISION, + qqq_close DOUBLE PRECISION, + spy_intraday_return_pct DOUBLE PRECISION, + vix_open DOUBLE PRECISION, + vix_level DOUBLE PRECISION, + advance_decline_ratio DOUBLE PRECISION, + up_volume_down_volume_ratio DOUBLE PRECISION, + tape_regime_flag VARCHAR, + PRIMARY KEY (timestamp) +); + +CREATE TABLE IF NOT EXISTS dilution_events ( + id VARCHAR PRIMARY KEY, + symbol VARCHAR, + event_type VARCHAR, + announced_at TIMESTAMP, + effective_date DATE, + share_count BIGINT, + price_per_share DOUBLE PRECISION, + gross_proceeds DOUBLE PRECISION +); + +CREATE TABLE IF NOT EXISTS short_interest ( + symbol VARCHAR, + report_date DATE, + short_interest_shares BIGINT, + short_interest_pct_float DOUBLE PRECISION, + days_to_cover DOUBLE PRECISION, + PRIMARY KEY (symbol, report_date) +); + +-- ===================================================================== +-- Features / setups / outcomes / backtests +-- ===================================================================== +CREATE TABLE IF NOT EXISTS engineered_features ( + id VARCHAR PRIMARY KEY, + timestamp TIMESTAMP, + symbol VARCHAR, + session_date DATE, + feature_version VARCHAR, + gap_pct DOUBLE PRECISION, + premarket_gap_pct DOUBLE PRECISION, + relative_volume DOUBLE PRECISION, + time_of_day_adjusted_relative_volume DOUBLE PRECISION, + float_rotation_pct DOUBLE PRECISION, + vwap DOUBLE PRECISION, + distance_from_vwap DOUBLE PRECISION, + ema9 DOUBLE PRECISION, + ema20 DOUBLE PRECISION, + pullback_volume_ratio DOUBLE PRECISION, + pullback_candle_type VARCHAR, + catalyst_freshness_minutes DOUBLE PRECISION, + catalyst_type VARCHAR, + tape_regime VARCHAR, + spy_intraday_return DOUBLE PRECISION, + vix_level DOUBLE PRECISION, + metadata_json VARCHAR +); + +CREATE TABLE IF NOT EXISTS setup_events ( + setup_id VARCHAR PRIMARY KEY, + symbol VARCHAR, + setup_time TIMESTAMP, + session_date DATE, + setup_name VARCHAR DEFAULT 'first_pullback', + setup_version VARCHAR DEFAULT 'v1', + entry_reference_price DOUBLE PRECISION, + invalidation_price DOUBLE PRECISION, + target_r_multiple DOUBLE PRECISION DEFAULT 2.0, + impulse_start_price DOUBLE PRECISION, + impulse_end_price DOUBLE PRECISION, + impulse_pct DOUBLE PRECISION, + pullback_low DOUBLE PRECISION, + pullback_depth_pct DOUBLE PRECISION, + pullback_bars INTEGER, + pullback_volume_ratio DOUBLE PRECISION, + above_vwap_flag BOOLEAN, + vwap_at_trigger DOUBLE PRECISION, + gap_pct DOUBLE PRECISION, + relative_volume DOUBLE PRECISION, + float_rotation_at_trigger DOUBLE PRECISION, + catalyst_freshness_at_trigger DOUBLE PRECISION, + session_minute_number INTEGER, + tape_regime_at_trigger VARCHAR, + spy_intraday_return_pct_at_trigger DOUBLE PRECISION +); + +CREATE TABLE IF NOT EXISTS outcome_labels ( + setup_id VARCHAR PRIMARY KEY, + label_version VARCHAR DEFAULT 'v1', + max_upside_next_5m DOUBLE PRECISION, + max_upside_next_15m DOUBLE PRECISION, + max_upside_next_60m DOUBLE PRECISION, + max_drawdown_next_5m DOUBLE PRECISION, + max_drawdown_next_15m DOUBLE PRECISION, + max_drawdown_next_60m DOUBLE PRECISION, + reached_1r_before_minus_1r BOOLEAN, + reached_2r_before_minus_1r BOOLEAN, + held_vwap_5m BOOLEAN, + held_vwap_15m BOOLEAN, + trend_day_flag BOOLEAN, + failed_breakout_flag BOOLEAN, + time_to_max_upside_minutes DOUBLE PRECISION, + time_to_max_drawdown_minutes DOUBLE PRECISION +); + +CREATE TABLE IF NOT EXISTS backtest_runs ( + run_id VARCHAR PRIMARY KEY, + run_timestamp TIMESTAMP, + strategy_name VARCHAR, + strategy_version VARCHAR, + feature_version VARCHAR, + date_range_start DATE, + date_range_end DATE, + parameter_config_json VARCHAR, + summary_metrics_json VARCHAR, + total_trades INTEGER, + win_rate DOUBLE PRECISION, + expectancy DOUBLE PRECISION, + max_drawdown DOUBLE PRECISION, + profit_factor DOUBLE PRECISION +); + +-- ===================================================================== +-- Paper trading +-- ===================================================================== +CREATE TABLE IF NOT EXISTS simulated_orders ( + order_id VARCHAR PRIMARY KEY, + session_id VARCHAR, + symbol VARCHAR, + created_at TIMESTAMP, + side VARCHAR, + order_type VARCHAR, + qty INTEGER, + intended_entry DOUBLE PRECISION, + stop_price DOUBLE PRECISION, + target_price DOUBLE PRECISION, + status VARCHAR, + strategy_name VARCHAR, + setup_id VARCHAR +); + +CREATE TABLE IF NOT EXISTS simulated_fills ( + fill_id VARCHAR PRIMARY KEY, + order_id VARCHAR, + symbol VARCHAR, + fill_time TIMESTAMP, + fill_price DOUBLE PRECISION, + qty INTEGER, + slippage DOUBLE PRECISION, + commission DOUBLE PRECISION, + venue VARCHAR +); + +CREATE TABLE IF NOT EXISTS strategy_runs ( + run_id VARCHAR PRIMARY KEY, + started_at TIMESTAMP, + ended_at TIMESTAMP, + mode VARCHAR, + strategy_name VARCHAR, + strategy_version VARCHAR, + config_json VARCHAR, + summary_json VARCHAR +); + +CREATE TABLE IF NOT EXISTS paper_sessions ( + session_id VARCHAR PRIMARY KEY, + trade_date DATE, + started_at TIMESTAMP, + ended_at TIMESTAMP, + starting_equity DOUBLE PRECISION, + ending_equity DOUBLE PRECISION, + realized_pnl DOUBLE PRECISION, + unrealized_pnl DOUBLE PRECISION, + total_orders INTEGER, + total_fills INTEGER, + total_setups INTEGER, + notes VARCHAR +); + +CREATE TABLE IF NOT EXISTS paper_missed_trades ( + audit_id VARCHAR PRIMARY KEY, + session_id VARCHAR, + symbol VARCHAR, + setup_id VARCHAR, + reason VARCHAR, + setup_time TIMESTAMP, + evaluated_at TIMESTAMP, + attempted_entry_time TIMESTAMP, + entry_cutoff VARCHAR, + minutes_past_cutoff DOUBLE PRECISION, + success_score_pct DOUBLE PRECISION, + criteria_passed INTEGER, + criteria_total INTEGER, + provider_status VARCHAR, + provider_degraded_count INTEGER, + source_status VARCHAR, + source_degraded_count INTEGER, + source_backoff_count INTEGER, + created_at TIMESTAMP, + details_json VARCHAR +); + +-- ===================================================================== +-- Raw landing (append-only) + telemetry + scanner snapshots +-- ===================================================================== +CREATE TABLE IF NOT EXISTS raw_fetch_attempts ( + id VARCHAR PRIMARY KEY, + source VARCHAR NOT NULL, + capability VARCHAR NOT NULL, + request_window_start TIMESTAMP, + request_window_end TIMESTAMP, + fetched_at TIMESTAMP NOT NULL, + ingest_run_id VARCHAR, + http_status INTEGER, + item_count INTEGER, + error_msg VARCHAR, + payload_hash VARCHAR, + parser_version VARCHAR +); +CREATE INDEX IF NOT EXISTS idx_raw_fetch_attempts_source_window + ON raw_fetch_attempts (source, capability, request_window_start, request_window_end); +CREATE INDEX IF NOT EXISTS idx_raw_fetch_attempts_ingest_run + ON raw_fetch_attempts (ingest_run_id); + +CREATE TABLE IF NOT EXISTS raw_news_items ( + id VARCHAR PRIMARY KEY, + fetch_attempt_id VARCHAR, + source VARCHAR, + raw_url VARCHAR, + raw_title VARCHAR, + raw_published_at VARCHAR, + raw_body_snippet VARCHAR, + raw_tickers VARCHAR, + payload_hash VARCHAR, + parser_version VARCHAR, + fetched_at TIMESTAMP, + ingest_run_id VARCHAR +); +CREATE INDEX IF NOT EXISTS idx_raw_news_items_fetch_attempt + ON raw_news_items (fetch_attempt_id); +CREATE INDEX IF NOT EXISTS idx_raw_news_items_source_published + ON raw_news_items (source, raw_published_at); + +CREATE TABLE IF NOT EXISTS telemetry_events ( + id VARCHAR PRIMARY KEY, + event_type VARCHAR, + session_id VARCHAR, + symbol VARCHAR, + source VARCHAR, + from_state VARCHAR, + to_state VARCHAR, + reason VARCHAR, + metadata_json VARCHAR, + created_at TIMESTAMP DEFAULT current_timestamp +); + +CREATE TABLE IF NOT EXISTS scanner_snapshots ( + id VARCHAR PRIMARY KEY, + snapshot_time TIMESTAMP, + symbol VARCHAR, + rank INTEGER, + price DOUBLE PRECISION, + gap_pct DOUBLE PRECISION, + premarket_gap_pct DOUBLE PRECISION, + cumulative_volume BIGINT, + minute_volume BIGINT, + premarket_volume BIGINT, + relative_volume DOUBLE PRECISION, + float_rotation_pct DOUBLE PRECISION, + vwap DOUBLE PRECISION, + distance_from_vwap DOUBLE PRECISION, + ema9 DOUBLE PRECISION, + ema20 DOUBLE PRECISION, + intraday_range DOUBLE PRECISION, + intraday_range_vs_atr DOUBLE PRECISION, + distance_to_premarket_high DOUBLE PRECISION, + distance_to_premarket_low DOUBLE PRECISION, + distance_to_day_high DOUBLE PRECISION, + distance_to_day_low DOUBLE PRECISION, + news_flag BOOLEAN, + catalyst_freshness_minutes DOUBLE PRECISION, + catalyst_type VARCHAR, + float_bucket VARCHAR, + market_cap_bucket VARCHAR, + spread_pct DOUBLE PRECISION, + halt_flag BOOLEAN, + dilution_risk_flag BOOLEAN, + sympathy_move_candidate_flag BOOLEAN, + lead_runner_symbol VARCHAR, + sympathy_cluster_id VARCHAR, + early_leader_flag BOOLEAN, + rank_stability DOUBLE PRECISION, + momentum_score DOUBLE PRECISION, + scanner_version VARCHAR +); + +-- ===================================================================== +-- Late-added columns (parity with storage/schema.py:create_schema) +-- Postgres supports ADD COLUMN IF NOT EXISTS (9.6+). +-- ===================================================================== +ALTER TABLE setup_events ADD COLUMN IF NOT EXISTS tape_regime_at_trigger VARCHAR; +ALTER TABLE setup_events ADD COLUMN IF NOT EXISTS spy_intraday_return_pct_at_trigger DOUBLE PRECISION; diff --git a/storage/projections.py b/storage/projections.py new file mode 100644 index 0000000..a36cb92 --- /dev/null +++ b/storage/projections.py @@ -0,0 +1,530 @@ +import json + +"""Projections for Milestone 2 read models. + +Aggregated views for querying sessions, symbols, approvals, and positions. +""" + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + pass + + +def query_session_summary(store) -> list[dict]: + """Query all session summaries from events.""" + events = store.query_events(event_type="session_summary") + + summaries = [] + for event in events: + payload = json.loads(event.get("payload_json", "{}")) + if "session_id" in payload: + summaries.append( + { + "session_id": payload["session_id"], + "event_id": event["id"], + "timestamp": event["timestamp"], + } + ) + + return summaries + + +def query_symbol_status(store, symbol: str | None = None) -> list[dict]: + """Query current status for symbols.""" + events = store.query_events( + event_type="symbol_state_changed", + symbol=symbol, + ) + + return [ + { + "event_id": event["id"], + "symbol": json.loads(event.get("payload_json", "{}")).get("symbol"), + "previous_state": json.loads(event.get("payload_json", "{}")).get( + "previous_state" + ), + "new_state": json.loads(event.get("payload_json", "{}")).get("new_state"), + "timestamp": event["timestamp"], + } + for event in events + ] + + +def query_approval_queue(store) -> list[dict]: + requested = store.query_events(event_type="order_approval_requested", limit=None) + approved = store.query_events(event_type="order_approved", limit=None) + rejected = store.query_events(event_type="order_rejected", limit=None) + submitted = store.query_events(event_type="order_submitted", limit=None) + cancelled = store.query_events(event_type="order_cancelled", limit=None) + + resolved_ids = { + str(json.loads(event.get("payload_json", "{}")).get("order_id")) + for event in approved + rejected + submitted + cancelled + if json.loads(event.get("payload_json", "{}")).get("order_id") is not None + } + + queue = [] + for event in requested: + payload = json.loads(event.get("payload_json", "{}")) + order_id = payload.get("order_id") + if order_id is None or str(order_id) in resolved_ids: + continue + queue.append( + { + "event_id": event["id"], + "order_id": str(order_id), + "symbol": payload.get("symbol"), + "requested_by": payload.get("requested_by"), + "approval_mode": payload.get("approval_mode"), + "execution_mode": payload.get("execution_mode"), + "execution_request": payload.get("execution_request", {}), + "timestamp": event["timestamp"], + } + ) + return queue + + +def query_positions_view(store) -> list[dict]: + """Query all positions (opened/closed events).""" + opened = store.query_events(event_type="position_opened") + closed = store.query_events(event_type="position_closed") + position_events = opened + closed + + positions = [] + for event in position_events: + payload = json.loads(event.get("payload_json", "{}")) + positions.append( + { + "event_id": event["id"], + "position_id": payload.get("position_id"), + "symbol": payload.get("symbol"), + "event_type": event["event_type"], + "entry_price": payload.get("entry_price"), + "quantity": payload.get("quantity"), + "stop_loss_price": payload.get("stop_loss_price"), + "exit_price": payload.get("exit_price") + if event["event_type"] == "position_closed" + else None, + "exit_reason": payload.get("exit_reason") + if event["event_type"] == "position_closed" + else None, + "realized_pnl": payload.get("realized_pnl") + if event["event_type"] == "position_closed" + else None, + "timestamp": event["timestamp"], + } + ) + + return positions + + +def query_event_timeline(store, session_id: str | None = None) -> list[dict]: + """Query full event timeline for a session.""" + return store.query_events(session_id=session_id, limit=None) + + +def query_settings_snapshot(store) -> dict: + """Query current settings snapshot.""" + events = store.query_events(event_type="settings_snapshot") + + if not events: + return {} + + latest_event = events[-1] + return json.loads(latest_event.get("payload_json", "{}")) + + +def query_ready_signals_snapshot(store, session_id: str | None = None) -> list[dict]: + """Latest signal_ready per symbol, newest first. Feeds the dashboard.""" + events = store.query_events( + event_type="signal_ready", session_id=session_id, limit=None + ) + latest_by_symbol: dict[str, dict] = {} + for event in events: + payload = json.loads(event.get("payload_json", "{}")) + symbol = payload.get("symbol") + if not symbol: + continue + signal_data = payload.get("signal_data") or {} + latest_by_symbol[str(symbol)] = { + "symbol": str(symbol), + "signal_type": payload.get("signal_type"), + "confidence": payload.get("confidence"), + "entry_price": signal_data.get("entry_price"), + "stop_loss_price": signal_data.get("stop_loss_price"), + "quality_score": signal_data.get("quality_score"), + "timestamp": event.get("timestamp"), + } + return sorted( + latest_by_symbol.values(), + key=lambda row: str(row.get("timestamp") or ""), + reverse=True, + ) + + +def query_watch_states_snapshot(store, session_id: str | None = None) -> list[dict]: + state_events = store.query_events( + event_type="symbol_state_changed", + session_id=session_id, + limit=None, + ) + criteria_events = store.query_events( + event_type="criteria_evaluated", + session_id=session_id, + limit=None, + ) + + latest_by_symbol: dict[str, dict] = {} + + def _blank(symbol_key: str) -> dict: + return { + "symbol": symbol_key, + "state": None, + "previous_state": None, + "state_reason": None, + "last_transition_at": None, + "last_score": None, + "last_criteria_at": None, + } + + for event in criteria_events: + payload = json.loads(event.get("payload_json", "{}")) + symbol = payload.get("symbol") + if not symbol: + continue + symbol_key = str(symbol) + entry = latest_by_symbol.setdefault(symbol_key, _blank(symbol_key)) + entry["last_score"] = payload.get("success_score_pct") + entry["last_criteria_at"] = event.get("timestamp") + + for event in state_events: + payload = json.loads(event.get("payload_json", "{}")) + symbol = payload.get("symbol") + if not symbol: + continue + symbol_key = str(symbol) + entry = latest_by_symbol.setdefault(symbol_key, _blank(symbol_key)) + entry["state"] = payload.get("new_state") + entry["previous_state"] = payload.get("previous_state") + entry["state_reason"] = payload.get("state_reason") + entry["last_transition_at"] = event.get("timestamp") + + return sorted(latest_by_symbol.values(), key=lambda row: row["symbol"]) + + +def query_account_summary_snapshot(store, broker_name: str | None = None) -> list[dict]: + events = store.query_events(event_type="account_summary_updated", limit=None) + latest_by_account: dict[tuple[str, str], dict] = {} + + for event in events: + payload = json.loads(event.get("payload_json", "{}")) + broker = str(payload.get("broker_name") or "") + account_id = str(payload.get("account_id") or "") + if not broker or not account_id: + continue + if broker_name is not None and broker != broker_name: + continue + + latest_by_account[(broker, account_id)] = { + "broker_name": broker, + "account_id": account_id, + "account_desc": payload.get("account_desc"), + "total_equity": payload.get("total_equity"), + "cash_balance": payload.get("cash_balance"), + "buying_power": payload.get("buying_power"), + "net_liquidating_value": payload.get("net_liquidating_value"), + "timestamp": event.get("timestamp"), + } + + return sorted( + latest_by_account.values(), + key=lambda row: (str(row["broker_name"]), str(row["account_id"])), + ) + + +def query_account_positions_snapshot( + store, broker_name: str | None = None +) -> list[dict]: + events = store.query_events(event_type="account_positions_updated", limit=None) + latest_by_account: dict[tuple[str, str], dict] = {} + + for event in events: + payload = json.loads(event.get("payload_json", "{}")) + broker = str(payload.get("broker_name") or "") + account_id = str(payload.get("account_id") or "") + if not broker or not account_id: + continue + if broker_name is not None and broker != broker_name: + continue + + latest_by_account[(broker, account_id)] = { + "broker_name": broker, + "account_id": account_id, + "positions": payload.get("positions", []), + "timestamp": event.get("timestamp"), + } + + return sorted( + latest_by_account.values(), + key=lambda row: (str(row["broker_name"]), str(row["account_id"])), + ) + + +def query_account_orders_snapshot(store, broker_name: str | None = None) -> list[dict]: + events = store.query_events(event_type="account_orders_updated", limit=None) + latest_by_account: dict[tuple[str, str], dict] = {} + + for event in events: + payload = json.loads(event.get("payload_json", "{}")) + broker = str(payload.get("broker_name") or "") + account_id = str(payload.get("account_id") or "") + if not broker or not account_id: + continue + if broker_name is not None and broker != broker_name: + continue + + latest_by_account[(broker, account_id)] = { + "broker_name": broker, + "account_id": account_id, + "orders": payload.get("orders", []), + "timestamp": event.get("timestamp"), + } + + return sorted( + latest_by_account.values(), + key=lambda row: (str(row["broker_name"]), str(row["account_id"])), + ) + + +def query_order_lifecycle_snapshot(store) -> list[dict]: + event_types = [ + "order_submitted", + "order_approval_requested", + "order_approved", + "order_rejected", + "order_filled", + "order_cancelled", + ] + latest_by_order: dict[str, dict] = {} + + for event_type in event_types: + events = store.query_events(event_type=event_type, limit=None) + for event in events: + payload = json.loads(event.get("payload_json", "{}")) + order_id = payload.get("order_id") + if not order_id: + continue + entry = latest_by_order.setdefault( + str(order_id), + { + "order_id": str(order_id), + "symbol": payload.get("symbol"), + "status": None, + "submitted_at": None, + "updated_at": None, + "side": None, + "quantity": None, + "price": None, + "fill_price": None, + "fill_quantity": None, + "approval_mode": None, + "rejection_reason": None, + }, + ) + + if event_type == "order_submitted": + entry["status"] = "submitted" + entry["submitted_at"] = event.get("timestamp") + entry["side"] = payload.get("side") + entry["quantity"] = payload.get("quantity") + entry["price"] = payload.get("price") + elif event_type == "order_approval_requested": + entry["status"] = "approval_requested" + entry["approval_mode"] = payload.get("approval_mode") + elif event_type == "order_approved": + entry["status"] = "approved" + elif event_type == "order_rejected": + entry["status"] = "rejected" + entry["rejection_reason"] = payload.get("rejection_reason") + elif event_type == "order_filled": + entry["status"] = "filled" + entry["fill_price"] = payload.get("fill_price") + entry["fill_quantity"] = payload.get("fill_quantity") + elif event_type == "order_cancelled": + entry["status"] = "cancelled" + entry["rejection_reason"] = payload.get("cancel_reason") + + entry["updated_at"] = event.get("timestamp") + + return sorted(latest_by_order.values(), key=lambda row: row["order_id"]) + + +# --------------------------------------------------------------------------- +# Interactive dashboard projections (criteria detail, session P&L, fills feed) +# --------------------------------------------------------------------------- + +# Human-readable labels for the nine setup criteria, in evaluation order. +CRITERIA_LABELS = { + "sufficient_data": "Sufficient data", + "gap": "Gap %", + "relative_volume": "Relative volume", + "impulse": "Impulse leg", + "pullback": "Pullback depth", + "pullback_volume": "Pullback volume", + "vwap": "Holding VWAP", + "candle_quality": "Candle quality", + "breakout": "Breakout trigger", +} + + +def query_symbol_criteria(store, symbol: str, session_id: str | None = None) -> dict: + """Latest per-criterion pass/fail breakdown for one symbol. + + Powers the click-to-expand criteria panel: returns every criterion with a + passed flag and label so a 'blocked' symbol becomes legible at a glance. + """ + events = store.query_events( + event_type="criteria_evaluated", + symbol=symbol, + session_id=session_id, + limit=None, + ) + if not events: + return {"symbol": symbol, "criteria": [], "score": None, "evaluated_at": None} + + latest = events[-1] + payload = json.loads(latest.get("payload_json", "{}")) + results = payload.get("criteria_results") or {} + passed = set(results.get("passed") or []) + failed = set(results.get("failed") or []) + + criteria = [] + for key, label in CRITERIA_LABELS.items(): + if key in passed: + state = True + elif key in failed: + state = False + else: + state = None # not evaluated this pass + criteria.append({"key": key, "label": label, "passed": state}) + + return { + "symbol": symbol, + "criteria": criteria, + "passed_count": payload.get("passed_criteria"), + "total_count": payload.get("total_criteria"), + "score": payload.get("success_score_pct"), + "status": payload.get("status"), + "evaluated_at": latest.get("timestamp"), + } + + +def query_fills_feed(store, limit: int = 50) -> list[dict]: + """Chronological fills + submissions for the activity feed (newest first).""" + rows: list[dict] = [] + for event_type in ("order_filled", "order_submitted"): + for event in store.query_events(event_type=event_type, limit=None): + payload = json.loads(event.get("payload_json", "{}")) + rows.append( + { + "kind": event_type, + "order_id": payload.get("order_id"), + "symbol": payload.get("symbol"), + "side": payload.get("side"), + "quantity": payload.get("fill_quantity") or payload.get("quantity"), + "price": payload.get("fill_price") or payload.get("price"), + "timestamp": event.get("timestamp"), + } + ) + rows.sort(key=lambda r: str(r.get("timestamp") or ""), reverse=True) + return rows[:limit] + + +def _r_multiple(entry, stop, exit_price, side: str) -> float | None: + """Realized R-multiple given entry/stop/exit. None if stop distance is 0.""" + try: + entry = float(entry) + stop = float(stop) + exit_price = float(exit_price) + except (TypeError, ValueError): + return None + risk = abs(entry - stop) + if risk <= 0: + return None + direction = 1.0 if side != "sell" else -1.0 + return round((exit_price - entry) * direction / risk, 2) + + +def query_session_pnl(store, session_id: str | None = None) -> dict: + """Day P&L + trade stats from position_closed events and live positions. + + Realized stats come from closed positions; unrealized comes from the most + recent account positions snapshot so the strip matches the broker. + """ + closed = store.query_events(event_type="position_closed", limit=None) + + realized = 0.0 + wins = 0 + losses = 0 + r_multiples: list[float] = [] + trades: list[dict] = [] + + for event in closed: + payload = json.loads(event.get("payload_json", "{}")) + pnl = payload.get("realized_pnl") + if pnl is None: + continue + pnl = float(pnl) + realized += pnl + if pnl > 0: + wins += 1 + elif pnl < 0: + losses += 1 + r = _r_multiple( + payload.get("entry_price"), + payload.get("stop_loss_price"), + payload.get("exit_price"), + payload.get("side") or "buy", + ) + if r is not None: + r_multiples.append(r) + trades.append( + { + "symbol": payload.get("symbol"), + "realized_pnl": round(pnl, 2), + "r_multiple": r, + "exit_reason": payload.get("exit_reason"), + "timestamp": event.get("timestamp"), + } + ) + + # unrealized from the latest positions snapshot (any broker) + unrealized = 0.0 + open_positions = 0 + snapshots = query_account_positions_snapshot(store) + if snapshots: + for position in snapshots[-1].get("positions") or []: + open_positions += 1 + try: + unrealized += float(position.get("unrealized_pl") or 0.0) + except (TypeError, ValueError): + pass + + closed_count = wins + losses + win_rate = round(wins / closed_count, 3) if closed_count else None + avg_r = round(sum(r_multiples) / len(r_multiples), 2) if r_multiples else None + + return { + "realized_pnl": round(realized, 2), + "unrealized_pnl": round(unrealized, 2), + "total_pnl": round(realized + unrealized, 2), + "wins": wins, + "losses": losses, + "closed_trades": closed_count, + "open_positions": open_positions, + "win_rate": win_rate, + "avg_r_multiple": avg_r, + "expectancy_r": avg_r, + "trades": list(reversed(trades))[:20], + } diff --git a/storage/replay.py b/storage/replay.py new file mode 100644 index 0000000..2e4d36a --- /dev/null +++ b/storage/replay.py @@ -0,0 +1,148 @@ +import json + +"""Replay/rebuild tooling for Milestone 2. + +Reconstruct state from event store for analysis and testing. +""" + +import logging +from typing import TYPE_CHECKING + +from .event_store import EventStore + +if TYPE_CHECKING: + pass + +logger = logging.getLogger(__name__) + + +class StateRebuilder: + """Rebuild trading state from event store.""" + + def __init__(self, store: EventStore): + self.store = store + + def rebuild_session(self, session_id: str) -> dict: + """Rebuild complete state for a session from events.""" + logger.info(f"Rebuilding session {session_id}") + + events = self.store.query_events(session_id=session_id) + + state = { + "session_id": session_id, + "symbols": {}, + "orders": {}, + "positions": {}, + "events": [], + } + + for event in events: + event_type = event["event_type"] + payload = json.loads(event.get("payload_json", "{}")) + symbol = payload.get("symbol") + + self._process_event(state, event_type, symbol, payload, event) + + logger.info(f"Rebuilt state for {len(events)} events") + return state + + def _process_event( + self, + state: dict, + event_type: str, + symbol: str | None, + payload: dict, + event: dict, + ): + """Process individual event and update state.""" + if event_type == "symbol_discovered": + state["symbols"][symbol] = { + "state": "discovered", + "discovered_at": event["timestamp"], + } + + elif event_type == "symbol_state_changed": + if symbol not in state["symbols"]: + state["symbols"][symbol] = { + "state": payload.get("previous_state") or "discovered", + "discovered_at": event["timestamp"], + } + state["symbols"][symbol]["state"] = payload["new_state"] + state["symbols"][symbol]["state_history"] = state["symbols"][symbol].get( + "state_history", [] + ) + state["symbols"][symbol]["state_history"].append( + { + "state": payload["new_state"], + "timestamp": event["timestamp"], + "reason": payload.get("state_reason"), + } + ) + + elif event_type == "signal_ready": + if symbol in state["symbols"]: + state["symbols"][symbol]["signals"] = state["symbols"][symbol].get( + "signals", [] + ) + state["symbols"][symbol]["signals"].append( + { + "signal_type": payload["signal_type"], + "confidence": payload["confidence"], + "timestamp": event["timestamp"], + } + ) + + elif event_type == "position_opened": + position_id = payload["position_id"] + state["positions"][position_id] = { + "symbol": symbol, + "status": "open", + "entry_price": payload["entry_price"], + "quantity": payload["quantity"], + "stop_loss_price": payload["stop_loss_price"], + "opened_at": event["timestamp"], + } + + elif event_type == "position_closed": + position_id = payload["position_id"] + if position_id in state["positions"]: + state["positions"][position_id].update( + { + "status": "closed", + "exit_price": payload["exit_price"], + "exit_reason": payload["exit_reason"], + "realized_pnl": payload["realized_pnl"], + "closed_at": event["timestamp"], + } + ) + + elif event_type == "order_filled": + state["events"].append( + { + "type": event_type, + "order_id": payload["order_id"], + "symbol": symbol, + "timestamp": event["timestamp"], + } + ) + + elif event_type == "order_submitted": + order_id = payload["order_id"] + state["orders"][order_id] = { + "symbol": symbol, + "status": "submitted", + "side": payload.get("side"), + "quantity": payload.get("quantity"), + "price": payload.get("price"), + "updated_at": event["timestamp"], + } + + elif event_type == "order_cancelled": + order_id = payload["order_id"] + state["orders"][order_id] = { + **state["orders"].get(order_id, {}), + "symbol": symbol, + "status": "cancelled", + "cancel_reason": payload.get("cancel_reason"), + "updated_at": event["timestamp"], + } diff --git a/storage/schema.py b/storage/schema.py new file mode 100644 index 0000000..2b737fb --- /dev/null +++ b/storage/schema.py @@ -0,0 +1,444 @@ +"""DuckDB schema definitions for all tables. + +Milestone 1, Task 1.1 +""" + +SCHEMA_SQL = """ +-- Symbols: universe metadata +CREATE TABLE IF NOT EXISTS symbols ( + symbol VARCHAR PRIMARY KEY, + security_name VARCHAR, + exchange VARCHAR, + asset_type VARCHAR DEFAULT 'common_stock', + sector VARCHAR, + industry VARCHAR, + shares_outstanding BIGINT, + float_shares BIGINT, + market_cap DOUBLE, + is_etf BOOLEAN DEFAULT FALSE, + is_otc BOOLEAN DEFAULT FALSE, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT current_timestamp, + updated_at TIMESTAMP DEFAULT current_timestamp +); + +-- Trading days: market calendar +CREATE TABLE IF NOT EXISTS trading_days ( + trade_date DATE PRIMARY KEY, + session_open TIMESTAMP, + session_close TIMESTAMP, + premarket_open TIMESTAMP, + afterhours_close TIMESTAMP, + is_half_day BOOLEAN DEFAULT FALSE, + is_holiday_adjacent BOOLEAN DEFAULT FALSE, + tape_regime VARCHAR, -- strong / neutral / weak + market_regime VARCHAR, -- bull / neutral / bear + volatility_regime VARCHAR, -- low / medium / high + gap_activity_level INTEGER, + momentum_density INTEGER +); + +-- Daily bars +CREATE TABLE IF NOT EXISTS daily_bars ( + symbol VARCHAR, + trade_date DATE, + open DOUBLE, + high DOUBLE, + low DOUBLE, + close DOUBLE, + volume BIGINT, + vwap DOUBLE, + previous_close DOUBLE, + true_range DOUBLE, + atr_14 DOUBLE, + rolling_avg_volume_20d DOUBLE, + rolling_avg_volume_50d DOUBLE, + prior_day_gain_pct DOUBLE, + multi_day_runner_flag BOOLEAN DEFAULT FALSE, + PRIMARY KEY (symbol, trade_date) +); + +-- Minute bars +CREATE TABLE IF NOT EXISTS minute_bars ( + symbol VARCHAR, + timestamp TIMESTAMP, + session_date DATE, + is_premarket BOOLEAN, + is_regular_hours BOOLEAN, + is_afterhours BOOLEAN, + open DOUBLE, + high DOUBLE, + low DOUBLE, + close DOUBLE, + volume BIGINT, + vwap DOUBLE, + spread_pct DOUBLE, + halt_status BOOLEAN DEFAULT FALSE, + source_provider VARCHAR DEFAULT 'synthetic', + quality_score DOUBLE, + PRIMARY KEY (symbol, timestamp) +); + +-- News / catalyst events +CREATE TABLE IF NOT EXISTS news_events ( + id VARCHAR, + symbol VARCHAR, + headline VARCHAR, + source VARCHAR, + published_at TIMESTAMP, + catalyst_freshness_minutes DOUBLE, + category VARCHAR, + sentiment DOUBLE, + is_earnings BOOLEAN DEFAULT FALSE, + is_offering BOOLEAN DEFAULT FALSE, + is_halt_related BOOLEAN DEFAULT FALSE, + PRIMARY KEY (id) +); + +-- Halt events +CREATE TABLE IF NOT EXISTS halt_events ( + id VARCHAR, + symbol VARCHAR, + halt_start TIMESTAMP, + halt_end TIMESTAMP, + halt_type VARCHAR, + price_at_halt DOUBLE, + resumed_price DOUBLE, + gap_on_resume_pct DOUBLE, + PRIMARY KEY (id) +); + +-- Market context (SPY/QQQ/VIX) +CREATE TABLE IF NOT EXISTS market_context ( + timestamp TIMESTAMP, + trade_date DATE, + spy_open DOUBLE, + spy_high DOUBLE, + spy_low DOUBLE, + spy_close DOUBLE, + spy_volume BIGINT, + qqq_open DOUBLE, + qqq_close DOUBLE, + spy_intraday_return_pct DOUBLE, + vix_open DOUBLE, + vix_level DOUBLE, + advance_decline_ratio DOUBLE, + up_volume_down_volume_ratio DOUBLE, + tape_regime_flag VARCHAR, + PRIMARY KEY (timestamp) +); + +-- Dilution / offering events +CREATE TABLE IF NOT EXISTS dilution_events ( + id VARCHAR PRIMARY KEY, + symbol VARCHAR, + event_type VARCHAR, + announced_at TIMESTAMP, + effective_date DATE, + share_count BIGINT, + price_per_share DOUBLE, + gross_proceeds DOUBLE +); + +-- Short interest history +CREATE TABLE IF NOT EXISTS short_interest ( + symbol VARCHAR, + report_date DATE, + short_interest_shares BIGINT, + short_interest_pct_float DOUBLE, + days_to_cover DOUBLE, + PRIMARY KEY (symbol, report_date) +); + +-- Versioned feature rows +CREATE TABLE IF NOT EXISTS engineered_features ( + id VARCHAR PRIMARY KEY, + timestamp TIMESTAMP, + symbol VARCHAR, + session_date DATE, + feature_version VARCHAR, + gap_pct DOUBLE, + premarket_gap_pct DOUBLE, + relative_volume DOUBLE, + time_of_day_adjusted_relative_volume DOUBLE, + float_rotation_pct DOUBLE, + vwap DOUBLE, + distance_from_vwap DOUBLE, + ema9 DOUBLE, + ema20 DOUBLE, + pullback_volume_ratio DOUBLE, + pullback_candle_type VARCHAR, + catalyst_freshness_minutes DOUBLE, + catalyst_type VARCHAR, + tape_regime VARCHAR, + spy_intraday_return DOUBLE, + vix_level DOUBLE, + metadata_json VARCHAR +); + +-- Setup events +CREATE TABLE IF NOT EXISTS setup_events ( + setup_id VARCHAR PRIMARY KEY, + symbol VARCHAR, + setup_time TIMESTAMP, + session_date DATE, + setup_name VARCHAR DEFAULT 'first_pullback', + setup_version VARCHAR DEFAULT 'v1', + entry_reference_price DOUBLE, + invalidation_price DOUBLE, + target_r_multiple DOUBLE DEFAULT 2.0, + impulse_start_price DOUBLE, + impulse_end_price DOUBLE, + impulse_pct DOUBLE, + pullback_low DOUBLE, + pullback_depth_pct DOUBLE, + pullback_bars INTEGER, + pullback_volume_ratio DOUBLE, + above_vwap_flag BOOLEAN, + vwap_at_trigger DOUBLE, + gap_pct DOUBLE, + relative_volume DOUBLE, + float_rotation_at_trigger DOUBLE, + catalyst_freshness_at_trigger DOUBLE, + session_minute_number INTEGER, + tape_regime_at_trigger VARCHAR, + spy_intraday_return_pct_at_trigger DOUBLE +); + +-- Outcome labels +CREATE TABLE IF NOT EXISTS outcome_labels ( + setup_id VARCHAR PRIMARY KEY, + label_version VARCHAR DEFAULT 'v1', + max_upside_next_5m DOUBLE, + max_upside_next_15m DOUBLE, + max_upside_next_60m DOUBLE, + max_drawdown_next_5m DOUBLE, + max_drawdown_next_15m DOUBLE, + max_drawdown_next_60m DOUBLE, + reached_1r_before_minus_1r BOOLEAN, + reached_2r_before_minus_1r BOOLEAN, + held_vwap_5m BOOLEAN, + held_vwap_15m BOOLEAN, + trend_day_flag BOOLEAN, + failed_breakout_flag BOOLEAN, + time_to_max_upside_minutes DOUBLE, + time_to_max_drawdown_minutes DOUBLE +); + +-- Backtest runs +CREATE TABLE IF NOT EXISTS backtest_runs ( + run_id VARCHAR PRIMARY KEY, + run_timestamp TIMESTAMP, + strategy_name VARCHAR, + strategy_version VARCHAR, + feature_version VARCHAR, + date_range_start DATE, + date_range_end DATE, + parameter_config_json VARCHAR, + summary_metrics_json VARCHAR, + total_trades INTEGER, + win_rate DOUBLE, + expectancy DOUBLE, + max_drawdown DOUBLE, + profit_factor DOUBLE +); + +-- Simulated orders for paper trading +CREATE TABLE IF NOT EXISTS simulated_orders ( + order_id VARCHAR PRIMARY KEY, + session_id VARCHAR, + symbol VARCHAR, + created_at TIMESTAMP, + side VARCHAR, + order_type VARCHAR, + qty INTEGER, + intended_entry DOUBLE, + stop_price DOUBLE, + target_price DOUBLE, + status VARCHAR, + strategy_name VARCHAR, + setup_id VARCHAR +); + +-- Simulated fills for paper trading +CREATE TABLE IF NOT EXISTS simulated_fills ( + fill_id VARCHAR PRIMARY KEY, + order_id VARCHAR, + symbol VARCHAR, + fill_time TIMESTAMP, + fill_price DOUBLE, + qty INTEGER, + slippage DOUBLE, + commission DOUBLE, + venue VARCHAR +); + +-- Strategy run metadata for paper/live sessions +CREATE TABLE IF NOT EXISTS strategy_runs ( + run_id VARCHAR PRIMARY KEY, + started_at TIMESTAMP, + ended_at TIMESTAMP, + mode VARCHAR, + strategy_name VARCHAR, + strategy_version VARCHAR, + config_json VARCHAR, + summary_json VARCHAR +); + +-- Daily paper trading session records +CREATE TABLE IF NOT EXISTS paper_sessions ( + session_id VARCHAR PRIMARY KEY, + trade_date DATE, + started_at TIMESTAMP, + ended_at TIMESTAMP, + starting_equity DOUBLE, + ending_equity DOUBLE, + realized_pnl DOUBLE, + unrealized_pnl DOUBLE, + total_orders INTEGER, + total_fills INTEGER, + total_setups INTEGER, + notes VARCHAR +); + +-- Live paper-trade missed-window / late-entry audit rows +CREATE TABLE IF NOT EXISTS paper_missed_trades ( + audit_id VARCHAR PRIMARY KEY, + session_id VARCHAR, + symbol VARCHAR, + setup_id VARCHAR, + reason VARCHAR, + setup_time TIMESTAMP, + evaluated_at TIMESTAMP, + attempted_entry_time TIMESTAMP, + entry_cutoff VARCHAR, + minutes_past_cutoff DOUBLE, + success_score_pct DOUBLE, + criteria_passed INTEGER, + criteria_total INTEGER, + provider_status VARCHAR, + provider_degraded_count INTEGER, + source_status VARCHAR, + source_degraded_count INTEGER, + source_backoff_count INTEGER, + created_at TIMESTAMP, + details_json VARCHAR +); + +-- Raw landing: one row per source+window fetch attempt (append-only) +CREATE TABLE IF NOT EXISTS raw_fetch_attempts ( + id VARCHAR PRIMARY KEY, + source VARCHAR NOT NULL, + capability VARCHAR NOT NULL, + request_window_start TIMESTAMP, + request_window_end TIMESTAMP, + fetched_at TIMESTAMP NOT NULL, + ingest_run_id VARCHAR, + http_status INTEGER, + item_count INTEGER, + error_msg VARCHAR, + payload_hash VARCHAR, + parser_version VARCHAR +); + +-- Raw landing: one row per raw news item (append-only) +CREATE TABLE IF NOT EXISTS raw_news_items ( + id VARCHAR PRIMARY KEY, + fetch_attempt_id VARCHAR, + source VARCHAR, + raw_url VARCHAR, + raw_title VARCHAR, + raw_published_at VARCHAR, + raw_body_snippet VARCHAR, + raw_tickers VARCHAR, + payload_hash VARCHAR, + parser_version VARCHAR, + fetched_at TIMESTAMP, + ingest_run_id VARCHAR +); + +-- Telemetry events: structured observability for fetch attempts, state transitions, enrichment outcomes +CREATE TABLE IF NOT EXISTS telemetry_events ( + id VARCHAR PRIMARY KEY, + event_type VARCHAR, + session_id VARCHAR, + symbol VARCHAR, + source VARCHAR, + from_state VARCHAR, + to_state VARCHAR, + reason VARCHAR, + metadata_json VARCHAR, + created_at TIMESTAMP DEFAULT current_timestamp +); + +-- Scanner Snapshots - Full scan state for historical tracking +CREATE TABLE IF NOT EXISTS scanner_snapshots ( + id VARCHAR PRIMARY KEY, + snapshot_time TIMESTAMP, + symbol VARCHAR, + rank INTEGER, + price DOUBLE, + gap_pct DOUBLE, + premarket_gap_pct DOUBLE, + cumulative_volume BIGINT, + minute_volume BIGINT, + premarket_volume BIGINT, + relative_volume DOUBLE, + float_rotation_pct DOUBLE, + vwap DOUBLE, + distance_from_vwap DOUBLE, + ema9 DOUBLE, + ema20 DOUBLE, + intraday_range DOUBLE, + intraday_range_vs_atr DOUBLE, + distance_to_premarket_high DOUBLE, + distance_to_premarket_low DOUBLE, + distance_to_day_high DOUBLE, + distance_to_day_low DOUBLE, + news_flag BOOLEAN, + catalyst_freshness_minutes DOUBLE, + catalyst_type VARCHAR, + float_bucket VARCHAR, + market_cap_bucket VARCHAR, + spread_pct DOUBLE, + halt_flag BOOLEAN, + dilution_risk_flag BOOLEAN, + sympathy_move_candidate_flag BOOLEAN, + lead_runner_symbol VARCHAR, + sympathy_cluster_id VARCHAR, + early_leader_flag BOOLEAN, + rank_stability DOUBLE, + momentum_score DOUBLE, + scanner_version VARCHAR +); +""" + + +def create_schema(con) -> None: + """Create all tables in the DuckDB connection.""" + for statement in SCHEMA_SQL.split(";"): + stmt = statement.strip() + if stmt: + con.execute(stmt) + con.execute( + "ALTER TABLE setup_events ADD COLUMN IF NOT EXISTS tape_regime_at_trigger VARCHAR" + ) + con.execute( + "ALTER TABLE setup_events ADD COLUMN IF NOT EXISTS spy_intraday_return_pct_at_trigger DOUBLE" + ) + con.execute( + "CREATE INDEX IF NOT EXISTS idx_raw_fetch_attempts_source_window " + "ON raw_fetch_attempts (source, capability, request_window_start, request_window_end)" + ) + con.execute( + "CREATE INDEX IF NOT EXISTS idx_raw_fetch_attempts_ingest_run " + "ON raw_fetch_attempts (ingest_run_id)" + ) + con.execute( + "CREATE INDEX IF NOT EXISTS idx_raw_news_items_fetch_attempt " + "ON raw_news_items (fetch_attempt_id)" + ) + con.execute( + "CREATE INDEX IF NOT EXISTS idx_raw_news_items_source_published " + "ON raw_news_items (source, raw_published_at)" + ) diff --git a/strategy/__init__.py b/strategy/__init__.py new file mode 100644 index 0000000..747845f --- /dev/null +++ b/strategy/__init__.py @@ -0,0 +1,11 @@ +"""Strategy core extraction package. + +Milestone 1 deliverables: +- Pure setup evaluation package +- Risk/cutoff logic package +- Deterministic tests for ready/blocked/late cases + +This package contains pure trading logic with no broker or UI dependency. +""" + +__version__ = "0.1.0" diff --git a/strategy/backtest/__init__.py b/strategy/backtest/__init__.py new file mode 100644 index 0000000..bfb019c --- /dev/null +++ b/strategy/backtest/__init__.py @@ -0,0 +1,6 @@ +"""Backtest package: deterministic engine and outcome labeling.""" + +from strategy.backtest.engine import BacktestEngine, BacktestResult, Trade +from strategy.backtest.outcomes import label_outcome, OutcomeLabel + +__all__ = ["BacktestEngine", "BacktestResult", "Trade", "label_outcome", "OutcomeLabel"] diff --git a/strategy/backtest/engine.py b/strategy/backtest/engine.py new file mode 100644 index 0000000..12f925a --- /dev/null +++ b/strategy/backtest/engine.py @@ -0,0 +1,197 @@ +"""Deterministic intraday backtest engine over minute bars. + +Walks bars chronologically, evaluates setups, fills at next bar open with +configurable slippage/commission, and exits on stop, target, or session end. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field + +import pandas as pd + +from config import BacktestConfig, Config +from strategy.evaluation.setup_evaluator import evaluate_setup +from strategy.risk.position_sizing import PositionSizingConfig, calculate_position_size + + +@dataclass +class Trade: + trade_id: str + symbol: str + entry_time: str + entry_price: float + stop_price: float + target_price: float + quantity: int + exit_time: str | None = None + exit_price: float | None = None + exit_reason: str | None = None + realized_pnl: float | None = None + r_multiple: float | None = None + + +@dataclass +class BacktestResult: + symbol: str + trades: list[Trade] = field(default_factory=list) + total_pnl: float = 0.0 + win_rate: float = 0.0 + evaluations: int = 0 + signals: int = 0 + + def summary(self) -> dict: + wins = [t for t in self.trades if (t.realized_pnl or 0) > 0] + return { + "symbol": self.symbol, + "trades": len(self.trades), + "wins": len(wins), + "win_rate": round(self.win_rate, 3), + "total_pnl": round(self.total_pnl, 2), + "evaluations": self.evaluations, + "signals": self.signals, + } + + +class BacktestEngine: + """Single-symbol intraday backtest over a session of minute bars.""" + + def __init__( + self, + config: Config | None = None, + equity: float = 100_000.0, + target_r: float = 2.0, + warmup_bars: int = 15, + eval_every: int = 5, + ): + self.config = config or Config() + self.bt: BacktestConfig = self.config.backtest + self.equity = equity + self.target_r = target_r + self.warmup_bars = warmup_bars + self.eval_every = eval_every + + def _entry_costs(self, price: float, quantity: int) -> tuple[float, float]: + slip = price * self.bt.slippage.base_spread_pct + commission = max( + self.bt.commission.minimum, quantity * self.bt.commission.per_share + ) + return slip, commission + + def run( + self, + bars: pd.DataFrame, + symbol: str, + previous_close: float | None = None, + avg_daily_volume: float | None = None, + ) -> BacktestResult: + result = BacktestResult(symbol=symbol) + bars = bars.reset_index(drop=True) + open_trade: Trade | None = None + + i = self.warmup_bars + while i < len(bars) - 1: + bar = bars.iloc[i] + + if open_trade is not None: + low, high = float(bar["low"]), float(bar["high"]) + exit_price = None + reason = None + if low <= open_trade.stop_price: + exit_price, reason = open_trade.stop_price, "stop_loss" + elif high >= open_trade.target_price: + exit_price, reason = open_trade.target_price, "target" + elif i == len(bars) - 2: + exit_price, reason = float(bar["close"]), "session_end" + if exit_price is not None: + self._close(open_trade, bar, exit_price, reason, result) + open_trade = None + i += 1 + continue + + if i % self.eval_every == 0: + window = bars.iloc[: i + 1] + # Evaluate as of the current bar's timestamp, not wall-clock + # now() — otherwise every setup is judged against the real + # clock and marked "late" past the entry cutoff (so a Sunday + # backtest would show zero trades). Bars are stored UTC-naive; + # the cutoff reasons in US/Eastern, so convert. + eval_time = None + try: + from datetime import timezone as _tz + from zoneinfo import ZoneInfo as _ZI + + last_ts = pd.Timestamp(window["timestamp"].iloc[-1]).to_pydatetime() + if last_ts.tzinfo is None: + last_ts = last_ts.replace(tzinfo=_tz.utc) + eval_time = last_ts.astimezone(_ZI("America/New_York")).replace( + tzinfo=None + ) + except Exception: # noqa: BLE001 - fall back to now() + eval_time = None + + evaluation = evaluate_setup( + window, + previous_close=previous_close, + avg_daily_volume=avg_daily_volume, + evaluation_time=eval_time, + ) + result.evaluations += 1 + if evaluation.status == "ready" and evaluation.setups: + result.signals += 1 + setup = evaluation.setups[0] + next_bar = bars.iloc[i + 1] + raw_entry = ( + float(next_bar["open"]) + if self.bt.entry_mode == "next_bar" + else float(bar["close"]) + ) + stop = float(setup["stop_loss_price"]) + sizing = calculate_position_size( + raw_entry, stop, equity=self.equity, + config=PositionSizingConfig(), + ) + if sizing.position_size > 0: + slip, commission = self._entry_costs( + raw_entry, sizing.position_size + ) + entry = raw_entry + slip + risk = entry - stop + open_trade = Trade( + trade_id=str(uuid.uuid4()), + symbol=symbol, + entry_time=str(next_bar.get("timestamp", i + 1)), + entry_price=round(entry, 4), + stop_price=round(stop, 4), + target_price=round(entry + self.target_r * risk, 4), + quantity=sizing.position_size, + ) + open_trade.realized_pnl = -commission + i += 1 # consumed next bar for entry + i += 1 + + if open_trade is not None: + last = bars.iloc[-1] + self._close(open_trade, last, float(last["close"]), "session_end", result) + + if result.trades: + wins = sum(1 for t in result.trades if (t.realized_pnl or 0) > 0) + result.win_rate = wins / len(result.trades) + return result + + def _close( + self, trade: Trade, bar, exit_price: float, reason: str | None, + result: BacktestResult, + ) -> None: + slip, commission = self._entry_costs(exit_price, trade.quantity) + fill = exit_price - slip + gross = (fill - trade.entry_price) * trade.quantity + trade.exit_time = str(bar.get("timestamp", "end")) + trade.exit_price = round(fill, 4) + trade.exit_reason = reason + trade.realized_pnl = round((trade.realized_pnl or 0.0) + gross - commission, 2) + risk = trade.entry_price - trade.stop_price + trade.r_multiple = round((fill - trade.entry_price) / risk, 3) if risk > 0 else 0.0 + result.trades.append(trade) + result.total_pnl += trade.realized_pnl diff --git a/strategy/backtest/outcomes.py b/strategy/backtest/outcomes.py new file mode 100644 index 0000000..39ed814 --- /dev/null +++ b/strategy/backtest/outcomes.py @@ -0,0 +1,67 @@ +"""Outcome labeling for setups (R-multiple based).""" + +from __future__ import annotations + +from dataclasses import dataclass + +import pandas as pd + + +@dataclass +class OutcomeLabel: + """Outcome of a setup given subsequent bars.""" + + setup_id: str + hit_target: bool + hit_stop: bool + r_multiple: float + max_favorable_r: float + max_adverse_r: float + bars_to_resolution: int + label: str # "win" | "loss" | "scratch" | "unresolved" + + +def label_outcome( + setup_id: str, + entry_price: float, + stop_price: float, + bars_after: pd.DataFrame, + target_r: float = 2.0, +) -> OutcomeLabel: + """Walk forward through bars to see whether target or stop hit first.""" + risk = entry_price - stop_price + if risk <= 0 or bars_after.empty: + return OutcomeLabel(setup_id, False, False, 0.0, 0.0, 0.0, 0, "unresolved") + + target_price = entry_price + target_r * risk + max_fav = 0.0 + max_adv = 0.0 + + for i, (_, bar) in enumerate(bars_after.iterrows(), start=1): + high = float(bar["high"]) + low = float(bar["low"]) + max_fav = max(max_fav, (high - entry_price) / risk) + max_adv = min(max_adv, (low - entry_price) / risk) + + stop_hit = low <= stop_price + target_hit = high >= target_price + if stop_hit and target_hit: + # Conservative: assume stop hit first within the bar + return OutcomeLabel( + setup_id, False, True, -1.0, max_fav, max_adv, i, "loss" + ) + if stop_hit: + return OutcomeLabel( + setup_id, False, True, -1.0, max_fav, max_adv, i, "loss" + ) + if target_hit: + return OutcomeLabel( + setup_id, True, False, target_r, max_fav, max_adv, i, "win" + ) + + final_r = (float(bars_after["close"].iloc[-1]) - entry_price) / risk + label = "scratch" if abs(final_r) < 0.25 else ("win" if final_r > 0 else "loss") + return OutcomeLabel( + setup_id, False, False, round(final_r, 3), max_fav, max_adv, + len(bars_after), label, + ) diff --git a/strategy/evaluation/__init__.py b/strategy/evaluation/__init__.py new file mode 100644 index 0000000..d579b55 --- /dev/null +++ b/strategy/evaluation/__init__.py @@ -0,0 +1,103 @@ +"""Strategy evaluation package. + +Implements Ross Cameron style momentum trading strategy: +- Structure detection (bull flags, first pullback, gap & go) +- Key levels tracking (VWAP, premarket high, opening range) +- Setup quality scoring +- Market regime detection +""" + +from strategy.evaluation.setup_evaluator import evaluate_setup +from strategy.evaluation.criteria import score_criteria, build_criteria_result +from strategy.evaluation.structure import ( + classify_setup, + detect_bull_flag, + detect_first_pullback, + detect_gap_and_go, + detect_hod_break, + StructureDetectionResult, + SetupType, +) +from strategy.evaluation.levels import ( + compute_key_levels, + calculate_vwap, + calculate_ema, + KeyLevels, + get_trigger_levels, + get_stop_levels, +) +from strategy.evaluation.quality import ( + calculate_setup_quality, + QualityScore, + QualityThresholds, +) +from strategy.evaluation.regime import ( + assess_market_regime, + get_regime_adjustment, + should_trade_in_regime, + MarketRegime, + RegimeAssessment, +) +from strategy.evaluation.first_candles import ( + calculate_first_candle_features, + classify_opening_strength, + FirstCandleFeatures, +) +from strategy.evaluation.volume_metrics import ( + calculate_enhanced_volume_metrics, + calculate_time_of_day_rvol, + calculate_float_rotation, + VolumeMetrics, +) +from strategy.evaluation.data_quality import ( + calculate_data_quality_score, + should_trade_symbol, + get_quality_grade, + DataQualityScore, +) + +__all__ = [ + # Setup evaluation + "evaluate_setup", + "score_criteria", + "build_criteria_result", + # Structure detection + "classify_setup", + "detect_bull_flag", + "detect_first_pullback", + "detect_gap_and_go", + "detect_hod_break", + "StructureDetectionResult", + "SetupType", + # Levels + "compute_key_levels", + "calculate_vwap", + "calculate_ema", + "KeyLevels", + "get_trigger_levels", + "get_stop_levels", + # Quality + "calculate_setup_quality", + "QualityScore", + "QualityThresholds", + # Regime + "assess_market_regime", + "get_regime_adjustment", + "should_trade_in_regime", + "MarketRegime", + "RegimeAssessment", + # First candles + "calculate_first_candle_features", + "classify_opening_strength", + "FirstCandleFeatures", + # Volume metrics + "calculate_enhanced_volume_metrics", + "calculate_time_of_day_rvol", + "calculate_float_rotation", + "VolumeMetrics", + # Data quality + "calculate_data_quality_score", + "should_trade_symbol", + "get_quality_grade", + "DataQualityScore", +] diff --git a/strategy/evaluation/criteria.py b/strategy/evaluation/criteria.py new file mode 100644 index 0000000..937d595 --- /dev/null +++ b/strategy/evaluation/criteria.py @@ -0,0 +1,26 @@ +"""Criteria scoring logic.""" + +from typing import TYPE_CHECKING +from strategy.models import CriteriaWeights, CriteriaResult + +if TYPE_CHECKING: + pass + + +def score_criteria( + results: list[CriteriaResult], + weights: CriteriaWeights, +) -> tuple[int, int, float]: + """Score criteria results and return count, total, and percentage.""" + passed = [r for r in results if r.passed] + weighted_sum = sum(getattr(weights, r.name) for r in passed) + total = sum(getattr(weights, r.name) for r in results) + pct = min(100.0, max(0.0, (weighted_sum / total * 100))) if total > 0 else 0.0 + return len(passed), len(results), pct + + +def build_criteria_result( + name: str, passed: bool, reason: str | None = None +) -> CriteriaResult: + """Build a criteria result.""" + return CriteriaResult(name=name, passed=passed, reason=reason) diff --git a/strategy/evaluation/data_quality.py b/strategy/evaluation/data_quality.py new file mode 100644 index 0000000..dd12ada --- /dev/null +++ b/strategy/evaluation/data_quality.py @@ -0,0 +1,81 @@ +"""Data quality scoring for minute-bar coverage and integrity.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import pandas as pd + + +@dataclass +class DataQualityScore: + """Data quality assessment for a symbol's session bars.""" + + score: float = 0.0 + grade: str = "F" + bar_count: int = 0 + coverage: float = 0.0 + max_gap_minutes: float = 0.0 + has_zero_volume_run: bool = False + issues: list[str] = field(default_factory=list) + + +def calculate_data_quality_score( + bars: pd.DataFrame, + expected_bars: int | None = None, +) -> DataQualityScore: + """Score bar data 0..1 from coverage, gaps, and volume integrity.""" + dq = DataQualityScore() + if bars.empty: + dq.issues.append("no_bars") + return dq + + dq.bar_count = len(bars) + expected = expected_bars or max(len(bars), 30) + dq.coverage = min(1.0, len(bars) / expected) + + score = dq.coverage + + ts = pd.to_datetime(bars["timestamp"]) if "timestamp" in bars.columns else None + if ts is not None and len(ts) > 1: + gaps = ts.diff().dt.total_seconds().fillna(60) / 60.0 + dq.max_gap_minutes = float(gaps.max()) + if dq.max_gap_minutes > 10: + score -= 0.2 + dq.issues.append(f"gap_{dq.max_gap_minutes:.0f}m") + + zero_run = (bars["volume"].astype(float) <= 0).rolling(5).sum().max() + if zero_run is not None and zero_run >= 5: + dq.has_zero_volume_run = True + score -= 0.2 + dq.issues.append("zero_volume_run") + + bad_prices = ( + (bars["high"] < bars["low"]) + | (bars["close"] <= 0) + | (bars["open"] <= 0) + ).sum() + if bad_prices: + score -= 0.3 + dq.issues.append(f"bad_prices_{int(bad_prices)}") + + dq.score = max(0.0, min(1.0, score)) + dq.grade = get_quality_grade(dq.score) + return dq + + +def get_quality_grade(score: float) -> str: + if score >= 0.9: + return "A" + if score >= 0.75: + return "B" + if score >= 0.6: + return "C" + if score >= 0.4: + return "D" + return "F" + + +def should_trade_symbol(dq: DataQualityScore, min_score: float = 0.6) -> bool: + """Whether data quality is sufficient to act on signals.""" + return dq.score >= min_score diff --git a/strategy/evaluation/first_candles.py b/strategy/evaluation/first_candles.py new file mode 100644 index 0000000..25b1860 --- /dev/null +++ b/strategy/evaluation/first_candles.py @@ -0,0 +1,61 @@ +"""First-candle features: opening drive strength classification.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import pandas as pd + + +@dataclass +class FirstCandleFeatures: + """Features of the first regular-hours candles.""" + + first_candle_range_pct: float = 0.0 + first_candle_body_pct: float = 0.0 + first_candle_green: bool = False + first_candle_volume: int = 0 + open_above_premarket_high: bool = False + opening_strength: str = "neutral" + + +def calculate_first_candle_features( + bars: pd.DataFrame, + premarket_high: float | None = None, + n_candles: int = 1, +) -> FirstCandleFeatures: + """Compute features from the first regular-hours candle(s).""" + feats = FirstCandleFeatures() + if bars.empty: + return feats + + if "is_regular_hours" in bars.columns: + rth = bars[bars["is_regular_hours"].astype(bool)] + else: + rth = bars + if rth.empty: + return feats + + first = rth.head(max(1, n_candles)) + o = float(first["open"].iloc[0]) + c = float(first["close"].iloc[-1]) + h = float(first["high"].max()) + lo = float(first["low"].min()) + if o > 0: + feats.first_candle_range_pct = (h - lo) / o + feats.first_candle_body_pct = abs(c - o) / o + feats.first_candle_green = c > o + feats.first_candle_volume = int(first["volume"].sum()) + if premarket_high is not None: + feats.open_above_premarket_high = o > premarket_high + feats.opening_strength = classify_opening_strength(feats) + return feats + + +def classify_opening_strength(feats: FirstCandleFeatures) -> str: + """Classify opening strength as strong / neutral / weak.""" + if feats.first_candle_green and feats.first_candle_body_pct >= 0.02: + return "strong" + if not feats.first_candle_green and feats.first_candle_body_pct >= 0.02: + return "weak" + return "neutral" diff --git a/strategy/evaluation/levels.py b/strategy/evaluation/levels.py new file mode 100644 index 0000000..51ebd45 --- /dev/null +++ b/strategy/evaluation/levels.py @@ -0,0 +1,116 @@ +"""Key intraday levels: VWAP, EMAs, premarket high/low, opening range. + +Pure functions over OHLCV DataFrames — no broker or UI dependencies. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd + + +@dataclass +class KeyLevels: + """Computed key levels for a symbol's session.""" + + vwap: float | None = None + ema_9: float | None = None + ema_20: float | None = None + premarket_high: float | None = None + premarket_low: float | None = None + opening_range_high: float | None = None + opening_range_low: float | None = None + high_of_day: float | None = None + low_of_day: float | None = None + previous_close: float | None = None + extras: dict = field(default_factory=dict) + + +def calculate_vwap(bars: pd.DataFrame) -> pd.Series: + """Cumulative volume-weighted average price over the given bars.""" + if bars.empty: + return pd.Series(dtype=float) + typical = (bars["high"] + bars["low"] + bars["close"]) / 3.0 + vol = bars["volume"].astype(float).clip(lower=0) + cum_vol = vol.cumsum().replace(0, np.nan) + vwap = (typical * vol).cumsum() / cum_vol + return vwap.ffill().fillna(typical) + + +def calculate_ema(values: pd.Series, period: int) -> pd.Series: + """Exponential moving average.""" + if values.empty: + return pd.Series(dtype=float) + return values.ewm(span=period, adjust=False).mean() + + +def compute_key_levels( + bars: pd.DataFrame, + previous_close: float | None = None, + opening_range_minutes: int = 5, +) -> KeyLevels: + """Compute key levels from a session's minute bars. + + Args: + bars: minute bars (chronological) with open/high/low/close/volume and + optionally is_premarket / is_regular_hours boolean columns. + previous_close: prior session close, if known. + opening_range_minutes: bars to include in the opening range. + """ + levels = KeyLevels(previous_close=previous_close) + if bars.empty: + return levels + + bars = bars.reset_index(drop=True) + vwap_series = calculate_vwap(bars) + levels.vwap = float(vwap_series.iloc[-1]) + closes = bars["close"].astype(float) + levels.ema_9 = float(calculate_ema(closes, 9).iloc[-1]) + levels.ema_20 = float(calculate_ema(closes, 20).iloc[-1]) + levels.high_of_day = float(bars["high"].max()) + levels.low_of_day = float(bars["low"].min()) + + if "is_premarket" in bars.columns: + pm = bars[bars["is_premarket"].astype(bool)] + if not pm.empty: + levels.premarket_high = float(pm["high"].max()) + levels.premarket_low = float(pm["low"].min()) + + if "is_regular_hours" in bars.columns: + rth = bars[bars["is_regular_hours"].astype(bool)] + else: + rth = bars + if not rth.empty: + opening = rth.head(opening_range_minutes) + levels.opening_range_high = float(opening["high"].max()) + levels.opening_range_low = float(opening["low"].min()) + + return levels + + +def get_trigger_levels(levels: KeyLevels) -> list[tuple[str, float]]: + """Candidate breakout trigger levels, ordered by priority.""" + out: list[tuple[str, float]] = [] + if levels.high_of_day is not None: + out.append(("high_of_day", levels.high_of_day)) + if levels.premarket_high is not None: + out.append(("premarket_high", levels.premarket_high)) + if levels.opening_range_high is not None: + out.append(("opening_range_high", levels.opening_range_high)) + return out + + +def get_stop_levels(levels: KeyLevels, entry_price: float) -> list[tuple[str, float]]: + """Candidate protective stop levels below the entry, ordered tightest first.""" + candidates: list[tuple[str, float]] = [] + if levels.vwap is not None and levels.vwap < entry_price: + candidates.append(("vwap", levels.vwap)) + if levels.ema_9 is not None and levels.ema_9 < entry_price: + candidates.append(("ema_9", levels.ema_9)) + if levels.opening_range_low is not None and levels.opening_range_low < entry_price: + candidates.append(("opening_range_low", levels.opening_range_low)) + if levels.low_of_day is not None and levels.low_of_day < entry_price: + candidates.append(("low_of_day", levels.low_of_day)) + return sorted(candidates, key=lambda kv: entry_price - kv[1]) diff --git a/strategy/evaluation/llm_integration.py b/strategy/evaluation/llm_integration.py new file mode 100644 index 0000000..291d7b8 --- /dev/null +++ b/strategy/evaluation/llm_integration.py @@ -0,0 +1,36 @@ +"""Optional LLM (Ollama) enrichment hooks. Disabled unless configured.""" + +from __future__ import annotations + +import json +import logging +import urllib.request + +logger = logging.getLogger(__name__) + + +def summarize_setup_with_llm( + setup: dict, + host: str = "http://localhost:11434", + model: str = "mistral", + timeout: int = 15, +) -> str | None: + """Ask a local Ollama model to summarize a setup. Returns None on failure.""" + prompt = ( + "Summarize this momentum trading setup in one sentence:\n" + + json.dumps(setup, default=str) + ) + try: + req = urllib.request.Request( + f"{host}/api/generate", + data=json.dumps( + {"model": model, "prompt": prompt, "stream": False} + ).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = json.loads(resp.read().decode()) + return body.get("response") + except Exception: + logger.debug("LLM enrichment unavailable", exc_info=True) + return None diff --git a/strategy/evaluation/quality.py b/strategy/evaluation/quality.py new file mode 100644 index 0000000..cd45eea --- /dev/null +++ b/strategy/evaluation/quality.py @@ -0,0 +1,83 @@ +"""Setup quality scoring.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class QualityThresholds: + """Thresholds used to grade setup quality.""" + + a_grade: float = 0.80 + b_grade: float = 0.65 + c_grade: float = 0.50 + min_tradeable: float = 0.50 + + +@dataclass +class QualityScore: + """Composite quality score for a detected setup.""" + + score: float = 0.0 + grade: str = "F" + components: dict = field(default_factory=dict) + tradeable: bool = False + + +def calculate_setup_quality( + gap_pct: float, + relative_volume: float, + structure_quality: float, + above_vwap: bool, + opening_strength: str = "neutral", + data_quality: float = 1.0, + thresholds: QualityThresholds | None = None, +) -> QualityScore: + """Blend setup ingredients into one 0..1 quality score. + + Weights: gap 20%, RVOL 25%, structure 30%, VWAP 10%, + opening strength 5%, data quality 10%. + """ + thresholds = thresholds or QualityThresholds() + + gap_component = min(1.0, max(0.0, gap_pct / 0.10)) + rvol_component = min(1.0, max(0.0, relative_volume / 5.0)) + structure_component = min(1.0, max(0.0, structure_quality)) + vwap_component = 1.0 if above_vwap else 0.0 + opening_component = {"strong": 1.0, "neutral": 0.5, "weak": 0.0}.get( + opening_strength, 0.5 + ) + dq_component = min(1.0, max(0.0, data_quality)) + + score = ( + 0.20 * gap_component + + 0.25 * rvol_component + + 0.30 * structure_component + + 0.10 * vwap_component + + 0.05 * opening_component + + 0.10 * dq_component + ) + + if score >= thresholds.a_grade: + grade = "A" + elif score >= thresholds.b_grade: + grade = "B" + elif score >= thresholds.c_grade: + grade = "C" + else: + grade = "F" + + return QualityScore( + score=round(score, 4), + grade=grade, + components={ + "gap": gap_component, + "relative_volume": rvol_component, + "structure": structure_component, + "vwap": vwap_component, + "opening_strength": opening_component, + "data_quality": dq_component, + }, + tradeable=score >= thresholds.min_tradeable, + ) diff --git a/strategy/evaluation/regime.py b/strategy/evaluation/regime.py new file mode 100644 index 0000000..920165e --- /dev/null +++ b/strategy/evaluation/regime.py @@ -0,0 +1,55 @@ +"""Market regime assessment (hot / normal / cold tape).""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + + +class MarketRegime(str, Enum): + HOT = "hot" + NORMAL = "normal" + COLD = "cold" + + +@dataclass +class RegimeAssessment: + regime: MarketRegime + score: float + reason: str + + +def assess_market_regime( + gappers_count: int = 0, + avg_gap_pct: float = 0.0, + spy_change_pct: float = 0.0, +) -> RegimeAssessment: + """Assess the tape from breadth of gappers and index behavior.""" + score = 0.0 + score += min(1.0, gappers_count / 10.0) * 0.5 + score += min(1.0, max(0.0, avg_gap_pct / 0.15)) * 0.3 + score += min(1.0, max(0.0, (spy_change_pct + 0.01) / 0.02)) * 0.2 + + if score >= 0.65: + regime = MarketRegime.HOT + reason = f"{gappers_count} gappers, avg gap {avg_gap_pct:.1%}" + elif score >= 0.35: + regime = MarketRegime.NORMAL + reason = "moderate breadth" + else: + regime = MarketRegime.COLD + reason = "few gappers / weak tape" + return RegimeAssessment(regime=regime, score=round(score, 3), reason=reason) + + +def get_regime_adjustment(regime: MarketRegime) -> float: + """Position-size multiplier for the regime.""" + return {MarketRegime.HOT: 1.0, MarketRegime.NORMAL: 0.75, MarketRegime.COLD: 0.5}[ + regime + ] + + +def should_trade_in_regime(regime: MarketRegime, allow_cold: bool = False) -> bool: + if regime == MarketRegime.COLD: + return allow_cold + return True diff --git a/strategy/evaluation/setup_evaluator.py b/strategy/evaluation/setup_evaluator.py new file mode 100644 index 0000000..6e24604 --- /dev/null +++ b/strategy/evaluation/setup_evaluator.py @@ -0,0 +1,205 @@ +"""Setup evaluator — the heart of Milestone 1. + +Pure function: bars + context in, SetupEvaluationResult (ready/blocked/late) out. +No broker, DB, or UI dependencies. +""" + +from __future__ import annotations + +from datetime import datetime, time + +import pandas as pd + +from strategy.models import ( + CriteriaWeights, + SetupCriteria, + SetupEvaluationResult, +) +from strategy.evaluation.criteria import build_criteria_result, score_criteria +from strategy.evaluation.levels import compute_key_levels, get_stop_levels +from strategy.evaluation.structure import SetupType, classify_setup +from strategy.evaluation.quality import calculate_setup_quality +from strategy.evaluation.first_candles import calculate_first_candle_features +from strategy.evaluation.volume_metrics import calculate_enhanced_volume_metrics +from strategy.evaluation.data_quality import calculate_data_quality_score +from strategy.risk.entry_cuts import EntryCutoffConfig, check_entry_cutoff + +MIN_BARS = 10 + + +def evaluate_setup( + bars: pd.DataFrame, + previous_close: float | None = None, + avg_daily_volume: float | None = None, + criteria: SetupCriteria | None = None, + weights: CriteriaWeights | None = None, + evaluation_time: datetime | None = None, + entry_cutoff: EntryCutoffConfig | None = None, + ready_score_pct: float = 60.0, +) -> SetupEvaluationResult: + """Evaluate a symbol's session bars for a momentum setup. + + Status semantics: + ready — criteria score >= ready_score_pct, valid structure, before cutoff + late — would be ready but the entry cutoff has passed + blocked — anything else (with the dominant blocking reason) + """ + criteria = criteria or SetupCriteria() + weights = weights or CriteriaWeights() + evaluation_time = evaluation_time or datetime.now() + + if bars is None or bars.empty or len(bars) < MIN_BARS: + return _blocked( + reason=f"insufficient_data ({0 if bars is None else len(bars)} bars, need {MIN_BARS})", + evaluation_time=evaluation_time, + ) + + bars = bars.reset_index(drop=True) + price = float(bars["close"].iloc[-1]) + open_price = float(bars["open"].iloc[0]) + ref_close = previous_close if previous_close and previous_close > 0 else open_price + gap_pct = (open_price - ref_close) / ref_close if ref_close > 0 else 0.0 + intraday_change = (price - open_price) / open_price if open_price > 0 else 0.0 + + dq = calculate_data_quality_score(bars) + vol = calculate_enhanced_volume_metrics(bars, avg_daily_volume=avg_daily_volume) + relative_volume = vol.relative_volume if avg_daily_volume else max( + 1.0, vol.last_bar_volume / vol.avg_bar_volume if vol.avg_bar_volume else 1.0 + ) + levels = compute_key_levels(bars, previous_close=previous_close) + structure = classify_setup( + bars, + premarket_high=levels.premarket_high, + session_high=levels.high_of_day, + min_quality_score=0.2, + ) + first = calculate_first_candle_features(bars, premarket_high=levels.premarket_high) + above_vwap = levels.vwap is not None and price >= levels.vwap + + results = [ + build_criteria_result( + "sufficient_data", + dq.score >= criteria.min_quality_score, + f"quality {dq.score:.2f} (min {criteria.min_quality_score})", + ), + build_criteria_result( + "gap", + gap_pct >= criteria.gap_pct_min or intraday_change >= criteria.gap_pct_min, + f"gap {gap_pct:.1%} / intraday {intraday_change:.1%} (min {criteria.gap_pct_min:.0%})", + ), + build_criteria_result( + "relative_volume", + relative_volume >= criteria.relative_volume_min, + f"rvol {relative_volume:.1f} (min {criteria.relative_volume_min})", + ), + build_criteria_result( + "impulse", + structure.is_valid and structure.setup_type != SetupType.NONE, + f"structure {structure.setup_type.value}: {structure.reason or 'ok'}", + ), + build_criteria_result( + "pullback", + structure.setup_type + in (SetupType.FIRST_PULLBACK, SetupType.BULL_FLAG, SetupType.GAP_AND_GO, + SetupType.HOD_BREAK, SetupType.CONTINUATION_FALLBACK) + and structure.is_valid, + "pullback/consolidation structure", + ), + build_criteria_result( + "pullback_volume", + structure.quality_score >= 0.3, + f"structure quality {structure.quality_score:.2f}", + ), + build_criteria_result("vwap", above_vwap, f"price vs VWAP {levels.vwap}"), + build_criteria_result( + "candle_quality", + first.opening_strength != "weak", + f"opening {first.opening_strength}", + ), + build_criteria_result( + "breakout", + structure.breakout_level is not None + and price >= structure.breakout_level * 0.995, + f"price {price:.2f} vs breakout {structure.breakout_level}", + ), + ] + + passed_n, total_n, score_pct = score_criteria(results, weights) + passed_names = [r.name for r in results if r.passed] + failed_names = [r.name for r in results if not r.passed] + + quality = calculate_setup_quality( + gap_pct=max(gap_pct, intraday_change), + relative_volume=relative_volume, + structure_quality=structure.quality_score, + above_vwap=above_vwap, + opening_strength=first.opening_strength, + data_quality=dq.score, + ) + + entry_price = structure.breakout_level or price + stop_candidates = get_stop_levels(levels, entry_price) + stop_price = structure.stop_level or ( + stop_candidates[0][1] if stop_candidates else round(entry_price * 0.97, 4) + ) + + setups = [] + if structure.is_valid and structure.setup_type != SetupType.NONE: + setups.append( + { + "setup_type": structure.setup_type.value, + "entry_price": round(float(entry_price), 4), + "stop_loss_price": round(float(stop_price), 4), + "quality_score": quality.score, + "quality_grade": quality.grade, + "confidence": round(score_pct / 100.0, 4), + } + ) + + base = dict( + evaluated_at=evaluation_time.isoformat(), + price=price, + gap_pct=round(gap_pct, 6), + relative_volume=round(relative_volume, 4), + criteria_passed=passed_n, + criteria_total=total_n, + success_score_pct=round(score_pct, 2), + criteria_names_passed=passed_names, + criteria_names_failed=failed_names, + setups=setups, + ) + + if score_pct < ready_score_pct or not setups: + reason = ( + f"score {score_pct:.0f}% < {ready_score_pct:.0f}%" + if score_pct < ready_score_pct + else "no valid setup structure" + ) + if failed_names: + reason += f" (failed: {', '.join(failed_names)})" + return SetupEvaluationResult(status="blocked", reason=reason, **base) + + cutoff = check_entry_cutoff( + trigger_time=time(evaluation_time.hour, evaluation_time.minute), + cutoff_config=entry_cutoff or EntryCutoffConfig(), + ) + if not cutoff.passed: + return SetupEvaluationResult( + status="late", reason=cutoff.reason or "past entry cutoff", **base + ) + + return SetupEvaluationResult(status="ready", reason=None, **base) + + +def _blocked(reason: str, evaluation_time: datetime) -> SetupEvaluationResult: + return SetupEvaluationResult( + status="blocked", + reason=reason, + evaluated_at=evaluation_time.isoformat(), + price=0.0, + gap_pct=0.0, + relative_volume=0.0, + criteria_passed=0, + criteria_total=0, + success_score_pct=0.0, + ) diff --git a/strategy/evaluation/storage.py b/strategy/evaluation/storage.py new file mode 100644 index 0000000..16f258a --- /dev/null +++ b/strategy/evaluation/storage.py @@ -0,0 +1,33 @@ +"""Persist setup evaluations into the research DB (setup_events table).""" + +from __future__ import annotations + +import uuid +from datetime import datetime + + +def store_setup_event(con, symbol: str, session_date, result: dict) -> str: + """Insert one evaluation result into setup_events. Returns setup_id.""" + setup_id = str(uuid.uuid4()) + setups = result.get("setups") or [] + best = setups[0] if setups else {} + con.execute( + """ + INSERT OR REPLACE INTO setup_events + (setup_id, symbol, setup_time, session_date, setup_name, + entry_reference_price, invalidation_price, gap_pct, relative_volume) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + setup_id, + symbol, + datetime.now(), + session_date, + best.get("setup_type") or "first_pullback", + best.get("entry_price"), + best.get("stop_loss_price"), + result.get("gap_pct"), + result.get("relative_volume"), + ], + ) + return setup_id diff --git a/strategy/evaluation/structure.py b/strategy/evaluation/structure.py new file mode 100644 index 0000000..4911386 --- /dev/null +++ b/strategy/evaluation/structure.py @@ -0,0 +1,570 @@ +"""Market structure detection for momentum setups. + +Implements bull flag, first pullback, and gap-and-go pattern detection +based on Ross Cameron momentum strategy specifications. +""" + +from dataclasses import dataclass +from enum import Enum +from typing import Literal + +import numpy as np +import pandas as pd + + +class SetupType(Enum): + """Types of momentum setups.""" + + GAP_AND_GO = "gap_and_go" + FIRST_PULLBACK = "first_pullback" + BULL_FLAG = "bull_flag" + HOD_BREAK = "hod_break" + CONTINUATION_FALLBACK = "continuation_fallback" + NONE = "none" + + +@dataclass +class StructureDetectionResult: + """Result of structure detection.""" + + setup_type: SetupType + is_valid: bool + impulse_start_idx: int | None = None + impulse_end_idx: int | None = None + consolidation_start_idx: int | None = None + consolidation_end_idx: int | None = None + breakout_level: float | None = None + stop_level: float | None = None + quality_score: float = 0.0 + reason: str | None = None + + +def detect_bull_flag( + bars: pd.DataFrame, + min_impulse_bars: int = 3, + max_consolidation_bars: int = 10, + min_impulse_pct: float = 0.03, +) -> StructureDetectionResult: + """Detect bull flag pattern. + + Pattern: Strong impulse leg up, followed by consolidation + with contracting range, then breakout. + + Args: + bars: OHLCV DataFrame + min_impulse_bars: Minimum bars for impulse leg + max_consolidation_bars: Maximum bars for consolidation + min_impulse_pct: Minimum impulse percentage + + Returns: + StructureDetectionResult with pattern details + """ + if len(bars) < min_impulse_bars + 2: + return StructureDetectionResult( + setup_type=SetupType.BULL_FLAG, + is_valid=False, + reason="Insufficient bars for pattern detection", + ) + + highs = bars["high"].values + lows = bars["low"].values + closes = bars["close"].values + + # Find impulse leg (strong upward movement) + impulse_end = None + for i in range(min_impulse_bars, len(bars)): + # Calculate gain over last N bars + gain = (closes[i] - closes[i - min_impulse_bars]) / closes[i - min_impulse_bars] + if gain >= min_impulse_pct: + impulse_end = i + break + + if impulse_end is None: + return StructureDetectionResult( + setup_type=SetupType.BULL_FLAG, + is_valid=False, + reason="No impulse leg found", + ) + + impulse_start = max(0, impulse_end - min_impulse_bars) + + # Look for consolidation after impulse + consolidation_start = impulse_end + 1 + if consolidation_start >= len(bars): + return StructureDetectionResult( + setup_type=SetupType.BULL_FLAG, + is_valid=False, + reason="No consolidation phase after impulse", + ) + + # Find consolidation end (contracting range) + consolidation_end = consolidation_start + max_consolidation_idx = min( + consolidation_start + max_consolidation_bars, len(bars) - 1 + ) + + prev_range = highs[impulse_end] - lows[impulse_end] + for i in range(consolidation_start, max_consolidation_idx): + current_range = highs[i] - lows[i] + # Range should be contracting or stable + if current_range <= prev_range * 1.2: # Allow 20% expansion + consolidation_end = i + prev_range = min(prev_range, current_range) + else: + break + + consolidation_bars = consolidation_end - consolidation_start + 1 + if consolidation_bars < 2: + return StructureDetectionResult( + setup_type=SetupType.BULL_FLAG, + is_valid=False, + reason="Consolidation too short", + ) + + # Calculate quality score + impulse_pct = (closes[impulse_end] - closes[impulse_start]) / closes[impulse_start] + consolidation_depth = ( + highs[consolidation_start : consolidation_end + 1].max() + - lows[consolidation_start : consolidation_end + 1].min() + ) / closes[impulse_end] + + # Quality factors + impulse_quality = min(1.0, impulse_pct / 0.10) # Max at 10% impulse + consolidation_quality = max(0, 1.0 - (consolidation_bars / max_consolidation_bars)) + depth_quality = max( + 0, 1.0 - (consolidation_depth / 0.05) + ) # Penalty for deep pullback + + quality_score = ( + impulse_quality * 0.4 + consolidation_quality * 0.3 + depth_quality * 0.3 + ) + + # Breakout level is the high of the consolidation + breakout_level = highs[consolidation_start : consolidation_end + 1].max() + + # Stop level is the low of the consolidation or impulse + stop_level = min( + lows[impulse_start : consolidation_end + 1].min(), + lows[consolidation_start] * 0.995, + ) + + return StructureDetectionResult( + setup_type=SetupType.BULL_FLAG, + is_valid=True, + impulse_start_idx=impulse_start, + impulse_end_idx=impulse_end, + consolidation_start_idx=consolidation_start, + consolidation_end_idx=consolidation_end, + breakout_level=float(breakout_level), + stop_level=float(stop_level), + quality_score=float(quality_score), + reason=None, + ) + + +def detect_first_pullback( + bars: pd.DataFrame, + max_pullback_bars: int = 5, + max_pullback_depth_pct: float = 0.40, + impulse_search_bars: int = 120, +) -> StructureDetectionResult: + """Detect first pullback after impulse. + + Pattern: Strong move up, shallow pullback (not more than 40% of impulse), + then continuation. + + Args: + bars: OHLCV DataFrame + max_pullback_bars: Maximum bars for pullback + max_pullback_depth_pct: Maximum pullback as % of impulse + + Returns: + StructureDetectionResult with pattern details + """ + if len(bars) < 5: + return StructureDetectionResult( + setup_type=SetupType.FIRST_PULLBACK, + is_valid=False, + reason="Insufficient bars", + ) + + highs = bars["high"].values + lows = bars["low"].values + closes = bars["close"].values + + # Find impulse high in early session window (avoid late-day highs) + search_end = min(len(highs), impulse_search_bars) + if search_end < 5: + return StructureDetectionResult( + setup_type=SetupType.FIRST_PULLBACK, + is_valid=False, + reason="Insufficient early-session bars", + ) + + impulse_high_idx = int(np.argmax(highs[:search_end])) + impulse_high = highs[impulse_high_idx] + + # Calculate impulse from recent low + lookback = min(impulse_high_idx, 10) + if lookback < 3: + return StructureDetectionResult( + setup_type=SetupType.FIRST_PULLBACK, + is_valid=False, + reason="Impulse too short", + ) + + impulse_low = lows[impulse_high_idx - lookback : impulse_high_idx].min() + impulse_range = impulse_high - impulse_low + + if impulse_range / impulse_low < 0.03: # Need at least 3% impulse + return StructureDetectionResult( + setup_type=SetupType.FIRST_PULLBACK, + is_valid=False, + reason="Impulse too small", + ) + + # Look for pullback after impulse + if impulse_high_idx >= len(bars) - 1: + return StructureDetectionResult( + setup_type=SetupType.FIRST_PULLBACK, + is_valid=False, + reason="No pullback yet", + ) + + pullback_start = impulse_high_idx + 1 + pullback_end = min(pullback_start + max_pullback_bars - 1, len(bars) - 1) + + pullback_low = lows[pullback_start : pullback_end + 1].min() + pullback_depth = impulse_high - pullback_low + pullback_pct = pullback_depth / impulse_range + + if pullback_pct > max_pullback_depth_pct: + return StructureDetectionResult( + setup_type=SetupType.FIRST_PULLBACK, + is_valid=False, + reason=f"Pullback too deep: {pullback_pct:.1%}", + ) + + # Quality scoring + depth_quality = max(0, 1.0 - (pullback_pct / max_pullback_depth_pct)) + pullback_bars = pullback_end - pullback_start + 1 + speed_quality = max(0, 1.0 - (pullback_bars / max_pullback_bars)) + + quality_score = depth_quality * 0.6 + speed_quality * 0.4 + + # Entry on reclaim of pullback high + breakout_level = impulse_high + stop_level = pullback_low * 0.995 + + return StructureDetectionResult( + setup_type=SetupType.FIRST_PULLBACK, + is_valid=True, + impulse_start_idx=impulse_high_idx - lookback, + impulse_end_idx=impulse_high_idx, + consolidation_start_idx=pullback_start, + consolidation_end_idx=pullback_end, + breakout_level=float(breakout_level), + stop_level=float(stop_level), + quality_score=float(quality_score), + reason=None, + ) + + +def detect_gap_and_go( + bars: pd.DataFrame, + premarket_high: float | None = None, + min_gap_pct: float = 0.03, +) -> StructureDetectionResult: + """Detect gap-and-go pattern. + + Pattern: Stock gaps up, opens strong, and continues higher + breaking premarket or opening range highs. + + Args: + bars: OHLCV DataFrame (including premarket) + premarket_high: High of premarket session + min_gap_pct: Minimum gap percentage + + Returns: + StructureDetectionResult with pattern details + """ + if len(bars) < 5: + return StructureDetectionResult( + setup_type=SetupType.GAP_AND_GO, is_valid=False, reason="Insufficient bars" + ) + + opens = bars["open"].values + highs = bars["high"].values + closes = bars["close"].values + + # Check for gap (first bar open vs some reference) + # For simplicity, compare first bar open to recent lows + reference_price = bars["low"].iloc[: min(5, len(bars))].min() + gap_pct = (opens[0] - reference_price) / reference_price + + if gap_pct < min_gap_pct: + return StructureDetectionResult( + setup_type=SetupType.GAP_AND_GO, + is_valid=False, + reason=f"Gap too small: {gap_pct:.1%}", + ) + + # Determine breakout level + if premarket_high: + breakout_level = premarket_high + else: + # Use first few bars high + breakout_level = highs[: min(3, len(bars))].max() + + # Check if we've broken out + current_price = closes[-1] + if current_price < breakout_level * 0.97: + return StructureDetectionResult( + setup_type=SetupType.GAP_AND_GO, + is_valid=False, + reason="Waiting for breakout confirmation", + ) + + # Quality based on gap size and trend strength + gap_quality = min(1.0, gap_pct / 0.15) # Max at 15% gap + + # Check if price is holding above VWAP or continuing up + recent_trend = (closes[-1] - closes[min(len(closes) - 1, 3)]) / closes[ + min(len(closes) - 1, 3) + ] + trend_quality = 1.0 if recent_trend > 0 else 0.5 + + quality_score = gap_quality * 0.6 + trend_quality * 0.4 + + # Stop below recent pullback or opening range + stop_level = bars["low"].iloc[: min(10, len(bars))].min() * 0.995 + + return StructureDetectionResult( + setup_type=SetupType.GAP_AND_GO, + is_valid=True, + breakout_level=float(breakout_level), + stop_level=float(stop_level), + quality_score=float(quality_score), + reason=None, + ) + + +def detect_hod_break( + bars: pd.DataFrame, + session_high: float | None = None, +) -> StructureDetectionResult: + """Detect high-of-day breakout. + + Pattern: Stock consolidates under the HOD, then breaks through + with volume confirmation. + + Args: + bars: OHLCV DataFrame + session_high: Current session high (if known) + + Returns: + StructureDetectionResult with pattern details + """ + if len(bars) < 10: + return StructureDetectionResult( + setup_type=SetupType.HOD_BREAK, is_valid=False, reason="Insufficient bars" + ) + + highs = bars["high"].values + closes = bars["close"].values + + # Determine HOD + if session_high: + hod = session_high + else: + hod = highs.max() + + # Find consolidation under HOD + hod_idx = np.where(highs == hod)[0][0] if hod in highs else -1 + + if hod_idx < 0 or hod_idx < len(bars) - 5: + return StructureDetectionResult( + setup_type=SetupType.HOD_BREAK, + is_valid=False, + reason="No recent HOD to break", + ) + + # Look for consolidation before HOD + consolidation_start = max(0, hod_idx - 10) + consolidation_zone = highs[consolidation_start:hod_idx] + + # Quality: More touches of the level = better setup + touches = np.sum(consolidation_zone > hod * 0.995) + consolidation_quality = min(1.0, touches / 3) # Max at 3 touches + + # Check if we broke and holding + current_price = closes[-1] + if current_price < hod * 0.99: + return StructureDetectionResult( + setup_type=SetupType.HOD_BREAK, + is_valid=False, + reason="Waiting for HOD break", + ) + + quality_score = consolidation_quality * 0.7 + 0.3 # Base score for breaking + + stop_level = bars["low"].iloc[max(0, len(bars) - 5) :].min() * 0.995 + + return StructureDetectionResult( + setup_type=SetupType.HOD_BREAK, + is_valid=True, + breakout_level=float(hod), + stop_level=float(stop_level), + quality_score=float(quality_score), + reason=None, + ) + + +def detect_continuation_fallback( + bars: pd.DataFrame, + min_bars: int = 40, + impulse_window: int = 25, + min_impulse_pct: float = 0.015, + max_pullback_pct: float = 0.65, +) -> StructureDetectionResult: + if len(bars) < min_bars: + return StructureDetectionResult( + setup_type=SetupType.CONTINUATION_FALLBACK, + is_valid=False, + reason="Insufficient bars for fallback detection", + ) + + highs = bars["high"].values + lows = bars["low"].values + closes = bars["close"].values + + search_end = min(len(closes), max(min_bars, 120)) + if search_end < impulse_window + 5: + return StructureDetectionResult( + setup_type=SetupType.CONTINUATION_FALLBACK, + is_valid=False, + reason="Insufficient early-session window", + ) + + best_start = 0 + best_end = impulse_window + best_gain = -1.0 + for i in range(0, search_end - impulse_window): + j = i + impulse_window + start = closes[i] + end = closes[j] + if start <= 0: + continue + gain = (end - start) / start + if gain > best_gain: + best_gain = gain + best_start = i + best_end = j + + if best_gain < min_impulse_pct: + return StructureDetectionResult( + setup_type=SetupType.CONTINUATION_FALLBACK, + is_valid=False, + reason="No sufficient impulse for fallback", + ) + + impulse_high = highs[best_start : best_end + 1].max() + impulse_low = lows[best_start : best_end + 1].min() + impulse_range = impulse_high - impulse_low + if impulse_range <= 0: + return StructureDetectionResult( + setup_type=SetupType.CONTINUATION_FALLBACK, + is_valid=False, + reason="Invalid impulse range", + ) + + pullback_start = min(best_end + 1, len(bars) - 1) + pullback_end = min(pullback_start + 20, len(bars) - 1) + if pullback_end <= pullback_start: + return StructureDetectionResult( + setup_type=SetupType.CONTINUATION_FALLBACK, + is_valid=False, + reason="No pullback window", + ) + + pullback_low = lows[pullback_start : pullback_end + 1].min() + pullback_depth = max(0.0, impulse_high - pullback_low) + pullback_pct = pullback_depth / impulse_range + + if pullback_pct > max_pullback_pct: + return StructureDetectionResult( + setup_type=SetupType.CONTINUATION_FALLBACK, + is_valid=False, + reason=f"Fallback pullback too deep: {pullback_pct:.1%}", + ) + + breakout_level = float(impulse_high) + stop_level = float(pullback_low * 0.995) + if stop_level >= breakout_level: + return StructureDetectionResult( + setup_type=SetupType.CONTINUATION_FALLBACK, + is_valid=False, + reason="Invalid fallback risk geometry", + ) + + impulse_quality = min(1.0, best_gain / 0.05) + pullback_quality = max(0.0, 1.0 - (pullback_pct / max_pullback_pct)) + current_price = closes[-1] + proximity = max(0.0, 1.0 - abs((breakout_level - current_price) / breakout_level)) + quality_score = float( + impulse_quality * 0.45 + pullback_quality * 0.35 + proximity * 0.20 + ) + + return StructureDetectionResult( + setup_type=SetupType.CONTINUATION_FALLBACK, + is_valid=True, + impulse_start_idx=int(best_start), + impulse_end_idx=int(best_end), + consolidation_start_idx=int(pullback_start), + consolidation_end_idx=int(pullback_end), + breakout_level=breakout_level, + stop_level=stop_level, + quality_score=quality_score, + reason=None, + ) + + +def classify_setup( + bars: pd.DataFrame, + premarket_high: float | None = None, + session_high: float | None = None, + min_quality_score: float = 0.2, +) -> StructureDetectionResult: + """Classify which setup type is present. + + Tests all setup types and returns the best valid one. + + Args: + bars: OHLCV DataFrame + premarket_high: High of premarket session + session_high: Current session high + min_quality_score: Minimum quality to accept setup + + Returns: + StructureDetectionResult for best valid setup + """ + detectors = [ + detect_bull_flag(bars, min_impulse_pct=0.015), + detect_first_pullback(bars, max_pullback_depth_pct=0.6), + detect_gap_and_go(bars, premarket_high, min_gap_pct=0.02), + detect_hod_break(bars, session_high), + detect_continuation_fallback(bars), + ] + + # Filter valid setups meeting quality threshold + valid_setups = [ + s for s in detectors if s.is_valid and s.quality_score >= min_quality_score + ] + + if not valid_setups: + return StructureDetectionResult( + setup_type=SetupType.NONE, is_valid=False, reason="No valid setup detected" + ) + + # Return highest quality setup + return max(valid_setups, key=lambda s: s.quality_score) diff --git a/strategy/evaluation/tracking.py b/strategy/evaluation/tracking.py new file mode 100644 index 0000000..cc7929c --- /dev/null +++ b/strategy/evaluation/tracking.py @@ -0,0 +1,32 @@ +"""Lightweight in-memory tracking of evaluation history per symbol.""" + +from __future__ import annotations + +from collections import defaultdict + + +class EvaluationTracker: + """Tracks recent evaluation results per symbol for debounce/analysis.""" + + def __init__(self, max_history: int = 50): + self.max_history = max_history + self._history: dict[str, list[dict]] = defaultdict(list) + + def record(self, symbol: str, result: dict) -> None: + history = self._history[symbol] + history.append(result) + if len(history) > self.max_history: + del history[: len(history) - self.max_history] + + def last(self, symbol: str) -> dict | None: + history = self._history.get(symbol) + return history[-1] if history else None + + def consecutive_status(self, symbol: str, status: str) -> int: + count = 0 + for result in reversed(self._history.get(symbol, [])): + if result.get("status") == status: + count += 1 + else: + break + return count diff --git a/strategy/evaluation/volume_metrics.py b/strategy/evaluation/volume_metrics.py new file mode 100644 index 0000000..834f8dd --- /dev/null +++ b/strategy/evaluation/volume_metrics.py @@ -0,0 +1,67 @@ +"""Volume metrics: relative volume, time-of-day RVOL, float rotation.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import pandas as pd + + +@dataclass +class VolumeMetrics: + """Enhanced volume metrics for a symbol.""" + + session_volume: int = 0 + relative_volume: float = 0.0 + time_of_day_rvol: float = 0.0 + float_rotation: float = 0.0 + avg_bar_volume: float = 0.0 + last_bar_volume: int = 0 + + +def calculate_time_of_day_rvol( + session_volume: float, + avg_daily_volume: float, + minutes_elapsed: float, + session_minutes: float = 390.0, +) -> float: + """RVOL adjusted for how far through the session we are.""" + if avg_daily_volume <= 0 or minutes_elapsed <= 0: + return 0.0 + fraction = min(1.0, max(minutes_elapsed / session_minutes, 1e-9)) + expected = avg_daily_volume * fraction + return session_volume / expected if expected > 0 else 0.0 + + +def calculate_float_rotation(session_volume: float, float_shares: float) -> float: + """How many times the float has traded this session.""" + if float_shares <= 0: + return 0.0 + return session_volume / float_shares + + +def calculate_enhanced_volume_metrics( + bars: pd.DataFrame, + avg_daily_volume: float | None = None, + float_shares: float | None = None, +) -> VolumeMetrics: + """Compute volume metrics from session minute bars.""" + metrics = VolumeMetrics() + if bars.empty: + return metrics + + vol = bars["volume"].astype(float) + metrics.session_volume = int(vol.sum()) + metrics.avg_bar_volume = float(vol.mean()) + metrics.last_bar_volume = int(vol.iloc[-1]) + + if avg_daily_volume and avg_daily_volume > 0: + metrics.relative_volume = metrics.session_volume / avg_daily_volume + metrics.time_of_day_rvol = calculate_time_of_day_rvol( + metrics.session_volume, avg_daily_volume, minutes_elapsed=len(bars) + ) + if float_shares and float_shares > 0: + metrics.float_rotation = calculate_float_rotation( + metrics.session_volume, float_shares + ) + return metrics diff --git a/strategy/models.py b/strategy/models.py new file mode 100644 index 0000000..dbfb959 --- /dev/null +++ b/strategy/models.py @@ -0,0 +1,73 @@ +"""Strategy data models. + +Pydantic models for configuration and results. +""" + +from pydantic import BaseModel, Field + + +class SetupCriteria(BaseModel): + """Configuration for setup evaluation criteria.""" + + gap_pct_min: float = Field(default=0.05, description="Minimum gap percentage") + relative_volume_min: float = Field( + default=2.5, description="Minimum relative volume" + ) + impulse_size_min: float = Field(default=0.05, description="Minimum impulse size") + max_pullback_depth_pct: float = Field( + default=0.40, description="Max pullback depth percent" + ) + pullback_volume_ratio_max: float = Field( + default=0.6, description="Max pullback volume ratio" + ) + breakout_volume_ratio_min: float = Field( + default=1.5, description="Min breakout volume ratio" + ) + min_quality_score: float = Field( + default=0.60, description="Minimum data quality score (default: 0.60)" + ) + + +class CriteriaWeights(BaseModel): + """Weights for scoring criteria.""" + + sufficient_data: int = 10 + gap: int = 15 + relative_volume: int = 15 + impulse: int = 15 + pullback: int = 10 + pullback_volume: int = 10 + vwap: int = 10 + candle_quality: int = 5 + breakout: int = 10 + + +class CriteriaResult(BaseModel): + """Result of criteria evaluation.""" + + name: str + passed: bool + reason: str | None = None + + +class SetupEvaluationResult(BaseModel): + """Result of setup evaluation.""" + + status: str = Field(description="Status: ready, blocked, or late") + reason: str | None = Field( + default=None, description="Blocking reason if status is blocked/late" + ) + evaluated_at: str | None = Field(default=None, description="Evaluation timestamp") + price: float = Field(description="Current price") + gap_pct: float = Field(description="Gap percentage") + relative_volume: float = Field(description="Relative volume") + criteria_passed: int = Field(description="Number of criteria passed") + criteria_total: int = Field(description="Total number of criteria") + success_score_pct: float = Field(description="Success score percentage") + criteria_names_passed: list[str] = Field( + default_factory=list, description="Names of passed criteria" + ) + criteria_names_failed: list[str] = Field( + default_factory=list, description="Names of failed criteria" + ) + setups: list[dict] = Field(default_factory=list, description="Detected setups") diff --git a/strategy/risk/__init__.py b/strategy/risk/__init__.py new file mode 100644 index 0000000..326a784 --- /dev/null +++ b/strategy/risk/__init__.py @@ -0,0 +1,26 @@ +"""Risk and cutoff management module. + +Pure logic for position sizing, entry cutoffs, and risk controls. +No broker or UI dependencies. +""" + +from .entry_cuts import ( + EntryCutoffConfig, + check_entry_cutoff, + EntryCutoffResult, +) + +from .position_sizing import ( + PositionSizingConfig, + calculate_position_size, + PositionSizeResult, +) + +__all__ = [ + "EntryCutoffConfig", + "check_entry_cutoff", + "EntryCutoffResult", + "PositionSizingConfig", + "calculate_position_size", + "PositionSizeResult", +] diff --git a/strategy/risk/entry_cuts.py b/strategy/risk/entry_cuts.py new file mode 100644 index 0000000..9efe86d --- /dev/null +++ b/strategy/risk/entry_cuts.py @@ -0,0 +1,48 @@ +"""Entry cutoff logic.""" + +from datetime import time +from pydantic import BaseModel, Field + + +class EntryCutoffConfig(BaseModel): + """Configuration for entry cutoff rules.""" + + no_new_entries_after: time = Field( + default=time(10, 30), description="Cutoff time for new entries" + ) + minutes_past_cutoff_max: int = Field( + default=5, description="Max minutes past cutoff to allow entry" + ) + + +class EntryCutoffResult(BaseModel): + """Result of entry cutoff check.""" + + passed: bool = Field(description="Whether entry is allowed") + reason: str | None = Field(default=None, description="Reason if entry is blocked") + + +def check_entry_cutoff( + trigger_time: time, + cutoff_config: EntryCutoffConfig = EntryCutoffConfig(), +) -> EntryCutoffResult: + """Check if entry is allowed based on cutoff time.""" + cutoff_time = cutoff_config.no_new_entries_after + + if trigger_time < cutoff_time: + return EntryCutoffResult(passed=True, reason=None) + + minutes_past_cutoff = (trigger_time.hour * 60 + trigger_time.minute) - ( + cutoff_time.hour * 60 + cutoff_time.minute + ) + + if minutes_past_cutoff <= cutoff_config.minutes_past_cutoff_max: + return EntryCutoffResult( + passed=True, + reason=f"{minutes_past_cutoff} min past cutoff (max {cutoff_config.minutes_past_cutoff_max})", + ) + + return EntryCutoffResult( + passed=False, + reason=f"{minutes_past_cutoff} min past cutoff (exceeds max {cutoff_config.minutes_past_cutoff_max})", + ) diff --git a/strategy/risk/position_sizing.py b/strategy/risk/position_sizing.py new file mode 100644 index 0000000..150b514 --- /dev/null +++ b/strategy/risk/position_sizing.py @@ -0,0 +1,51 @@ +"""Position sizing logic.""" + +from pydantic import BaseModel, Field + + +class PositionSizingConfig(BaseModel): + """Configuration for position sizing.""" + + risk_per_trade_pct: float = Field( + default=0.01, description="Risk per trade as percent of equity" + ) + default_equity: float = Field( + default=100000.0, description="Default equity for sizing" + ) + + +class PositionSizeResult(BaseModel): + """Result of position sizing calculation.""" + + position_size: int = Field(description="Position size in shares") + dollar_amount: float = Field(description="Dollar amount of position") + risk_amount: float = Field(description="Dollar risk amount") + + +def calculate_position_size( + entry_price: float, + stop_loss_price: float, + equity: float | None = None, + config: PositionSizingConfig = PositionSizingConfig(), +) -> PositionSizeResult: + """Calculate position size based on risk per trade.""" + account_equity = equity or config.default_equity + risk_amount = account_equity * config.risk_per_trade_pct + + risk_per_share = abs(entry_price - stop_loss_price) + + if risk_per_share <= 0: + return PositionSizeResult( + position_size=0, + dollar_amount=0.0, + risk_amount=risk_amount, + ) + + position_size = int(risk_amount / risk_per_share) + dollar_amount = position_size * entry_price + + return PositionSizeResult( + position_size=position_size, + dollar_amount=dollar_amount, + risk_amount=risk_amount, + ) diff --git a/strategy/universe.py b/strategy/universe.py new file mode 100644 index 0000000..7466e6d --- /dev/null +++ b/strategy/universe.py @@ -0,0 +1,28 @@ +"""Tradeable universe filtering (price band, volume, share class).""" + +from __future__ import annotations + +from config import UniverseConfig + + +def symbol_in_universe( + price: float, + avg_volume_20d: float | None = None, + is_etf: bool = False, + is_otc: bool = False, + config: UniverseConfig | None = None, +) -> tuple[bool, str | None]: + """Check whether a symbol qualifies for the trading universe.""" + config = config or UniverseConfig() + if price < config.price_min or price > config.price_max: + return False, f"price {price:.2f} outside [{config.price_min}, {config.price_max}]" + if ( + avg_volume_20d is not None + and avg_volume_20d < config.min_avg_volume_20d + ): + return False, f"avg volume {avg_volume_20d:,.0f} < {config.min_avg_volume_20d:,}" + if config.exclude_etf and is_etf: + return False, "ETF excluded" + if config.exclude_otc and is_otc: + return False, "OTC excluded" + return True, None diff --git a/telemetry/__init__.py b/telemetry/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/telemetry/events.py b/telemetry/events.py new file mode 100644 index 0000000..d2149ba --- /dev/null +++ b/telemetry/events.py @@ -0,0 +1,102 @@ +"""Telemetry event emission and querying for runtime observability.""" + +from __future__ import annotations + +import json +import logging +import uuid +from enum import Enum + +logger = logging.getLogger(__name__) + + +class TelemetryEventType(str, Enum): + SOURCE_DEGRADED = "SOURCE_DEGRADED" + SOURCE_RECOVERED = "SOURCE_RECOVERED" + SOURCE_COOLDOWN = "SOURCE_COOLDOWN" + WATCHLIST_ADD = "WATCHLIST_ADD" + WATCHLIST_DROP = "WATCHLIST_DROP" + CANDIDATE_STATE_CHANGE = "CANDIDATE_STATE_CHANGE" + FETCH_ATTEMPT = "FETCH_ATTEMPT" + ENRICHMENT_QUEUED = "ENRICHMENT_QUEUED" + ENRICHMENT_DONE = "ENRICHMENT_DONE" + ENRICHMENT_FAILED = "ENRICHMENT_FAILED" + + +def emit_event( + con, + event_type: TelemetryEventType | str, + session_id: str | None = None, + symbol: str | None = None, + source: str | None = None, + from_state: str | None = None, + to_state: str | None = None, + reason: str | None = None, + metadata: dict | None = None, +) -> None: + event_id = str(uuid.uuid4()) + event_type_val = ( + event_type.value + if isinstance(event_type, TelemetryEventType) + else str(event_type) + ) + metadata_json = json.dumps(metadata) if metadata is not None else None + try: + con.execute( + """ + INSERT INTO telemetry_events + (id, event_type, session_id, symbol, source, from_state, to_state, reason, metadata_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + event_id, + event_type_val, + session_id, + symbol, + source, + from_state, + to_state, + reason, + metadata_json, + ], + ) + except Exception: + logger.exception( + "telemetry emit_event failed — event_type=%s symbol=%s", + event_type_val, + symbol, + ) + + +def query_events( + con, + session_id: str | None = None, + symbol: str | None = None, + source: str | None = None, + limit: int = 100, +) -> list[dict]: + clauses: list[str] = [] + params: list = [] + + if session_id is not None: + clauses.append("session_id = ?") + params.append(session_id) + if symbol is not None: + clauses.append("symbol = ?") + params.append(symbol) + if source is not None: + clauses.append("source = ?") + params.append(source) + + where = ("WHERE " + " AND ".join(clauses)) if clauses else "" + sql = f"SELECT * FROM telemetry_events {where} ORDER BY created_at DESC LIMIT ?" + params.append(limit) + + rows = con.execute(sql, params).fetchall() + cols = [ + desc[0] + for desc in con.execute( + f"SELECT * FROM telemetry_events {where} LIMIT 0", params[:-1] + ).description + ] + return [dict(zip(cols, row)) for row in rows] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py new file mode 100644 index 0000000..44582fb --- /dev/null +++ b/tests/integration/test_end_to_end.py @@ -0,0 +1,180 @@ +"""End-to-end integration: ingestion -> watcher -> approval -> execution. + +Exercises the exact pipeline run_live_paper.py drives, with a fake Alpaca +client, asserting the event-sourced projections the dashboard reads. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from alpaca_paper.execution import AlpacaPaperExecutor +from alpaca_paper.sync import AlpacaPaperSync +from research.ingestion.market_data import ingest_daily_history, ingest_live_minute_bars +from research.ingestion.watcher_task import ResearchWatchlistProvider +from runtime.watcher import Watcher, WatcherConfig +from storage.db import get_connection +from storage.event_schema import EventMode +from storage.event_store import EventStore +from storage.projections import ( + query_account_summary_snapshot, + query_approval_queue, + query_order_lifecycle_snapshot, + query_ready_signals_snapshot, + query_watch_states_snapshot, +) +from trading_execution import ExecutionSettings, TradingExecutionService +from trading_mode import TradingModeSettings + +SESSION = datetime(2026, 6, 11, tzinfo=timezone.utc) +SESSION_DATE = SESSION.date() + + +def iso(dt): + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def bull_flag_payload(start_utc): + """Same shape as tests.synthetic.bull_flag_bars, as Alpaca API dicts.""" + specs = [ + (12.50, 12.60, 12.45, 12.55, 60_000), + (12.55, 12.65, 12.50, 12.60, 55_000), + (12.60, 12.70, 12.55, 12.65, 50_000), + (12.65, 12.95, 12.62, 12.92, 120_000), + (12.92, 13.25, 12.90, 13.20, 150_000), + (13.20, 13.55, 13.18, 13.50, 180_000), + (13.50, 13.85, 13.48, 13.80, 200_000), + (13.80, 13.82, 13.62, 13.66, 70_000), + (13.66, 13.70, 13.55, 13.60, 55_000), + (13.60, 13.66, 13.52, 13.58, 45_000), + (13.58, 13.95, 13.56, 13.92, 220_000), + (13.92, 14.05, 13.88, 14.00, 190_000), + ] + return [ + {"t": iso(start_utc + timedelta(minutes=i)), "o": o, "h": h, "l": l, + "c": c, "v": v} + for i, (o, h, l, c, v) in enumerate(specs) + ] + + +class FakeAlpaca: + """Market data + trading endpoints used across the pipeline.""" + + def __init__(self): + # 14:00 UTC = 10:00 ET regular hours + self.minute = {"GOOD": bull_flag_payload(SESSION.replace(hour=14))} + self.daily = { + "GOOD": [ + {"t": iso(SESSION - timedelta(days=2)), "o": 9.8, "h": 10.2, + "l": 9.6, "c": 9.9, "v": 480_000}, + {"t": iso(SESSION - timedelta(days=1)), "o": 9.9, "h": 10.3, + "l": 9.8, "c": 10.0, "v": 520_000}, + ] + } + self.submitted = [] + self.fills = [] + + # market data + def get_minute_bars(self, symbols, start_iso, end_iso=None, feed=None, limit=10_000): + return {s: self.minute.get(s, []) for s in symbols} + + def get_daily_bars(self, symbols, start_iso, end_iso=None): + return {s: self.daily.get(s, []) for s in symbols} + + # trading + def submit_order(self, **kw): + self.submitted.append(kw) + return {"id": f"broker-{len(self.submitted)}", "status": "accepted"} + + def get_account(self): + return {"account_number": "PA-E2E", "equity": "100000", + "cash": "50000", "buying_power": "200000"} + + def get_positions(self): + return [ + {"symbol": o["symbol"], "qty": str(o["qty"]), + "avg_entry_price": "13.9", "current_price": "14.1", + "unrealized_pl": "2.0"} + for o in self.submitted if o["side"] == "buy" + ] + + def get_orders(self, status="all", limit=100): + return [{"id": f"broker-{i+1}", "symbol": o["symbol"], + "status": "filled", "side": o["side"], "qty": str(o["qty"])} + for i, o in enumerate(self.submitted)] + + +@pytest.fixture +def pipeline(): + client = FakeAlpaca() + research_con = get_connection(":memory:") + store = EventStore(":memory:") + watcher = Watcher( + store, + ResearchWatchlistProvider(research_con, price_min=1, price_max=20), + WatcherConfig(session_id="e2e", mode=EventMode.PAPER, min_quality=0.0), + ) + executor = AlpacaPaperExecutor(store, client=client, session_id="e2e") + execution = TradingExecutionService( + store, executor=executor, + settings=ExecutionSettings(auto_approve=False, max_orders_per_tick=2), + trading_mode=TradingModeSettings(execution_mode="alpaca_paper"), + session_id="e2e", + ) + sync = AlpacaPaperSync(store, client=client, session_id="e2e") + yield client, research_con, store, watcher, execution, sync + store.close() + research_con.close() + + +def test_full_paper_trading_pipeline(pipeline): + client, research_con, store, watcher, execution, sync = pipeline + + # 1. ingest: daily baseline + live minute bars + daily = ingest_daily_history(research_con, client, ["GOOD"]) + assert daily.daily_rows == 2 + minute = ingest_live_minute_bars(research_con, client, ["GOOD"]) + assert minute.minute_rows == 12 + + # 2. watcher tick discovers + signals ready + tick = watcher.tick(SESSION_DATE) + assert tick.ready == ["GOOD"] + signals = query_ready_signals_snapshot(store, session_id="e2e") + assert signals[0]["symbol"] == "GOOD" + assert signals[0]["entry_price"] > signals[0]["stop_loss_price"] > 0 + states = {w["symbol"]: w["state"] for w in query_watch_states_snapshot(store)} + assert states["GOOD"] == "ready" + + # 3. broker sync populates account snapshots + sync.sync_all() + assert query_account_summary_snapshot(store, broker_name="alpaca_paper") + + # 4. execution tick: signal -> pending approval + out = execution.tick() + assert len(out["approvals_requested"]) == 1 + queue = query_approval_queue(store) + assert queue[0]["symbol"] == "GOOD" + assert queue[0]["approval_mode"] == "manual" + + # 5. manual approval submits the bracket order to the broker + result = execution.approve_order(queue[0]["order_id"], approved_by="test") + assert result["ok"] + assert client.submitted[0]["symbol"] == "GOOD" + assert client.submitted[0]["side"] == "buy" + + # 6. projections converge: order submitted, queue drained + assert query_approval_queue(store) == [] + lifecycle = {o["order_id"]: o for o in query_order_lifecycle_snapshot(store)} + entry = lifecycle[queue[0]["order_id"]] + assert entry["symbol"] == "GOOD" + assert entry["side"] == "buy" + + # 7. re-sync shows the open position; exit closes it + sync.sync_all() + exit_result = execution.submit_exit_order("GOOD") + assert exit_result["ok"] + assert client.submitted[-1]["side"] == "sell" + + # 8. watcher debounce: second tick emits no duplicate signal + watcher.tick(SESSION_DATE) + assert len(store.query_events(event_type="signal_ready", symbol="GOOD")) == 1 diff --git a/tests/synthetic.py b/tests/synthetic.py new file mode 100644 index 0000000..6951b62 --- /dev/null +++ b/tests/synthetic.py @@ -0,0 +1,90 @@ +"""Synthetic minute-bar builders shared across the test suite.""" + +from __future__ import annotations + +from datetime import datetime, timedelta + +import pandas as pd + +SESSION_DATE = datetime(2026, 6, 11).date() +# Bars are stored UTC-naive throughout the pipeline (ingestion converts to UTC +# then drops tzinfo). 13:30 UTC == 09:30 America/New_York in June (EDT), so this +# fixture exercises the same timezone path as live data: the watcher converts +# the bar timestamp UTC -> Eastern before the entry-cutoff check. +SESSION_OPEN = datetime(2026, 6, 11, 13, 30) # 09:30 ET expressed in UTC + + +def make_bars(specs: list[tuple[float, float, float, float, int]], + start: datetime | None = None, + symbol: str = "TEST") -> pd.DataFrame: + """Build a minute-bar frame from (open, high, low, close, volume) tuples.""" + start = start or SESSION_OPEN + rows = [] + cum_pv = 0.0 + cum_v = 0 + for i, (o, h, l, c, v) in enumerate(specs): + ts = start + timedelta(minutes=i) + typical = (h + l + c) / 3.0 + cum_pv += typical * v + cum_v += v + rows.append( + { + "symbol": symbol, + "timestamp": ts, + "session_date": SESSION_DATE, + "is_premarket": False, + "is_regular_hours": True, + "open": o, + "high": h, + "low": l, + "close": c, + "volume": v, + "vwap": cum_pv / cum_v if cum_v else c, + "quality_score": 1.0, + } + ) + return pd.DataFrame(rows) + + +def bull_flag_bars(symbol: str = "TEST") -> pd.DataFrame: + """Gap-up + impulse + light pullback + breakout: should evaluate ready. + + Pair with previous_close=10.0 and avg_daily_volume=500_000 so the gap + (~25%) and relative volume criteria pass. + """ + specs = [ + # base + (12.50, 12.60, 12.45, 12.55, 60_000), + (12.55, 12.65, 12.50, 12.60, 55_000), + (12.60, 12.70, 12.55, 12.65, 50_000), + # impulse leg on expanding volume + (12.65, 12.95, 12.62, 12.92, 120_000), + (12.92, 13.25, 12.90, 13.20, 150_000), + (13.20, 13.55, 13.18, 13.50, 180_000), + (13.50, 13.85, 13.48, 13.80, 200_000), + # shallow pullback on lighter volume (holds above vwap) + (13.80, 13.82, 13.62, 13.66, 70_000), + (13.66, 13.70, 13.55, 13.60, 55_000), + (13.60, 13.66, 13.52, 13.58, 45_000), + # breakout bar through pullback high + (13.58, 13.95, 13.56, 13.92, 220_000), + (13.92, 14.05, 13.88, 14.00, 190_000), + ] + return make_bars(specs, symbol=symbol) + + +def fading_bars(symbol: str = "FADE") -> pd.DataFrame: + """Steady downtrend below vwap: should evaluate blocked.""" + specs = [] + price = 10.0 + for i in range(15): + o = price + c = price - 0.08 + specs.append((o, o + 0.02, c - 0.03, c, 40_000)) + price = c + return make_bars(specs, symbol=symbol) + + +def tiny_bars(symbol: str = "TINY") -> pd.DataFrame: + """Too few bars: should block with insufficient_data.""" + return make_bars([(5.0, 5.1, 4.9, 5.05, 10_000)] * 3, symbol=symbol) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/alpaca_paper/__init__.py b/tests/unit/alpaca_paper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/alpaca_paper/test_execution_and_sync.py b/tests/unit/alpaca_paper/test_execution_and_sync.py new file mode 100644 index 0000000..0c17b37 --- /dev/null +++ b/tests/unit/alpaca_paper/test_execution_and_sync.py @@ -0,0 +1,124 @@ +"""Alpaca paper executor + account sync tests (mocked client).""" + +import pytest + +from alpaca_paper.client import AlpacaApiError +from alpaca_paper.execution import AlpacaPaperExecutor, ExecutionRequest +from alpaca_paper.sync import AlpacaPaperSync +from storage.event_store import EventStore +from storage.projections import ( + query_account_orders_snapshot, + query_account_positions_snapshot, + query_account_summary_snapshot, +) + + +class FakeClient: + def __init__(self): + self.submitted = [] + self.fail_with = None + + def submit_order(self, **kw): + if self.fail_with: + raise self.fail_with + self.submitted.append(kw) + return {"id": "broker-123", "status": "accepted", "filled_avg_price": None} + + def get_account(self): + return { + "account_number": "PA-TEST", + "equity": "100250.50", + "cash": "60000", + "buying_power": "200501", + } + + def get_positions(self): + return [ + { + "symbol": "GOOD", + "qty": "10", + "avg_entry_price": "13.90", + "current_price": "14.10", + "unrealized_pl": "2.0", + } + ] + + def get_orders(self, status="all", limit=100): + return [{"id": "broker-123", "symbol": "GOOD", "status": "filled"}] + + +@pytest.fixture +def store(): + s = EventStore(":memory:") + yield s + s.close() + + +def test_executor_submits_and_emits(store): + client = FakeClient() + executor = AlpacaPaperExecutor(store, client=client, session_id="t") + request = ExecutionRequest( + symbol="GOOD", side="buy", quantity=10, + entry_price=14.0, stop_loss_price=13.45, + ) + result = executor.execute(request) + assert result.ok + assert result.broker_order_id == "broker-123" + assert client.submitted[0]["symbol"] == "GOOD" + submitted = store.query_events(event_type="order_submitted") + assert len(submitted) == 1 + + +def test_executor_rejects_zero_quantity(store): + executor = AlpacaPaperExecutor(store, client=FakeClient(), session_id="t") + result = executor.execute(ExecutionRequest(symbol="GOOD", side="buy", quantity=0)) + assert not result.ok + assert "quantity" in (result.error or "") + + +def test_executor_api_failure_emits_cancelled(store): + client = FakeClient() + client.fail_with = AlpacaApiError(403, "forbidden") + executor = AlpacaPaperExecutor(store, client=client, session_id="t") + result = executor.execute(ExecutionRequest(symbol="GOOD", side="buy", quantity=5)) + assert not result.ok + assert store.query_events(event_type="order_cancelled") + + +def test_execution_request_payload_round_trip(): + request = ExecutionRequest( + symbol="GOOD", side="buy", quantity=7, + entry_price=10.0, stop_loss_price=9.5, take_profit_price=11.0, + ) + clone = ExecutionRequest.from_payload(request.to_payload()) + assert clone.order_id == request.order_id + assert clone.symbol == "GOOD" + assert clone.quantity == 7 + assert clone.stop_loss_price == 9.5 + + +def test_sync_all_populates_snapshots(store): + sync = AlpacaPaperSync(store, client=FakeClient(), session_id="t") + sync.sync_all() + accounts = query_account_summary_snapshot(store, broker_name="alpaca_paper") + assert accounts and accounts[0]["total_equity"] == pytest.approx(100250.50) + positions = query_account_positions_snapshot(store, broker_name="alpaca_paper") + assert positions[-1]["positions"][0]["symbol"] == "GOOD" + orders = query_account_orders_snapshot(store, broker_name="alpaca_paper") + assert orders[-1]["orders"][0]["broker_order_id"] == "broker-123" + + +def test_sync_survives_client_failure(store): + class Down(FakeClient): + def get_account(self): + raise AlpacaApiError(500, "down") + + def get_positions(self): + raise AlpacaApiError(500, "down") + + def get_orders(self, status="all", limit=100): + raise AlpacaApiError(500, "down") + + sync = AlpacaPaperSync(store, client=Down(), session_id="t") + sync.sync_all() # must not raise + assert query_account_summary_snapshot(store) == [] diff --git a/tests/unit/api/__init__.py b/tests/unit/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/api/test_dashboard_api.py b/tests/unit/api/test_dashboard_api.py new file mode 100644 index 0000000..0d6f23d --- /dev/null +++ b/tests/unit/api/test_dashboard_api.py @@ -0,0 +1,258 @@ +"""Dashboard API endpoint tests over a real local HTTP server.""" + +import json +import urllib.request +from datetime import datetime + +import pytest + +from api.main import DashboardState, serve_in_background +from storage.event_schema import EventMode, SignalReadyEvent +from storage.event_store import EventStore +from tests.unit.test_trading_execution import FakeExecutor, make_service + +T0 = datetime(2026, 6, 11, 9, 45) + + +@pytest.fixture +def server(tmp_path): + db_path = str(tmp_path / "events.duckdb") + # seed a ready signal + store = EventStore(db_path) + store.emit( + SignalReadyEvent( + timestamp=T0, mode=EventMode.PAPER, correlation_id="t", + message="GOOD ready", symbol="GOOD", signal_type="bull_flag", + confidence=0.9, + signal_data={"entry_price": 14.0, "stop_loss_price": 13.45, + "quality_score": 0.7}, + ) + ) + service = make_service(store, FakeExecutor()) + service.request_approvals_for_ready_signals() + + state = DashboardState(db_path, execution_service=service, + execution_mode="alpaca_paper") + httpd, _ = serve_in_background(state, host="127.0.0.1", port=0) + base = f"http://127.0.0.1:{httpd.server_address[1]}" + yield base, store + httpd.shutdown() + store.close() + + +def get(base, path): + with urllib.request.urlopen(base + path, timeout=5) as resp: + return resp.status, json.loads(resp.read()) + + +def post(base, path, body): + request = urllib.request.Request( + base + path, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=5) as resp: + return resp.status, json.loads(resp.read()) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read()) + + +def test_snapshots_endpoint_shape(server): + base, _ = server + status, data = get(base, "/api/snapshots") + assert status == 200 + for key in ("execution_mode", "approval_queue", "ready_signals", + "watch_states", "accounts", "positions", "orders", + "order_lifecycle"): + assert key in data + assert data["execution_mode"] == "alpaca_paper" + assert data["ready_signals"][0]["symbol"] == "GOOD" + assert len(data["approval_queue"]) == 1 + + +def test_approval_queue_endpoint(server): + base, _ = server + status, data = get(base, "/api/approval-queue") + assert status == 200 + assert data["approval_queue"][0]["symbol"] == "GOOD" + + +def test_approve_endpoint_executes_and_clears_queue(server): + base, _ = server + _, data = get(base, "/api/approval-queue") + order_id = data["approval_queue"][0]["order_id"] + status, result = post(base, "/api/trading/approvals/approve", + {"order_id": order_id}) + assert status == 200 and result["ok"] + _, after = get(base, "/api/approval-queue") + assert after["approval_queue"] == [] + + +def test_reject_endpoint(server): + base, _ = server + _, data = get(base, "/api/approval-queue") + order_id = data["approval_queue"][0]["order_id"] + status, result = post(base, "/api/trading/approvals/reject", + {"order_id": order_id, "reason": "test"}) + assert status == 200 and result["ok"] + + +def test_approve_unknown_order_returns_400(server): + base, _ = server + status, result = post(base, "/api/trading/approvals/approve", + {"order_id": "nope"}) + assert status == 400 + assert not result["ok"] + + +def test_unknown_route_404(server): + base, _ = server + try: + urllib.request.urlopen(base + "/api/nope", timeout=5) + raised = False + except urllib.error.HTTPError as exc: + raised = exc.code == 404 + assert raised + + +def test_dashboard_html_served(server): + base, _ = server + with urllib.request.urlopen(base + "/", timeout=5) as resp: + assert resp.status == 200 + body = resp.read().decode() + assert "Momentum" in body and "/api/snapshots" in body + + +def test_actions_503_without_execution_service(tmp_path): + state = DashboardState(str(tmp_path / "e.duckdb"), execution_service=None) + httpd, _ = serve_in_background(state, host="127.0.0.1", port=0) + base = f"http://127.0.0.1:{httpd.server_address[1]}" + try: + status, result = post(base, "/api/trading/exit-order", {"symbol": "GOOD"}) + assert status == 503 + assert not result["ok"] + finally: + httpd.shutdown() + + +# --------------------------------------------------------------------------- +# Interactive endpoints (SSE, criteria, bars, manual inject) + new snapshot keys +# --------------------------------------------------------------------------- + +def test_snapshot_includes_pnl_and_fills(server): + base, _ = server + _, data = get(base, "/api/snapshots") + assert "pnl" in data and "fills" in data and "has_execution" in data + assert data["has_execution"] is True + # pnl has the expected stat keys even with no closed trades yet + for key in ("realized_pnl", "unrealized_pnl", "total_pnl", "win_rate", + "open_positions", "closed_trades"): + assert key in data["pnl"] + + +def test_criteria_endpoint(tmp_path): + from storage.event_schema import CriteriaEvaluatedEvent + db_path = str(tmp_path / "crit.duckdb") + store = EventStore(db_path) + store.emit( + CriteriaEvaluatedEvent( + timestamp=T0, mode=EventMode.PAPER, message="c", symbol="GOOD", + criteria_results={"passed": ["gap", "vwap"], "failed": ["impulse"]}, + total_criteria=9, passed_criteria=2, success_score_pct=22.0, + ) + ) + state = DashboardState(db_path, execution_mode="alpaca_paper") + httpd, _ = serve_in_background(state, host="127.0.0.1", port=0) + base = f"http://127.0.0.1:{httpd.server_address[1]}" + try: + status, data = get(base, "/api/criteria?symbol=GOOD") + assert status == 200 + assert data["symbol"] == "GOOD" + assert data["score"] == 22.0 + by_key = {c["key"]: c["passed"] for c in data["criteria"]} + assert by_key["gap"] is True + assert by_key["impulse"] is False + finally: + httpd.shutdown() + store.close() + + +def test_bars_endpoint_with_research_db(tmp_path): + from research.ingestion.market_data import ( + ingest_daily_history, + ingest_live_minute_bars, + ) + from storage.db import get_connection + from tests.integration.test_end_to_end import FakeAlpaca + + con = get_connection(":memory:") + client = FakeAlpaca() + ingest_daily_history(con, client, ["GOOD"]) + ingest_live_minute_bars(con, client, ["GOOD"]) + + db_path = str(tmp_path / "bars.duckdb") + EventStore(db_path).close() + state = DashboardState(db_path, execution_mode="alpaca_paper", research_con=con) + httpd, _ = serve_in_background(state, host="127.0.0.1", port=0) + base = f"http://127.0.0.1:{httpd.server_address[1]}" + try: + status, data = get(base, "/api/bars?symbol=GOOD&minutes=40") + assert status == 200 + # falls back to the latest available session for the symbol + assert len(data["bars"]) == 12 + assert all("c" in b for b in data["bars"]) + finally: + httpd.shutdown() + + +def test_bars_endpoint_no_research_db(server): + base, _ = server + status, data = get(base, "/api/bars?symbol=GOOD") + assert status == 200 + assert data["bars"] == [] + + +def test_watch_add_endpoint(tmp_path): + db_path = str(tmp_path / "watch.duckdb") + EventStore(db_path).close() + injected = [] + state = DashboardState( + db_path, execution_mode="alpaca_paper", + watch_inject=lambda sym: injected.append(sym), + ) + httpd, _ = serve_in_background(state, host="127.0.0.1", port=0) + base = f"http://127.0.0.1:{httpd.server_address[1]}" + try: + status, result = post(base, "/api/watch/add", {"symbol": "tsla"}) + assert status == 200 and result["ok"] + assert result["symbol"] == "TSLA" + assert injected == ["TSLA"] + finally: + httpd.shutdown() + + +def test_watch_add_503_without_inject(server): + base, _ = server # this server has no watch_inject configured + status, result = post(base, "/api/watch/add", {"symbol": "TSLA"}) + assert status == 503 + assert not result["ok"] + + +def test_stream_delivers_snapshot_frame(server): + base, _ = server + # read the SSE stream until we see one snapshot event, then bail + req = urllib.request.urlopen(base + "/api/stream", timeout=5) + try: + buf = b"" + got = False + for _ in range(50): + line = req.readline() + if not line: + break + buf += line + if b"event: snapshot" in buf and buf.rstrip().endswith(b"}"): + got = True + break + assert got, "no snapshot frame received from SSE stream" + finally: + req.close() diff --git a/tests/unit/research/__init__.py b/tests/unit/research/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/research/test_ingestion.py b/tests/unit/research/test_ingestion.py new file mode 100644 index 0000000..cca7898 --- /dev/null +++ b/tests/unit/research/test_ingestion.py @@ -0,0 +1,158 @@ +"""Research ingestion tests: session flags, upserts, scans, providers.""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from research import query as rq +from research.ingestion.market_data import ( + classify_session, + ingest_daily_history, + ingest_live_minute_bars, + upsert_daily_bars, + upsert_minute_bars, +) +from research.ingestion.signals import scan_gappers, store_scanner_snapshot +from research.ingestion.watcher_task import ResearchWatchlistProvider +from storage.db import get_connection + +SESSION = datetime(2026, 6, 11, tzinfo=timezone.utc) + + +@pytest.fixture +def con(): + c = get_connection(":memory:") + yield c + c.close() + + +def iso(dt): + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def minute_payload(start_utc, minutes, base=5.0, step=0.01, volume=50_000): + bars = [] + price = base + for i in range(minutes): + ts = start_utc + timedelta(minutes=i) + bars.append({"t": iso(ts), "o": price, "h": price + 0.05, + "l": price - 0.03, "c": price + step, "v": volume, + "vw": price}) + price += step + return bars + + +class FakeAlpaca: + def __init__(self, minute=None, daily=None, fail=False): + self.minute = minute or {} + self.daily = daily or {} + self.fail = fail + + def get_minute_bars(self, symbols, start_iso, end_iso=None, feed=None, limit=10_000): + if self.fail: + raise RuntimeError("api down") + return {s: self.minute.get(s, []) for s in symbols} + + def get_daily_bars(self, symbols, start_iso, end_iso=None): + if self.fail: + raise RuntimeError("api down") + return {s: self.daily.get(s, []) for s in symbols} + + +def test_classify_session_boundaries(): + # 9:30 ET == 13:30 UTC in June (EDT) + def at(h, m): + return classify_session(datetime(2026, 6, 11, h, m, tzinfo=timezone.utc)) + + assert at(13, 29)[1:] == (True, False, False) # 9:29 ET premarket + assert at(13, 30)[1:] == (False, True, False) # 9:30 ET regular open + assert at(19, 59)[1:] == (False, True, False) # 3:59 ET regular + assert at(20, 0)[1:] == (False, False, True) # 4:00 ET afterhours + assert at(7, 0)[1:] == (False, False, False) # 3:00 ET closed + + +def test_upsert_minute_bars_sets_session_flags(con): + # 13:00 UTC = 9:00 ET premarket; 14:00 UTC = 10:00 ET regular + bars = minute_payload(SESSION.replace(hour=13), 5) + \ + minute_payload(SESSION.replace(hour=14), 5, base=5.2) + count = upsert_minute_bars(con, "ABCD", bars) + assert count == 10 + pre, reg = con.execute( + "SELECT SUM(CASE WHEN is_premarket THEN 1 ELSE 0 END)," + " SUM(CASE WHEN is_regular_hours THEN 1 ELSE 0 END)" + " FROM minute_bars" + ).fetchone() + assert (pre, reg) == (5, 5) + # idempotent: re-ingesting the same bars must not duplicate + upsert_minute_bars(con, "ABCD", bars) + total = con.execute("SELECT COUNT(*) FROM minute_bars").fetchone()[0] + assert total == 10 + + +def test_upsert_daily_bars_derives_previous_close(con): + daily = [ + {"t": iso(SESSION - timedelta(days=2)), "o": 4, "h": 4.5, "l": 3.9, "c": 4.2, "v": 1_000_000}, + {"t": iso(SESSION - timedelta(days=1)), "o": 4.2, "h": 4.4, "l": 4.0, "c": 4.1, "v": 900_000}, + ] + assert upsert_daily_bars(con, "ABCD", daily) == 2 + prev = rq.query_previous_close(con, "ABCD", SESSION.date()) + assert prev == pytest.approx(4.1) + adv = rq.query_avg_daily_volume(con, "ABCD", SESSION.date()) + assert adv == pytest.approx(950_000) + + +def test_ingest_live_minute_bars_with_fake_client(con): + client = FakeAlpaca(minute={"ABCD": minute_payload(SESSION.replace(hour=14), 30)}) + result = ingest_live_minute_bars(con, client, ["ABCD"], lookback_minutes=60) + assert result.minute_rows == 30 + assert result.errors == [] + bars = rq.query_minute_bars(con, "ABCD", SESSION.date()) + assert len(bars) == 30 + assert list(bars["timestamp"]) == sorted(bars["timestamp"]) + + +def test_ingest_failures_are_reported_not_raised(con): + result = ingest_live_minute_bars(con, FakeAlpaca(fail=True), ["ABCD"]) + assert result.minute_rows == 0 + assert result.errors + result = ingest_daily_history(con, FakeAlpaca(fail=True), ["ABCD"]) + assert result.daily_rows == 0 + assert result.errors + + +def test_session_symbols_price_band_configurable(con): + upsert_minute_bars(con, "CHEAP", minute_payload(SESSION.replace(hour=14), 12, base=5.0)) + upsert_minute_bars(con, "MEGA", minute_payload(SESSION.replace(hour=14), 12, base=500.0)) + in_band = rq.query_session_symbols(con, SESSION.date(), price_min=1, price_max=20) + assert [r["symbol"] for r in in_band] == ["CHEAP"] + # the old hardcoded 1-20 band was the bug; a wide band must include MEGA + wide = rq.query_session_symbols(con, SESSION.date(), price_min=0, price_max=10_000) + assert {r["symbol"] for r in wide} == {"CHEAP", "MEGA"} + + +def test_gapper_scan_and_snapshot(con): + # previous close 4.10, today ~5.0+ -> gap > 20%; adv 950k vs cum volume high + upsert_daily_bars(con, "GAPR", [ + {"t": iso(SESSION - timedelta(days=2)), "o": 4, "h": 4.5, "l": 3.9, "c": 4.2, "v": 1_000_000}, + {"t": iso(SESSION - timedelta(days=1)), "o": 4.2, "h": 4.4, "l": 4.0, "c": 4.1, "v": 900_000}, + ]) + upsert_minute_bars(con, "GAPR", minute_payload(SESSION.replace(hour=14), 60, base=5.0, volume=80_000)) + candidates = scan_gappers(con, SESSION.date(), min_gap_pct=5, min_relative_volume=2) + assert candidates and candidates[0].symbol == "GAPR" + assert candidates[0].gap_pct > 20 + assert store_scanner_snapshot(con, candidates) == len(candidates) + rows = con.execute("SELECT symbol, rank FROM scanner_snapshots").fetchall() + assert rows[0] == ("GAPR", 1) + + +def test_research_watchlist_provider_feeds_watcher(con): + upsert_daily_bars(con, "GAPR", [ + {"t": iso(SESSION - timedelta(days=1)), "o": 4.2, "h": 4.4, "l": 4.0, "c": 4.1, "v": 900_000}, + ]) + upsert_minute_bars(con, "GAPR", minute_payload(SESSION.replace(hour=14), 15, base=5.0)) + provider = ResearchWatchlistProvider(con, price_min=1, price_max=20) + candidates = provider.get_candidates(SESSION.date()) + assert candidates[0].symbol == "GAPR" + assert candidates[0].previous_close == pytest.approx(4.1) + bars = provider.get_bars("GAPR", SESSION.date()) + assert len(bars) == 15 diff --git a/tests/unit/runtime/__init__.py b/tests/unit/runtime/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/runtime/test_watcher.py b/tests/unit/runtime/test_watcher.py new file mode 100644 index 0000000..cc45cb5 --- /dev/null +++ b/tests/unit/runtime/test_watcher.py @@ -0,0 +1,103 @@ +"""Watcher state machine tests (Milestone 4).""" + +from datetime import date + +import pandas as pd +import pytest + +from runtime.watcher import WatchCandidate, Watcher, WatcherConfig +from storage.event_schema import EventMode +from storage.event_store import EventStore +from tests.synthetic import SESSION_DATE, bull_flag_bars, fading_bars, tiny_bars + + +class FakeProvider: + def __init__(self): + self.candidates: list[WatchCandidate] = [] + self.bars: dict[str, pd.DataFrame] = {} + + def get_candidates(self, session_date: date): + return self.candidates + + def get_bars(self, symbol: str, session_date: date): + return self.bars.get(symbol, pd.DataFrame()) + + +@pytest.fixture +def store(): + s = EventStore(":memory:") + yield s + s.close() + + +@pytest.fixture +def provider(): + p = FakeProvider() + p.candidates = [ + WatchCandidate("GOOD", previous_close=10.0, avg_daily_volume=500_000), + WatchCandidate("FADE", previous_close=10.0, avg_daily_volume=500_000), + WatchCandidate("TINY", previous_close=5.0), + ] + p.bars = { + "GOOD": bull_flag_bars("GOOD"), + "FADE": fading_bars("FADE"), + "TINY": tiny_bars("TINY"), + } + return p + + +def make_watcher(store, provider, **kw): + config = WatcherConfig( + session_id="test-session", mode=EventMode.PAPER, min_quality=0.0, **kw + ) + return Watcher(store, provider, config) + + +def test_tick_classifies_ready_blocked(store, provider): + watcher = make_watcher(store, provider) + result = watcher.tick(SESSION_DATE) + # TINY has < min_bars, so it stays "watching" and is not evaluated + assert result.evaluated == 2 + assert "GOOD" in result.ready + assert "FADE" in result.blocked + assert "TINY" not in result.ready and "TINY" not in result.blocked + assert set(result.discovered) == {"GOOD", "FADE", "TINY"} + + +def test_tick_emits_canonical_events(store, provider): + make_watcher(store, provider).tick(SESSION_DATE) + assert len(store.query_events(event_type="symbol_discovered")) == 3 + assert len(store.query_events(event_type="signal_ready", symbol="GOOD")) == 1 + assert len(store.query_events(event_type="criteria_evaluated")) == 2 + assert len(store.query_events(event_type="signal_blocked")) >= 1 + states = store.query_events(event_type="symbol_state_changed", symbol="GOOD") + assert states # discovered -> ... -> ready transitions recorded + + +def test_signal_ready_debounced_across_ticks(store, provider): + watcher = make_watcher(store, provider) + watcher.tick(SESSION_DATE) + watcher.tick(SESSION_DATE) + # ready emitted once even though GOOD stays ready + assert len(store.query_events(event_type="signal_ready", symbol="GOOD")) == 1 + # discovery also only once + assert len(store.query_events(event_type="symbol_discovered", symbol="GOOD")) == 1 + + +def test_max_symbols_cap(store, provider): + watcher = make_watcher(store, provider, max_symbols=1) + result = watcher.tick(SESSION_DATE) + assert result.evaluated == 1 + + +def test_provider_failure_is_caught(store): + class Broken: + def get_candidates(self, session_date): + return [WatchCandidate("BOOM")] + + def get_bars(self, symbol, session_date): + raise RuntimeError("data source down") + + result = make_watcher(store, Broken()).tick(SESSION_DATE) + assert result.errors + assert result.ready == [] diff --git a/tests/unit/schwab/__init__.py b/tests/unit/schwab/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/schwab/test_schwab_integration.py b/tests/unit/schwab/test_schwab_integration.py new file mode 100644 index 0000000..08a3f46 --- /dev/null +++ b/tests/unit/schwab/test_schwab_integration.py @@ -0,0 +1,135 @@ +"""Schwab integration tests (Milestone 3): token store, lifecycle, fallbacks.""" + +import json +import time + +import pytest + +from schwab.auth.lifecycle import TokenLifecycle +from schwab.auth.token_store import TokenBundle, TokenStore +from schwab.health.models import HealthStatus +from schwab.health.reporter import HealthReporter +from schwab.market.client import SchwabApiError, SchwabMarketClient +from schwab.orders.reader import OrdersReader +from schwab.positions.reader import PositionsReader +from schwab.settings import SchwabSettings +from storage.event_store import EventStore + + +@pytest.fixture +def settings(tmp_path): + return SchwabSettings(token_path=str(tmp_path / "tokens.json")) + + +def make_bundle(expires_in=3600, refresh="refresh-tok"): + return TokenBundle( + access_token="access-tok", + refresh_token=refresh, + expires_at=time.time() + expires_in, + token_type="Bearer", + scope="api", + ) + + +def test_token_store_round_trip_and_permissions(settings): + store = TokenStore(settings.token_path) + assert store.load() is None + store.save(make_bundle()) + loaded = store.load() + assert loaded is not None + assert loaded.access_token == "access-tok" + assert not loaded.is_expired + import os + + # The token file is locked to owner-only (0o600) on POSIX so OAuth tokens + # are never world-readable. Windows does not implement POSIX permission + # bits, so os.chmod can't produce 0o600 there — the round-trip above is the + # meaningful check on that platform. + if os.name != "nt": + mode = os.stat(settings.token_path).st_mode & 0o777 + assert mode == 0o600 # tokens must never be world-readable + + +def test_token_bundle_expiry(): + assert make_bundle(expires_in=-10).is_expired + assert not make_bundle(expires_in=600).is_expired + + +def test_lifecycle_status_unauthenticated(settings): + lifecycle = TokenLifecycle(settings) + status = lifecycle.status() + assert status["authenticated"] is False + assert lifecycle.get_access_token() is None + + +def test_lifecycle_status_authenticated(settings): + TokenStore(settings.token_path).save(make_bundle()) + lifecycle = TokenLifecycle(settings) + status = lifecycle.status() + assert status["authenticated"] is True + assert status["expired"] is False + assert lifecycle.get_access_token() == "access-tok" + + +def test_positions_reader_falls_back_when_unauthenticated(settings): + reader = PositionsReader(settings=settings, lifecycle=TokenLifecycle(settings)) + summary = reader.read_account_summary() + assert summary.is_fallback + assert summary.account_id == "SCHWAB-UNAUTH" + positions = reader.read_positions() + assert positions.is_fallback + assert positions.positions == [] + + +def test_orders_reader_empty_when_unauthenticated(settings): + reader = OrdersReader(settings=settings, lifecycle=TokenLifecycle(settings)) + assert reader.read_orders() == [] + + +def test_market_client_raises_401_without_token(settings): + client = SchwabMarketClient(settings=settings, lifecycle=TokenLifecycle(settings)) + with pytest.raises(SchwabApiError) as exc_info: + client.get_quotes(["AAPL"]) + assert exc_info.value.status == 401 + + +def test_health_reporter_unauthenticated_and_event_emission(settings): + store = EventStore(":memory:") + reporter = HealthReporter( + store=store, settings=settings, + lifecycle=TokenLifecycle(settings), session_id="t", + ) + report = reporter.check() + d = report.to_dict() + assert d["status"] in { + HealthStatus.DOWN.value, HealthStatus.UNAUTHENTICATED.value, "down", "unauthenticated", + } + events = store.query_events(event_type="broker_health_changed") + assert events, "first check must emit a broker_health_changed event" + payload = json.loads(events[0]["payload_json"]) + assert payload["broker_name"] == "schwab" + assert payload["new_health"] + # second check with no change should not re-emit + reporter.check() + assert len(store.query_events(event_type="broker_health_changed")) == 1 + store.close() + + +def test_health_reporter_recovers_with_token(tmp_path): + settings = SchwabSettings( + token_path=str(tmp_path / "tokens.json"), + broker_app_key="key", broker_app_secret="secret", + ) + store = EventStore(":memory:") + lifecycle = TokenLifecycle(settings) + reporter = HealthReporter( + store=store, settings=settings, lifecycle=lifecycle, session_id="t" + ) + reporter.check() + TokenStore(settings.token_path).save(make_bundle()) + second = reporter.check() + events = store.query_events(event_type="broker_health_changed") + assert len(events) == 2 # status changed -> new event + payload = json.loads(events[-1]["payload_json"]) + assert payload["previous_health"] != payload["new_health"] + store.close() diff --git a/tests/unit/storage/__init__.py b/tests/unit/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/storage/test_event_store.py b/tests/unit/storage/test_event_store.py new file mode 100644 index 0000000..174c830 --- /dev/null +++ b/tests/unit/storage/test_event_store.py @@ -0,0 +1,176 @@ +"""Event store ordering, payload round-trip, and projection tests (M2).""" + +from datetime import datetime, timedelta + +import pytest + +from storage.event_schema import ( + AccountPositionsUpdatedEvent, + AccountSummaryUpdatedEvent, + EventMode, + OrderApprovalRequestedEvent, + OrderApprovedEvent, + OrderRejectedEvent, + OrderSubmittedEvent, + SignalReadyEvent, + SymbolStateChangedEvent, +) +from storage.event_store import EventStore +from storage.projections import ( + query_account_positions_snapshot, + query_account_summary_snapshot, + query_approval_queue, + query_order_lifecycle_snapshot, + query_ready_signals_snapshot, + query_watch_states_snapshot, +) + +T0 = datetime(2026, 6, 11, 9, 30) + + +@pytest.fixture +def store(): + s = EventStore(":memory:") + yield s + s.close() + + +def _signal(symbol="ABCD", ts=T0, entry=5.5, stop=5.2, quality=0.7): + return SignalReadyEvent( + timestamp=ts, + mode=EventMode.PAPER, + message=f"{symbol} ready", + symbol=symbol, + signal_type="bull_flag", + confidence=0.8, + signal_data={"entry_price": entry, "stop_loss_price": stop, "quality_score": quality}, + ) + + +def test_emit_and_query_round_trip(store): + event_id = store.emit(_signal()) + rows = store.query_events(event_type="signal_ready") + assert len(rows) == 1 + assert rows[0]["id"] == event_id + import json + payload = json.loads(rows[0]["payload_json"]) + assert payload["symbol"] == "ABCD" + assert payload["signal_data"]["entry_price"] == 5.5 + + +def test_query_orders_by_timestamp_ascending(store): + for i in (3, 1, 2): + store.emit(_signal(symbol=f"S{i}", ts=T0 + timedelta(minutes=i))) + rows = store.query_events(event_type="signal_ready") + times = [r["timestamp"] for r in rows] + assert times == sorted(times) + + +def test_query_filters_by_symbol_and_session(store): + a = _signal(symbol="AAA") + a.correlation_id = "sess-1" + b = _signal(symbol="BBB") + b.correlation_id = "sess-2" + store.emit(a) + store.emit(b) + assert len(store.query_events(symbol="AAA")) == 1 + assert len(store.query_events(session_id="sess-2")) == 1 + + +def test_ready_signals_snapshot_keeps_latest_per_symbol(store): + store.emit(_signal(symbol="ABCD", ts=T0, entry=5.0)) + store.emit(_signal(symbol="ABCD", ts=T0 + timedelta(minutes=5), entry=6.0)) + snapshot = query_ready_signals_snapshot(store) + assert len(snapshot) == 1 + assert snapshot[0]["entry_price"] == 6.0 + assert snapshot[0]["stop_loss_price"] == 5.2 + assert snapshot[0]["quality_score"] == 0.7 + + +def _approval(order_id, symbol="ABCD", ts=T0): + return OrderApprovalRequestedEvent( + timestamp=ts, + mode=EventMode.PAPER, + message="approval requested", + order_id=order_id, + symbol=symbol, + requested_by="execution", + approval_mode="manual", + execution_mode="alpaca_paper", + execution_request={"symbol": symbol, "side": "buy", "quantity": 10}, + ) + + +def test_approval_queue_pending_only(store): + store.emit(_approval("o-1", "AAA")) + store.emit(_approval("o-2", "BBB", ts=T0 + timedelta(minutes=1))) + store.emit(_approval("o-3", "CCC", ts=T0 + timedelta(minutes=2))) + store.emit( + OrderApprovedEvent( + timestamp=T0 + timedelta(minutes=3), mode=EventMode.PAPER, + message="ok", order_id="o-1", symbol="AAA", approved_by="test", + ) + ) + store.emit( + OrderRejectedEvent( + timestamp=T0 + timedelta(minutes=3), mode=EventMode.PAPER, + message="no", order_id="o-2", symbol="BBB", + rejected_by="test", rejection_reason="risk", + ) + ) + queue = query_approval_queue(store) + assert [q["order_id"] for q in queue] == ["o-3"] + assert queue[0]["execution_request"]["quantity"] == 10 + + +def test_order_lifecycle_tracks_status(store): + store.emit( + OrderSubmittedEvent( + timestamp=T0 + timedelta(minutes=1), mode=EventMode.PAPER, + message="submitted", order_id="o-9", symbol="ZZZZ", + side="buy", quantity=10, price=5.5, + ) + ) + lifecycle = {o["order_id"]: o for o in query_order_lifecycle_snapshot(store)} + assert lifecycle["o-9"]["status"] == "submitted" + assert lifecycle["o-9"]["quantity"] == 10 + assert lifecycle["o-9"]["side"] == "buy" + + +def test_watch_states_snapshot(store): + store.emit( + SymbolStateChangedEvent( + timestamp=T0, mode=EventMode.PAPER, message="x", + symbol="ABCD", previous_state="watching", new_state="ready", + state_reason="score 85", + ) + ) + snapshot = query_watch_states_snapshot(store) + assert snapshot[0]["symbol"] == "ABCD" + assert snapshot[0]["state"] == "ready" + + +def test_account_snapshots_latest_per_account(store): + for equity in (100_000.0, 101_500.0): + store.emit( + AccountSummaryUpdatedEvent( + timestamp=T0, mode=EventMode.PAPER, message="acct", + broker_name="alpaca_paper", account_id="paper", + account_desc="Alpaca Paper", total_equity=equity, + cash_balance=50_000.0, buying_power=200_000.0, + net_liquidating_value=equity, + ) + ) + accounts = query_account_summary_snapshot(store) + assert len(accounts) == 1 + assert accounts[0]["total_equity"] == 101_500.0 + + store.emit( + AccountPositionsUpdatedEvent( + timestamp=T0, mode=EventMode.PAPER, message="pos", + broker_name="alpaca_paper", account_id="paper", + positions=[{"symbol": "ABCD", "quantity": 10}], + ) + ) + positions = query_account_positions_snapshot(store, broker_name="alpaca_paper") + assert positions[-1]["positions"][0]["symbol"] == "ABCD" diff --git a/tests/unit/storage/test_interactive_projections.py b/tests/unit/storage/test_interactive_projections.py new file mode 100644 index 0000000..fe4f298 --- /dev/null +++ b/tests/unit/storage/test_interactive_projections.py @@ -0,0 +1,133 @@ +"""Tests for the interactive-dashboard projections.""" + +from datetime import datetime, timedelta + +import pytest + +from storage.event_schema import ( + AccountPositionsUpdatedEvent, + CriteriaEvaluatedEvent, + EventMode, + OrderFilledEvent, + OrderSubmittedEvent, + PositionClosedEvent, +) +from storage.event_store import EventStore +from storage.projections import ( + CRITERIA_LABELS, + query_fills_feed, + query_session_pnl, + query_symbol_criteria, +) + +T0 = datetime(2026, 6, 11, 9, 45) + + +@pytest.fixture +def store(): + s = EventStore(":memory:") + yield s + s.close() + + +def test_symbol_criteria_breakdown(store): + store.emit( + CriteriaEvaluatedEvent( + timestamp=T0, mode=EventMode.PAPER, message="c", symbol="GOOD", + criteria_results={ + "passed": ["gap", "relative_volume", "vwap", "breakout"], + "failed": ["impulse", "pullback"], + }, + total_criteria=9, passed_criteria=4, success_score_pct=44.0, + ) + ) + result = query_symbol_criteria(store, "GOOD") + assert result["symbol"] == "GOOD" + assert result["score"] == 44.0 + by_key = {c["key"]: c for c in result["criteria"]} + # every known criterion is represented, in canonical order + assert [c["key"] for c in result["criteria"]] == list(CRITERIA_LABELS) + assert by_key["gap"]["passed"] is True + assert by_key["impulse"]["passed"] is False + # a criterion neither passed nor failed (not evaluated) is None + assert by_key["candle_quality"]["passed"] is None + + +def test_symbol_criteria_uses_latest_evaluation(store): + for score, passed in [(20.0, ["gap"]), (80.0, ["gap", "vwap", "breakout"])]: + store.emit( + CriteriaEvaluatedEvent( + timestamp=T0, mode=EventMode.PAPER, message="c", symbol="GOOD", + criteria_results={"passed": passed, "failed": []}, + total_criteria=9, passed_criteria=len(passed), success_score_pct=score, + ) + ) + result = query_symbol_criteria(store, "GOOD") + assert result["score"] == 80.0 + + +def test_symbol_criteria_empty_when_unseen(store): + result = query_symbol_criteria(store, "NOPE") + assert result["criteria"] == [] + assert result["score"] is None + + +def test_fills_feed_orders_newest_first(store): + store.emit( + OrderSubmittedEvent( + timestamp=T0, mode=EventMode.PAPER, message="s", order_id="o1", + symbol="AAA", side="buy", quantity=10, price=5.0, + ) + ) + store.emit( + OrderFilledEvent( + timestamp=T0 + timedelta(minutes=1), mode=EventMode.PAPER, message="f", + order_id="o1", symbol="AAA", fill_price=5.02, fill_quantity=10, + ) + ) + feed = query_fills_feed(store) + assert feed[0]["kind"] == "order_filled" + assert feed[0]["price"] == 5.02 + assert feed[-1]["kind"] == "order_submitted" + + +def test_session_pnl_realized_and_unrealized(store): + # two closed trades: one win, one loss + store.emit( + PositionClosedEvent( + timestamp=T0, mode=EventMode.PAPER, message="win", position_id="p1", + symbol="AAA", exit_price=11.0, exit_reason="target", realized_pnl=150.0, + ) + ) + store.emit( + PositionClosedEvent( + timestamp=T0 + timedelta(minutes=5), mode=EventMode.PAPER, message="loss", + position_id="p2", symbol="BBB", exit_price=9.0, exit_reason="stop", + realized_pnl=-60.0, + ) + ) + # one open position with unrealized gain + store.emit( + AccountPositionsUpdatedEvent( + timestamp=T0 + timedelta(minutes=6), mode=EventMode.PAPER, message="pos", + broker_name="alpaca_paper", account_id="paper", + positions=[{"symbol": "CCC", "quantity": 10, "unrealized_pl": 40.0}], + ) + ) + pnl = query_session_pnl(store) + assert pnl["realized_pnl"] == 90.0 + assert pnl["unrealized_pnl"] == 40.0 + assert pnl["total_pnl"] == 130.0 + assert pnl["wins"] == 1 + assert pnl["losses"] == 1 + assert pnl["win_rate"] == 0.5 + assert pnl["open_positions"] == 1 + assert pnl["closed_trades"] == 2 + + +def test_session_pnl_empty(store): + pnl = query_session_pnl(store) + assert pnl["realized_pnl"] == 0.0 + assert pnl["total_pnl"] == 0.0 + assert pnl["win_rate"] is None + assert pnl["trades"] == [] diff --git a/tests/unit/test_backtest_engine.py b/tests/unit/test_backtest_engine.py new file mode 100644 index 0000000..6445113 --- /dev/null +++ b/tests/unit/test_backtest_engine.py @@ -0,0 +1,89 @@ +"""Backtest engine regression tests. + +The engine must evaluate setups against each bar's own timestamp, not the +wall clock. If it used datetime.now(), every setup in a session replayed +outside ~09:30-11:30 ET would be marked "late" past the entry cutoff and the +backtest would report zero trades — making the whole tool silently useless on +evenings and weekends. +""" + +from datetime import datetime, timedelta + +import pandas as pd + +from strategy.backtest.engine import BacktestEngine + + +def _bars(specs, symbol="TEST"): + # 09:30 ET expressed UTC-naive (matches how bars are stored), Friday session + start = datetime(2026, 6, 12, 13, 30) + rows = [] + cum_pv = cum_v = 0.0 + for i, (o, h, l, c, v) in enumerate(specs): + ts = start + timedelta(minutes=i) + typ = (h + l + c) / 3.0 + cum_pv += typ * v + cum_v += v + rows.append( + dict(symbol=symbol, timestamp=ts, session_date=start.date(), + is_premarket=False, is_regular_hours=True, + open=o, high=h, low=l, close=c, volume=v, + vwap=cum_pv / cum_v if cum_v else c, quality_score=1.0) + ) + return pd.DataFrame(rows) + + +def _momentum_day(): + """Gap-up base (warmup) -> impulse -> pullback -> breakout after bar 15.""" + specs = [ + (12.50, 12.62, 12.45, 12.55, 60_000), (12.55, 12.66, 12.50, 12.60, 52_000), + (12.60, 12.70, 12.54, 12.64, 48_000), (12.64, 12.72, 12.58, 12.66, 46_000), + (12.66, 12.74, 12.60, 12.68, 44_000), (12.68, 12.76, 12.62, 12.70, 43_000), + (12.70, 12.78, 12.64, 12.72, 42_000), (12.72, 12.80, 12.66, 12.74, 41_000), + (12.74, 12.82, 12.68, 12.76, 40_000), (12.76, 12.84, 12.70, 12.78, 40_000), + (12.78, 12.86, 12.72, 12.80, 39_000), (12.80, 12.88, 12.74, 12.82, 38_000), + (12.82, 12.90, 12.76, 12.84, 38_000), (12.84, 12.92, 12.78, 12.86, 37_000), + (12.86, 12.94, 12.80, 12.88, 37_000), (12.88, 12.96, 12.82, 12.90, 36_000), + (12.90, 13.20, 12.88, 13.16, 130_000), (13.16, 13.50, 13.14, 13.46, 165_000), + (13.46, 13.80, 13.44, 13.76, 195_000), (13.76, 14.05, 13.74, 14.00, 210_000), + (14.00, 14.04, 13.82, 13.86, 72_000), (13.86, 13.90, 13.74, 13.80, 58_000), + (13.80, 13.86, 13.72, 13.78, 48_000), (13.78, 14.15, 13.76, 14.12, 235_000), + ] + price = specs[-1][3] + for _ in range(16): + o = price + c = round(price + 0.10, 2) + specs.append((o, round(max(o, c) + 0.04, 2), round(min(o, c) - 0.04, 2), c, 95_000)) + price = c + return _bars(specs, "GAPR") + + +def test_backtest_signals_fire_regardless_of_wall_clock(): + # Runs at whatever real time the suite executes (often evenings/weekends). + # If the engine used now(), this would be 0 signals / 0 trades. + engine = BacktestEngine(equity=100_000.0) + result = engine.run( + _momentum_day(), "GAPR", previous_close=10.0, avg_daily_volume=500_000 + ) + assert result.signals >= 1, "engine found no setups — wall-clock cutoff bug?" + assert len(result.trades) >= 1 + # each trade is a complete round trip with an exit reason + for trade in result.trades: + assert trade.exit_reason in {"target", "stop_loss", "session_end"} + assert trade.entry_price > 0 + + +def test_backtest_fader_produces_no_signal(): + """A steady downtrend below VWAP should never trigger a long.""" + specs = [] + price = 9.0 + for _ in range(34): + o = price + c = round(price - 0.07, 2) + specs.append((o, round(o + 0.02, 2), round(c - 0.03, 2), c, 40_000)) + price = c + engine = BacktestEngine(equity=100_000.0) + result = engine.run(_bars(specs, "FADE"), "FADE", + previous_close=9.2, avg_daily_volume=400_000) + assert result.signals == 0 + assert len(result.trades) == 0 diff --git a/tests/unit/test_setup_evaluator.py b/tests/unit/test_setup_evaluator.py new file mode 100644 index 0000000..30acec0 --- /dev/null +++ b/tests/unit/test_setup_evaluator.py @@ -0,0 +1,65 @@ +"""Deterministic strategy evaluation tests (Milestone 1).""" + +from datetime import datetime + +from strategy.evaluation.setup_evaluator import evaluate_setup +from tests.synthetic import bull_flag_bars, fading_bars, tiny_bars + +EVAL_TIME = datetime(2026, 6, 11, 9, 45) + + +def test_bull_flag_is_ready(): + result = evaluate_setup( + bull_flag_bars(), + previous_close=10.0, + avg_daily_volume=500_000, + evaluation_time=EVAL_TIME, + ) + assert result.status == "ready" + assert result.success_score_pct >= 60.0 + setup = result.setups[0] + assert setup["entry_price"] > 0 + assert 0 < setup["stop_loss_price"] < setup["entry_price"] + assert 0 <= setup["quality_score"] <= 1 + + +def test_fading_tape_is_blocked(): + result = evaluate_setup( + fading_bars(), + previous_close=10.0, + avg_daily_volume=500_000, + evaluation_time=EVAL_TIME, + ) + assert result.status == "blocked" + assert result.success_score_pct < 60.0 + + +def test_insufficient_data_is_blocked(): + result = evaluate_setup(tiny_bars(), previous_close=5.0, evaluation_time=EVAL_TIME) + assert result.status == "blocked" + assert "insufficient_data" in (result.reason or "") + + +def test_after_cutoff_is_late(): + result = evaluate_setup( + bull_flag_bars(), + previous_close=10.0, + avg_daily_volume=500_000, + evaluation_time=datetime(2026, 6, 11, 15, 30), + ) + assert result.status == "late" + + +def test_criteria_results_cover_all_weights(): + result = evaluate_setup( + bull_flag_bars(), + previous_close=10.0, + avg_daily_volume=500_000, + evaluation_time=EVAL_TIME, + ) + expected = { + "sufficient_data", "gap", "relative_volume", "impulse", "pullback", + "pullback_volume", "vwap", "candle_quality", "breakout", + } + evaluated = set(result.criteria_names_passed) | set(result.criteria_names_failed) + assert expected <= evaluated diff --git a/tests/unit/test_trading_execution.py b/tests/unit/test_trading_execution.py new file mode 100644 index 0000000..7d4f778 --- /dev/null +++ b/tests/unit/test_trading_execution.py @@ -0,0 +1,341 @@ +"""Trading execution service tests: signal -> approval -> order -> exit.""" + +from datetime import datetime, timedelta + +import pytest + +from alpaca_paper.execution import ExecutionRequest, ExecutionResult +from storage.event_schema import ( + AccountPositionsUpdatedEvent, + EventMode, + OrderFilledEvent, + SignalReadyEvent, +) +from storage.event_store import EventStore +from storage.projections import query_approval_queue +from trading_execution import ExecutionSettings, TradingExecutionService +from trading_mode import TradingModeSettings + +T0 = datetime(2026, 6, 11, 9, 45) + + +class FakeExecutor: + broker_name = "alpaca_paper" + + def __init__(self, status: str = "accepted"): + self.executed: list[ExecutionRequest] = [] + self.cancelled: list[dict] = [] + self.ok = True + self.status = status # "accepted" = resting/unfilled; "filled" = instant + + def execute(self, request: ExecutionRequest) -> ExecutionResult: + self.executed.append(request) + return ExecutionResult( + ok=self.ok, + order_id=request.order_id, + broker_order_id="broker-1" if self.ok else None, + status=self.status if self.ok else "error", + error=None if self.ok else "broker down", + ) + + def cancel_entry(self, order_id, broker_order_id, symbol, reason): + self.cancelled.append( + {"order_id": order_id, "symbol": symbol, "reason": reason} + ) + return ExecutionResult(ok=True, order_id=order_id, status="cancelled") + + +@pytest.fixture +def store(): + s = EventStore(":memory:") + yield s + s.close() + + +class Clock: + """A controllable clock for deterministic wall-clock timeout tests.""" + + def __init__(self, start: datetime | None = None): + self.t = start or datetime(2026, 6, 11, 9, 45) + + def __call__(self) -> datetime: + return self.t + + def advance(self, minutes: float) -> None: + self.t += timedelta(minutes=minutes) + + +def emit_signal(store, symbol="GOOD", entry=14.0, stop=13.45): + store.emit( + SignalReadyEvent( + timestamp=T0, + mode=EventMode.PAPER, + correlation_id="t", + message=f"{symbol} ready", + symbol=symbol, + signal_type="bull_flag", + confidence=0.9, + signal_data={ + "entry_price": entry, + "stop_loss_price": stop, + "quality_score": 0.7, + }, + ) + ) + + +def emit_positions(store, positions): + store.emit( + AccountPositionsUpdatedEvent( + timestamp=T0, + mode=EventMode.PAPER, + message="pos", + broker_name="alpaca_paper", + account_id="paper", + positions=positions, + ) + ) + + +def make_service(store, executor=None, price_provider=None, now_fn=None, **settings_kw): + defaults = dict( + enabled=True, auto_approve=False, max_orders_per_tick=2, + max_concurrent_positions=3, risk_per_trade_pct=0.01, + default_equity=100_000.0, + ) + defaults.update(settings_kw) + return TradingExecutionService( + store, + executor=executor or FakeExecutor(), + settings=ExecutionSettings(**defaults), + trading_mode=TradingModeSettings(execution_mode="alpaca_paper"), + session_id="t", + price_provider=price_provider, + now_fn=now_fn, + ) + + +def test_ready_signal_creates_approval_request(store): + emit_signal(store) + service = make_service(store) + requested = service.request_approvals_for_ready_signals() + assert len(requested) == 1 # returns order ids + queue = query_approval_queue(store) + assert len(queue) == 1 + request = queue[0]["execution_request"] + # risk sizing: 1% of 100k = $1000 risk / $0.55 per share ≈ 1818 shares + assert request["quantity"] == int(1000 / 0.55) + assert request["stop_loss_price"] == 13.45 + # take profit at entry + 2R + assert request["take_profit_price"] == pytest.approx(14.0 + 2 * 0.55) + + +def test_manual_approval_executes_order(store): + emit_signal(store) + executor = FakeExecutor() + service = make_service(store, executor) + service.request_approvals_for_ready_signals() + order_id = query_approval_queue(store)[0]["order_id"] + + result = service.approve_order(order_id, approved_by="dashboard") + assert result["ok"] + assert executor.executed[0].symbol == "GOOD" + assert store.query_events(event_type="order_approved") + assert query_approval_queue(store) == [] # no longer pending + + +def test_rejection_clears_queue_without_executing(store): + emit_signal(store) + executor = FakeExecutor() + service = make_service(store, executor) + service.request_approvals_for_ready_signals() + order_id = query_approval_queue(store)[0]["order_id"] + + result = service.reject_order(order_id, reason="too extended") + assert result["ok"] + assert executor.executed == [] + assert query_approval_queue(store) == [] + assert store.query_events(event_type="order_rejected") + + +def test_approving_unknown_order_fails_cleanly(store): + service = make_service(store) + assert not service.approve_order("nope")["ok"] + assert not service.reject_order("nope")["ok"] + + +def test_auto_approve_tick_executes(store): + emit_signal(store) + executor = FakeExecutor() + service = make_service(store, executor, auto_approve=True) + out = service.tick() + assert len(out["approvals_requested"]) == 1 + assert len(out["auto_executed"]) == 1 + assert executor.executed[0].symbol == "GOOD" + + +def test_no_duplicate_requests_for_same_symbol(store): + emit_signal(store) + service = make_service(store) + assert len(service.request_approvals_for_ready_signals()) == 1 + assert service.request_approvals_for_ready_signals() == [] + assert len(query_approval_queue(store)) == 1 + + +def test_held_symbols_are_skipped(store): + emit_signal(store) + emit_positions(store, [{"symbol": "GOOD", "quantity": 10}]) + service = make_service(store) + assert service.request_approvals_for_ready_signals() == [] + + +def test_max_concurrent_positions_guard(store): + emit_signal(store, symbol="NEWS") + emit_positions( + store, + [{"symbol": s, "quantity": 1} for s in ("AAA", "BBB", "CCC")], + ) + service = make_service(store, max_concurrent_positions=3) + assert service.request_approvals_for_ready_signals() == [] + assert store.query_events(event_type="risk_rule_triggered") + + +def test_exit_order_closes_position(store): + emit_positions(store, [{"symbol": "GOOD", "quantity": 10}]) + executor = FakeExecutor() + service = make_service(store, executor) + result = service.submit_exit_order("GOOD") + assert result["ok"] + exit_request = executor.executed[0] + assert exit_request.side == "sell" + assert exit_request.quantity == 10 + + +def test_exit_order_without_position_fails(store): + service = make_service(store) + result = service.submit_exit_order("NOPE") + assert not result["ok"] + + +# --------------------------------------------------------------------------- +# Auto-arm entry mechanism: tunable reward, entry order type, and back-out +# --------------------------------------------------------------------------- + +def test_reward_multiple_is_configurable(store): + emit_signal(store) # entry 14.0 stop 13.45 -> risk 0.55 + service = make_service(store, reward_multiple=3.0) + service.request_approvals_for_ready_signals() + req = query_approval_queue(store)[0]["execution_request"] + assert req["take_profit_price"] == pytest.approx(14.0 + 3.0 * 0.55) + + +def test_entry_order_type_flows_into_request(store): + emit_signal(store) + service = make_service(store, entry_order_type="limit") + service.request_approvals_for_ready_signals() + req = query_approval_queue(store)[0]["execution_request"] + assert req["order_type"] == "limit" + + +def test_unfilled_entry_is_armed_then_times_out(store): + emit_signal(store) + executor = FakeExecutor(status="accepted") # resting limit, never fills + clock = Clock() + service = make_service( + store, executor, auto_approve=True, now_fn=clock, + entry_timeout_bars=2, entry_invalidate_pct=-1.0, # disable price-break + ) + # request + auto-approve -> order is armed, not yet expired + service.tick() + assert len(service._armed) == 1 + assert executor.cancelled == [] + # 1 minute later: under the 2-minute window -> still armed + clock.advance(1) + assert service.expire_stale_entries() == [] + assert len(service._armed) == 1 + # 2 minutes after arming -> timed out and backed out + clock.advance(1) + out = service.expire_stale_entries() + assert len(out) == 1 + assert len(executor.cancelled) == 1 + assert "timed out" in executor.cancelled[0]["reason"] + assert service._armed == {} + + +def test_fast_guard_does_not_shorten_timeout(store): + """Checking many times a second must not trip the wall-clock timeout early.""" + emit_signal(store) + executor = FakeExecutor(status="accepted") + clock = Clock() + service = make_service( + store, executor, auto_approve=True, now_fn=clock, + entry_timeout_bars=2, entry_invalidate_pct=-1.0, + ) + service.tick() + # hammer the guard 50 times with no time passing -> no premature back-out + for _ in range(50): + assert service.expire_stale_entries() == [] + assert len(service._armed) == 1 + assert executor.cancelled == [] + + +def test_unfilled_entry_invalidated_by_price_break(store): + emit_signal(store, entry=14.0, stop=13.45) + executor = FakeExecutor(status="accepted") + price = {"GOOD": 14.10} # starts above the trigger + service = make_service( + store, executor, auto_approve=True, + entry_timeout_bars=0, # disable timeout; isolate price-break + entry_invalidate_pct=0.0, # any trade below entry invalidates + price_provider=lambda sym: price.get(sym), + ) + service.tick() + assert len(service._armed) == 1 + # price holds above entry -> no back-out + assert service.expire_stale_entries() == [] + # price breaks back below the entry trigger -> cancel + price["GOOD"] = 13.90 + out = service.expire_stale_entries() + assert len(out) == 1 + assert "invalidated" in executor.cancelled[0]["reason"] + assert service._armed == {} + + +def test_armed_entry_cleared_on_fill_without_cancel(store): + emit_signal(store) + executor = FakeExecutor(status="accepted") + service = make_service( + store, executor, auto_approve=True, entry_timeout_bars=1, + ) + service.tick() + assert len(service._armed) == 1 + order_id = next(iter(service._armed)) + # broker reports the fill via the event store + store.emit( + OrderFilledEvent( + timestamp=T0, mode=EventMode.PAPER, message="fill", + order_id=order_id, symbol="GOOD", fill_price=14.01, fill_quantity=10, + ) + ) + out = service.expire_stale_entries() + # filled -> tracking stops, nothing cancelled even though timeout=1 + assert out == [] + assert executor.cancelled == [] + assert service._armed == {} + + +def test_backout_frees_symbol_to_resignal(store): + emit_signal(store) + executor = FakeExecutor(status="accepted") + clock = Clock() + service = make_service( + store, executor, auto_approve=True, now_fn=clock, + entry_timeout_bars=1, entry_invalidate_pct=-1.0, + ) + service.tick() + symbol = service._armed[next(iter(service._armed))]["symbol"] + assert symbol in service._requested_symbols + clock.advance(1) # reach the 1-minute timeout + service.expire_stale_entries() + # after backing out, the symbol is no longer marked as requested + assert symbol not in service._requested_symbols diff --git a/trading_execution.py b/trading_execution.py new file mode 100644 index 0000000..8acf680 --- /dev/null +++ b/trading_execution.py @@ -0,0 +1,466 @@ +"""Trading execution service (Milestone 4/5 glue). + +Connects ready signals to broker execution with risk controls and an +approval workflow: + + signal_ready + -> risk checks (sizing, max concurrent, daily-loss circuit breaker) + -> order_approval_requested + -> [manual approval in the dashboard] or [auto-approve] + -> order_approved -> broker submit -> order_submitted (+ filled) + +Exit orders close existing broker positions on demand. +""" + +from __future__ import annotations + +import json +import logging +import os +from dataclasses import dataclass +from datetime import datetime + +from alpaca_paper.execution import ( + AlpacaPaperExecutor, + ExecutionRequest, +) +from storage.event_schema import ( + EventMode, + OrderApprovalRequestedEvent, + OrderApprovedEvent, + OrderRejectedEvent, + RiskRuleTriggeredEvent, +) +from storage.event_store import EventStore +from storage.projections import ( + query_account_positions_snapshot, + query_approval_queue, + query_ready_signals_snapshot, +) +from strategy.risk.position_sizing import ( + PositionSizingConfig, + calculate_position_size, +) +from trading_mode import TradingModeSettings + +logger = logging.getLogger(__name__) + + +@dataclass +class ExecutionSettings: + enabled: bool = True + auto_approve: bool = False + max_orders_per_tick: int = 1 + max_concurrent_positions: int = 3 + risk_per_trade_pct: float = 0.01 + default_equity: float = 100_000.0 + max_daily_loss_pct: float = 0.03 + # --- entry mechanism (Ross-Cameron-style; all tunable) ----------------- + # reward target as a multiple of risk (entry + reward_multiple * (entry-stop)) + reward_multiple: float = 2.0 + # how the entry order is placed: "limit" rests at the entry trigger so an + # unfilled order is a real, cancellable state; "market" fills immediately + entry_order_type: str = "limit" + # cancel an unfilled entry after this many minutes of resting (the "back + # out" time box; ~1 bar == 1 minute). 0 disables the timeout. Wall-clock + # based, so the invalidation guard can run far more often than this. + entry_timeout_bars: int = 2 + # cancel an unfilled entry if price trades back below the entry trigger by + # this fraction (e.g. 0.0 = any break below entry; 0.005 = 0.5% below). + # Set to a negative number to disable price-break invalidation. + entry_invalidate_pct: float = 0.0 + + @classmethod + def from_env(cls, env: dict[str, str] | None = None) -> "ExecutionSettings": + values = dict(os.environ) + if env is not None: + values.update(env) + + def flag(key: str, default: str) -> bool: + return values.get(key, default).strip().lower() in {"1", "true", "yes", "on"} + + order_type = values.get("TRADING_ENTRY_ORDER_TYPE", "limit").strip().lower() + if order_type not in {"limit", "market"}: + order_type = "limit" + + return cls( + enabled=flag("TRADING_EXECUTION_ENABLED", "1"), + auto_approve=flag("TRADING_AUTO_APPROVE", "0"), + max_orders_per_tick=int(values.get("TRADING_MAX_ORDERS_PER_TICK", "1")), + max_concurrent_positions=int( + values.get("TRADING_MAX_CONCURRENT_POSITIONS", "3") + ), + risk_per_trade_pct=float(values.get("TRADING_RISK_PER_TRADE_PCT", "0.01")), + default_equity=float(values.get("TRADING_DEFAULT_EQUITY", "100000")), + max_daily_loss_pct=float(values.get("TRADING_MAX_DAILY_LOSS_PCT", "0.03")), + reward_multiple=float(values.get("TRADING_REWARD_MULTIPLE", "2.0")), + entry_order_type=order_type, + entry_timeout_bars=int(values.get("TRADING_ENTRY_TIMEOUT_BARS", "2")), + entry_invalidate_pct=float( + values.get("TRADING_ENTRY_INVALIDATE_PCT", "0.0") + ), + ) + + +class TradingExecutionService: + """Drives the signal -> approval -> order pipeline each tick.""" + + def __init__( + self, + store: EventStore, + executor: AlpacaPaperExecutor | None = None, + settings: ExecutionSettings | None = None, + trading_mode: TradingModeSettings | None = None, + session_id: str | None = None, + equity: float | None = None, + price_provider=None, + now_fn=None, + ): + self.store = store + self.settings = settings or ExecutionSettings.from_env() + self.trading_mode = trading_mode or TradingModeSettings.from_env() + self.session_id = session_id + self.equity = equity or self.settings.default_equity + self.executor = executor or AlpacaPaperExecutor( + store, session_id=session_id + ) + self.mode = EventMode.PAPER + # optional callable: symbol -> latest price, used to invalidate unfilled + # entries that break back below the entry trigger before filling + self.price_provider = price_provider + # injectable clock so the entry timeout is wall-clock based (and so the + # invalidation check can run far more often than the timeout window + # without changing the timeout's meaning); tests pass a fake clock + self._now = now_fn or datetime.now + # signals we've already requested approval for this session + self._requested_symbols: set[str] = set() + # armed (submitted, awaiting fill) entries we may need to cancel. + # order_id -> {symbol, entry_price, broker_order_id, armed_at, checks} + self._armed: dict[str, dict] = {} + + # -- pipeline ----------------------------------------------------------- + + def _open_position_count(self) -> int: + snapshots = query_account_positions_snapshot( + self.store, broker_name=self.executor.broker_name + ) + if not snapshots: + return 0 + return len(snapshots[-1].get("positions") or []) + + def _held_symbols(self) -> set[str]: + snapshots = query_account_positions_snapshot( + self.store, broker_name=self.executor.broker_name + ) + if not snapshots: + return set() + return { + str(p.get("symbol")) + for p in snapshots[-1].get("positions") or [] + if p.get("symbol") + } + + def request_approvals_for_ready_signals(self) -> list[str]: + """Turn fresh ready signals into approval requests. Returns order ids.""" + if not self.settings.enabled: + return [] + + if self._open_position_count() >= self.settings.max_concurrent_positions: + self.store.emit( + RiskRuleTriggeredEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message="Max concurrent positions reached — no new entries", + rule_type="max_concurrent_positions", + rule_value=float(self.settings.max_concurrent_positions), + current_state={"open_positions": self._open_position_count()}, + action_taken="skipped_new_entries", + ) + ) + return [] + + held = self._held_symbols() + pending = {row["symbol"] for row in query_approval_queue(self.store)} + signals = query_ready_signals_snapshot(self.store, session_id=self.session_id) + + created: list[str] = [] + for signal in signals: + if len(created) >= self.settings.max_orders_per_tick: + break + symbol = signal["symbol"] + if symbol in held or symbol in pending or symbol in self._requested_symbols: + continue + entry = signal.get("entry_price") + stop = signal.get("stop_loss_price") + if not entry or not stop or stop >= entry: + continue + + sizing = calculate_position_size( + float(entry), + float(stop), + equity=self.equity, + config=PositionSizingConfig( + risk_per_trade_pct=self.settings.risk_per_trade_pct, + default_equity=self.settings.default_equity, + ), + ) + if sizing.position_size <= 0: + continue + + entry_f = float(entry) + stop_f = float(stop) + risk_per_share = entry_f - stop_f + request = ExecutionRequest( + symbol=symbol, + side="buy", + quantity=sizing.position_size, + entry_price=entry_f, + stop_loss_price=stop_f, + take_profit_price=round( + entry_f + self.settings.reward_multiple * risk_per_share, 2 + ), + order_type=self.settings.entry_order_type, + ) + approval_mode = "auto" if self.settings.auto_approve else "manual" + self.store.emit( + OrderApprovalRequestedEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=( + f"Approval requested ({approval_mode}): buy " + f"{request.quantity} {symbol} @ ~{entry} stop {stop}" + ), + order_id=request.order_id, + symbol=symbol, + requested_by="trading_execution", + approval_mode=approval_mode, + execution_mode=self.trading_mode.execution_mode, + execution_request=request.to_payload(), + ) + ) + self._requested_symbols.add(symbol) + created.append(request.order_id) + logger.info("approval requested for %s (%s)", symbol, request.order_id) + + return created + + def approve_order( + self, order_id: str, approved_by: str = "dashboard", notes: str | None = None + ) -> dict: + """Approve a pending order and execute it.""" + entry = self._find_pending(order_id) + if entry is None: + return {"ok": False, "error": f"order {order_id} not pending"} + request = ExecutionRequest.from_payload(entry["execution_request"]) + self.store.emit( + OrderApprovedEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=f"Order {order_id} approved by {approved_by}", + order_id=order_id, + symbol=request.symbol, + approved_by=approved_by, + approval_notes=notes, + ) + ) + result = self.executor.execute(request) + # If the entry didn't fill immediately (a resting limit at the trigger), + # arm it so expire_stale_entries() can back out on timeout or a break + # below the entry. Market orders that fill on submit are never armed. + if result.ok and result.status not in {"filled", "cancelled"} and request.entry_price: + self._armed[order_id] = { + "symbol": request.symbol, + "entry_price": float(request.entry_price), + "broker_order_id": result.broker_order_id, + "armed_at": self._now(), + "checks": 0, + } + return { + "ok": result.ok, + "order_id": order_id, + "broker_order_id": result.broker_order_id, + "status": result.status, + "error": result.error, + } + + def reject_order( + self, order_id: str, rejected_by: str = "dashboard", reason: str = "manual" + ) -> dict: + entry = self._find_pending(order_id) + if entry is None: + return {"ok": False, "error": f"order {order_id} not pending"} + self.store.emit( + OrderRejectedEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=f"Order {order_id} rejected by {rejected_by}: {reason}", + order_id=order_id, + symbol=entry.get("symbol") or "", + rejected_by=rejected_by, + rejection_reason=reason, + ) + ) + # allow the symbol to re-signal later + self._requested_symbols.discard(entry.get("symbol") or "") + return {"ok": True, "order_id": order_id} + + def process_auto_approvals(self) -> list[dict]: + """Auto-approve any pending requests marked approval_mode=auto.""" + results = [] + for entry in query_approval_queue(self.store): + if entry.get("approval_mode") == "auto": + results.append( + self.approve_order(entry["order_id"], approved_by="auto") + ) + return results + + def submit_exit_order(self, symbol: str, reason: str = "manual_exit") -> dict: + """Close an open broker position for symbol with a market sell.""" + held = self._held_symbols() + quantity = None + snapshots = query_account_positions_snapshot( + self.store, broker_name=self.executor.broker_name + ) + if snapshots: + for p in snapshots[-1].get("positions") or []: + if str(p.get("symbol")) == symbol: + quantity = int(abs(float(p.get("quantity") or 0))) + if symbol not in held or not quantity: + return {"ok": False, "error": f"no open position for {symbol}"} + request = ExecutionRequest(symbol=symbol, side="sell", quantity=quantity) + result = self.executor.execute(request) + return { + "ok": result.ok, + "order_id": request.order_id, + "broker_order_id": result.broker_order_id, + "status": result.status, + "error": result.error, + "reason": reason, + } + + def _filled_order_ids(self) -> set[str]: + ids: set[str] = set() + for e in self.store.query_events(event_type="order_filled", limit=None): + payload = json.loads(e.get("payload_json", "{}")) + if payload.get("order_id"): + ids.add(str(payload["order_id"])) + return ids + + def _cancelled_order_ids(self) -> set[str]: + ids: set[str] = set() + for e in self.store.query_events(event_type="order_cancelled", limit=None): + payload = json.loads(e.get("payload_json", "{}")) + if payload.get("order_id"): + ids.add(str(payload["order_id"])) + return ids + + def expire_stale_entries(self) -> list[dict]: + """Back out of unfilled entries that timed out or broke their trigger. + + This is the disciplined other half of auto-arming: once a setup is + recognized we enter with conviction, but if the fill never comes (the + move didn't follow through) we don't sit there forever — we cancel on a + time box or when price falls back through the entry, freeing risk + budget and the concurrent-position slot for the next setup. + + Safe to call as often as you like: the price-break check uses the live + ``price_provider`` each call, while the timeout is wall-clock based, so + running this every few seconds reacts fast without shortening the + timeout window. + """ + if not self._armed: + return [] + + filled = self._filled_order_ids() + cancelled = self._cancelled_order_ids() + invalidate_pct = self.settings.entry_invalidate_pct + timeout_bars = self.settings.entry_timeout_bars + now = self._now() + actions: list[dict] = [] + + for order_id in list(self._armed): + armed = self._armed[order_id] + symbol = armed["symbol"] + + # already resolved at the broker -> stop tracking + if order_id in filled: + self._armed.pop(order_id, None) + continue + if order_id in cancelled: + self._armed.pop(order_id, None) + self._requested_symbols.discard(symbol) + continue + + armed["checks"] += 1 + reason = None + + # (a) price-break invalidation: traded back below the entry trigger. + # Uses the live last-trade price, so this reacts tick-by-tick rather + # than waiting for a bar to close. + if invalidate_pct >= 0 and self.price_provider is not None: + try: + last = self.price_provider(symbol) + except Exception: # noqa: BLE001 + last = None + if last is not None: + threshold = armed["entry_price"] * (1.0 - invalidate_pct) + if float(last) < threshold: + reason = ( + f"entry invalidated: {float(last):.4f} < trigger " + f"{threshold:.4f}" + ) + + # (b) time box: unfilled for too long (wall-clock minutes ~ bars) + if reason is None and timeout_bars > 0: + elapsed_min = (now - armed["armed_at"]).total_seconds() / 60.0 + if elapsed_min >= timeout_bars: + reason = f"entry timed out: unfilled {elapsed_min:.1f} min" + + if reason is not None: + self.executor.cancel_entry( + order_id, armed.get("broker_order_id"), symbol, reason + ) + self.store.emit( + RiskRuleTriggeredEvent( + timestamp=datetime.now(), + mode=self.mode, + correlation_id=self.session_id, + message=f"Backed out of {symbol} entry — {reason}", + rule_type="entry_backout", + rule_value=float(timeout_bars), + current_state={ + "symbol": symbol, + "order_id": order_id, + "checks": armed["checks"], + }, + action_taken="cancelled_unfilled_entry", + ) + ) + self._armed.pop(order_id, None) + self._requested_symbols.discard(symbol) + actions.append({"order_id": order_id, "symbol": symbol, "reason": reason}) + + return actions + + def tick(self) -> dict: + """One execution pass: expire stale entries, request approvals, auto-execute.""" + backed_out = self.expire_stale_entries() + requested = self.request_approvals_for_ready_signals() + auto = self.process_auto_approvals() if self.settings.auto_approve else [] + return { + "approvals_requested": requested, + "auto_executed": auto, + "backed_out": backed_out, + } + + # -- helpers -------------------------------------------------------------- + + def _find_pending(self, order_id: str) -> dict | None: + for entry in query_approval_queue(self.store): + if str(entry.get("order_id")) == str(order_id): + return entry + return None diff --git a/trading_mode.py b/trading_mode.py new file mode 100644 index 0000000..84c9d11 --- /dev/null +++ b/trading_mode.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Literal, cast + + +TradingMode = Literal["schwab_live", "alpaca_paper"] + + +@dataclass(frozen=True) +class TradingModeSettings: + execution_mode: TradingMode = "alpaca_paper" + + @classmethod + def from_env(cls, env: dict[str, str] | None = None) -> "TradingModeSettings": + values = dict(os.environ) + if env is not None: + values.update(env) + mode = values.get("TRADING_EXECUTION_MODE", "alpaca_paper") + if mode not in {"schwab_live", "alpaca_paper"}: + mode = "alpaca_paper" + return cls(execution_mode=cast(TradingMode, mode))