Compare commits
No files matched your search
@@ -0,0 +1,48 @@
|
||||
# VCS / CI
|
||||
.git
|
||||
.github
|
||||
.gitignore
|
||||
|
||||
# Secrets and runtime data (never bake into the image; provided via Secret/PVC)
|
||||
.env
|
||||
.env.*
|
||||
data/
|
||||
*.duckdb
|
||||
*.wal
|
||||
|
||||
# Local Grafana stack (provisioned via the Helm chart instead)
|
||||
grafana/
|
||||
|
||||
# Deploy artifacts don't belong inside the app image
|
||||
deploy/
|
||||
Dockerfile
|
||||
.dockerignore
|
||||
|
||||
# Python caches / build / venvs
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
build/
|
||||
dist/
|
||||
.venv
|
||||
venv/
|
||||
ENV/
|
||||
env/
|
||||
.ruff_cache/
|
||||
|
||||
# Tests and coverage
|
||||
tests/
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# Editor / OS cruft
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
.DS_Store
|
||||
|
||||
# Large artifacts
|
||||
*.zip
|
||||
archive/
|
||||
uv.lock
|
||||
+214
-166
@@ -1,190 +1,238 @@
|
||||
# ============================================================
|
||||
# Momentum — environment configuration (copy to .env and fill)
|
||||
# Regenerated 2026-07-09 from the live config. Values kept here
|
||||
# are the current tuned defaults; <REDACTED> = secret, fill from
|
||||
# your protected store. See deploy/docker/README.md for which
|
||||
# layer (compose vs .env) each group belongs in.
|
||||
# ============================================================
|
||||
|
||||
# ---- Database (PostgreSQL) — the single datastore (event store + bar warehouse)
|
||||
# Either a full URL:
|
||||
DATABASE_URL=postgresql://USER:PASSWORD@HOST:5432/momentum
|
||||
# ...or individual vars (used only if DATABASE_URL is unset):
|
||||
# PGHOST=127.0.0.1
|
||||
# PGPORT=5432
|
||||
# PGUSER=admin
|
||||
# PGPASSWORD=
|
||||
# PGDATABASE=momentum
|
||||
TOKEN_ENCRYPTION_KEY=<REDACTED>
|
||||
|
||||
# ---- Alpaca (market data + paper broker) — REQUIRED for live paper trading
|
||||
ALPACA_API_KEY=
|
||||
ALPACA_SECRET_KEY=
|
||||
# Alpaca paper trading endpoint (default is fine)
|
||||
APCA_API_BASE_URL=https://paper-api.alpaca.markets
|
||||
# These two aliases are also read by the paper client:
|
||||
APCA_API_KEY_ID=
|
||||
APCA_API_SECRET_KEY=
|
||||
DASHBOARD_PORT=8765
|
||||
# Dashboard POST auth + operator command bus (the :8765 kill switch).
|
||||
# DASHBOARD_TOKEN is REQUIRED-when-commands-on: /api/command returns 503 unless
|
||||
# it is set (commands never run unauthenticated); it also gates approve/reject/
|
||||
# exit POSTs. Empty = read-only dashboard, no command bus. Keep it out of logs.
|
||||
DASHBOARD_TOKEN=
|
||||
# Enable the operator command bus (flatten-all / flatten-symbol / pause / resume).
|
||||
# Needs DASHBOARD_TOKEN set too; the loop consumes commands on its 5s guard.
|
||||
COMMAND_BUS_ENABLED=0
|
||||
#curl.exe -H "APCA-API-KEY-ID: $key" -H "APCA-API-SECRET-KEY: $secret" https://paper-api.alpaca.markets/v2/account
|
||||
# Alpaca API (primary data provider)
|
||||
ALPACA_API_KEY=<REDACTED>
|
||||
ALPACA_SECRET_KEY=<REDACTED>
|
||||
ALPACA_BASE_URL=https://paper-api.alpaca.markets/v2
|
||||
|
||||
# ---- Schwab (live broker, Milestone 3) — OPTIONAL for paper trading
|
||||
SCHWAB_MARKET_DATA_APP_KEY=
|
||||
SCHWAB_MARKET_DATA_APP_SECRET=
|
||||
SCHWAB_BROKER_APP_KEY=
|
||||
SCHWAB_BROKER_APP_SECRET=
|
||||
SCHWAB_REDIRECT_URI=
|
||||
SCHWAB_TOKEN_PATH=data/schwab_tokens.json
|
||||
# Polygon (optional fallback)
|
||||
POLYGON_API_KEY=<REDACTED>
|
||||
|
||||
# ---- Optional research / news providers
|
||||
POLYGON_API_KEY=
|
||||
FINNHUB_API_KEY=
|
||||
Alpha_Vantage_Api_Key=
|
||||
# Additional news and filing sources
|
||||
FINNHUB_API_KEY=<REDACTED>
|
||||
FINNHUB_EMAIL=fez5587@gmail.com
|
||||
SEC_API_KEY=<REDACTED>
|
||||
DATABENTO_API_KEY=<REDACTED>
|
||||
|
||||
# News-catalyst discovery (optional): RSS feeds as name=url, comma-separated.
|
||||
# When set, tickers mentioned in recent news ($TICKER or (NASDAQ:TICKER)) are
|
||||
# added to the watchlist. Unset = disabled (no effect on the loop).
|
||||
# NEWS_FEEDS=prnewswire=https://www.prnewswire.com/rss/news-releases-list.rss,businesswire=https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRWQ==
|
||||
# Data directory for parquet files and DuckDB
|
||||
DATA_DIR=./data
|
||||
# Boot warns (log + one Telegram note) if data/learned_params.json is older than
|
||||
# this many days — a stale file means the nightly tuner stopped running. Never blocks boot.
|
||||
LEARNED_PARAMS_MAX_AGE_DAYS=3
|
||||
|
||||
# ---- Local LLM news/catalyst enrichment (Ollama) ----
|
||||
# A local model classifies ingested headlines (catalyst type, sentiment,
|
||||
# conviction, dilution) into news_catalyst_cache — surfaced on the dashboard
|
||||
# approval queue. Runs off the hot path; degrades to a no-op if Ollama is down.
|
||||
# OLLAMA_ENABLED=1
|
||||
# OLLAMA_HOST=http://localhost:11434
|
||||
# OLLAMA_MODEL=qwen2.5:7b-instruct # fits 12GB; best small-model JSON.
|
||||
# # alts: llama3.1:8b-instruct-q4_K_M, mistral:7b-instruct
|
||||
# NEWS_ENRICH_ENABLED=1
|
||||
# NEWS_ENRICH_INTERVAL_SECONDS=120
|
||||
# NEWS_ENRICH_LOOKBACK_HOURS=12
|
||||
# NEWS_ENRICH_BATCH_LIMIT=50 # max headlines classified per pass (GPU budget)
|
||||
# Phase 2 (automation — each ships OFF; separate from the advisory above so the
|
||||
# dashboard never silently changes scoring):
|
||||
# score blend — fold the catalyst into the setup quality score AND graded arming
|
||||
# rank (bullish catalyst boosts, confirmed dilution sinks below the armed cut).
|
||||
# NEWS_CATALYST_SCORE_ENABLED=0
|
||||
# NEWS_RANK_BOOST_GAIN=2.0 # arming-rank multiplier = 1 + gain*score
|
||||
# NEWS_DILUTIVE_PENALTY=0.1 # arming-rank multiplier for a confirmed dilutive name
|
||||
# dilution veto — block confirmed dilutive offerings before sizing.
|
||||
# NEWS_DILUTION_VETO_ENABLED=0
|
||||
# NEWS_DILUTION_VETO_CONVICTION=0.6
|
||||
DISCORD_WEBHOOK_URL=<REDACTED>
|
||||
DISCORD_TRADING_WEBHOOK_URL=<REDACTED>
|
||||
DISCORD_DEVELOPER_WEBHOOK_URL=<REDACTED>
|
||||
|
||||
# ============================================================
|
||||
# Live paper-trading orchestrator (run_live_paper.py)
|
||||
# ============================================================
|
||||
# Telegram notifier (bot @traitor_momentum_bot) — gated; no-op if unset
|
||||
TELEGRAM_BOT_TOKEN=<REDACTED>
|
||||
TELEGRAM_CHAT_ID='2082795871'
|
||||
# Dead-man's switch (qabot container): the loop's own notifier dies WITH the
|
||||
# loop, so the always-up qabot watches max(events.timestamp) from outside.
|
||||
DEADMAN_STALE_MIN=5 # alert when no events for this many minutes during market hours
|
||||
DEADMAN_DAILY_PING=1 # nightly OK-ping after 20:30 ET — proves the watchdog itself is alive
|
||||
|
||||
# Execution destination: alpaca_paper (safe) or schwab_live (REAL MONEY)
|
||||
TRADING_EXECUTION_MODE=alpaca_paper
|
||||
TRADING_EXECUTION_ENABLED=1
|
||||
TRADING_EXECUTION_INTERVAL_SECONDS=30
|
||||
TRADING_AUTO_APPROVE=0 # 0 = manual approval in the dashboard, 1 = auto-arm (no approval)
|
||||
TRADING_MAX_ORDERS_PER_TICK=1
|
||||
# NOTE: concurrency is set ONCE in the risk section below (was duplicated here).
|
||||
TRADING_RISK_PER_TRADE_PCT=0.01 # fraction of equity risked per trade
|
||||
# --- entry mechanism (Ross-Cameron-style auto-arm; all tunable) ---
|
||||
TRADING_REWARD_MULTIPLE=2.0 # target = entry + this * (entry - stop); 2R default
|
||||
TRADING_ENTRY_ORDER_TYPE=limit # limit = rest at trigger (cancellable); market = fill now
|
||||
TRADING_ENTRY_TIMEOUT_BARS=2 # cancel an unfilled entry after N minutes resting (0 = off)
|
||||
TRADING_ENTRY_INVALIDATE_PCT=0.0 # cancel if price breaks this far below entry (0 = any break; <0 = off)
|
||||
TRADING_ENTRY_GUARD_INTERVAL_SECONDS=5 # how often to re-check armed entries vs the LIVE last-trade price
|
||||
TRADING_MAX_DAILY_LOSS_PCT=0.03 # daily-loss circuit breaker
|
||||
# Nightly pg_dump (db-backup compose service, 20:10 ET Mon-Fri) — the only
|
||||
# automated backup of the production Postgres. See deploy/docker/backup_db.sh.
|
||||
BACKUP_DIR=/data/backups # inside the db-backup container (app-data volume)
|
||||
BACKUP_KEEP_DAYS=14 # prune dumps older than this many days
|
||||
|
||||
#Market Data these are free plans so work within days/months capacity.
|
||||
Alpha_Vantage_Api_Key=<REDACTED>
|
||||
Twelve_Data_Api_Key=<REDACTED>
|
||||
Marketstack_Api_Key=<REDACTED>
|
||||
Financial_Modeling_Prep_Api_Key=<REDACTED>
|
||||
EOD_Historical_Data_Api_Key=<REDACTED>
|
||||
|
||||
#News Data these are free plans so work within days/months capacity.
|
||||
NewsAPI_Api_Key=<REDACTED>
|
||||
NewsData.io_Api_Key=<REDACTED>
|
||||
Marketaux_Api_Key=<REDACTED>
|
||||
MediaStack_Api_Key=<REDACTED>
|
||||
|
||||
#ngrok key
|
||||
ngrok_key=<REDACTED>
|
||||
|
||||
# Intraday watcher (signal engine)
|
||||
WATCHER_ENABLED=1
|
||||
WATCHER_INTERVAL_SECONDS=60
|
||||
# WATCHER_EVENT_DB_PATH is legacy and ignored — the datastore is PostgreSQL (see DATABASE_URL above)
|
||||
WATCHER_MAX_SYMBOLS=25
|
||||
WATCHER_MIN_BARS=30
|
||||
WATCHER_MIN_QUALITY=0.30
|
||||
# Comma-separated symbols to follow all day. If empty, candidates are
|
||||
# discovered from whatever has minute bars in market.duckdb for today.
|
||||
WATCHER_SYMBOLS=AAPL,TSLA,NVDA,AMD,SOFI
|
||||
# Price band for auto-discovered candidates (Ross Cameron default 1-20).
|
||||
WATCHER_PRICE_MIN=1
|
||||
WATCHER_PRICE_MAX=20
|
||||
# Active strategy setups (comma-separated). The backtest sweep found the
|
||||
# opening-range breakout is the only profitable one; empty = all setups.
|
||||
# WATCHER_MIN_BARS / WATCHER_READY_SCORE are left to the nightly tuner
|
||||
# (data/learned_params.json). Uncomment to pin a manual override.
|
||||
# WATCHER_MIN_BARS=5
|
||||
# --- faster loop cadence: react sooner once a setup forms (helps at the margin;
|
||||
# the real lateness is the setup design — see learn_params.py) ---
|
||||
LIVE_BARS_INTERVAL_SECONDS=20
|
||||
# Live ingest only re-pulls the last few minutes (closed bars never change); the
|
||||
# first pass still covers the whole session so the opening range is present.
|
||||
# This is what fixed the ~46s ingest that froze the single-threaded loop.
|
||||
LIVE_BARS_REFRESH_MINUTES=20
|
||||
WATCHER_INTERVAL_SECONDS=15
|
||||
DISCOVER_INTERVAL_SECONDS=120
|
||||
# Active setups. The backtest sweep found the opening-range breakout is the only
|
||||
# profitable one (others lose and fire later); empty = all setups. See learn_params.py.
|
||||
STRATEGY_SETUPS=opening_range_break
|
||||
|
||||
# Live minute-bar ingestion (feeds the watcher with active market data)
|
||||
LIVE_BARS_ENABLED=1
|
||||
LIVE_BARS_INTERVAL_SECONDS=60
|
||||
LIVE_BARS_LOOKBACK_MINUTES=120
|
||||
# Data feed: live realtime uses iex (free, realtime, but ~2-5% of true volume).
|
||||
# Backfill/backtest uses sip (full consolidated tape) so rvol/$-volume reflect
|
||||
# REAL volume — the iex thinness silently corrupts every volume-based decision.
|
||||
ALPACA_DATA_FEED=iex
|
||||
# --- news / catalyst ranking ------------------------------------------------
|
||||
# A fresh catalyst (PR/earnings/FDA/offering) is WHY a small-cap gaps and runs.
|
||||
# These RSS feeds are ingested each discover pass; tickers with recent news are
|
||||
# (1) folded into the watch universe (caught earlier) and (2) boosted up the
|
||||
# armed-trigger ranking. Format: name=url,name=url. Empty = news disabled.
|
||||
NEWS_FEEDS=prnewswire=https://www.prnewswire.com/rss/news-releases-list.rss,businesswire=https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRWQ==,globenewswire=https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies
|
||||
NEWS_RANK_BOOST=2.0 # multiply a catalyst-backed gapper's rank score
|
||||
NEWS_LOOKBACK_HOURS=8 # how fresh the catalyst must be
|
||||
# Manual per-symbol priority nudge (a SMALL boost to names you want watched).
|
||||
# Keep modest (1.1-1.3) so it doesn't override the gap/volume logic. Empty = off.
|
||||
# e.g. SYMBOL_WEIGHTS=SOFI:1.2,RIVN:1.15
|
||||
SYMBOL_WEIGHTS=
|
||||
|
||||
# --- fast armed-trigger path (runtime/triggers.py) ---------------------------
|
||||
# Keep the N most-promising gappers "armed" with a pre-computed opening-range
|
||||
# trigger/stop and FIRE the instant live price crosses (a marketable order that
|
||||
# actually fills on a runner) — instead of waiting for a 1-min bar to close and
|
||||
# resting a limit that never fills. This is what catches the move in time.
|
||||
TRIGGER_ENABLED=1
|
||||
TRIGGER_MAX_ARMED=10 # how many gappers to keep ready at once (batched price fetch scales fine)
|
||||
TRIGGER_INTERVAL_SECONDS=4 # how often to check live prices / fire
|
||||
TRIGGER_ARM_INTERVAL_SECONDS=20 # how often to re-rank + recompute triggers
|
||||
TRIGGER_GAP_MIN=10.0 # aligned with TRADING_MIN_ENTRY_GAP_PCT (population gate) 2026-07-03
|
||||
TRIGGER_RVOL_MIN=2.0 # min relative volume to be eligible to fire
|
||||
# Entry-quality filters (diagnostic + out-of-sample verified, verify-entry-filters):
|
||||
# capping price at $5 and skipping blow-off gaps (>60%) ~DOUBLED held-out
|
||||
# expectancy (+0.76 vs +0.39R, win 86% vs 71%). $10-20 stocks were net losers;
|
||||
# >150% gaps fade. Fewer, much higher-quality trades.
|
||||
TRIGGER_GAP_MAX=35 # 35-150% band is dead (+0.02R year-scale) — cap earns its keep there
|
||||
# mega-gapper exception (2026-07-06, the LHSW/NXTS fix): re-admit REAL >150% squeezes.
|
||||
# Year replay n=2,879 w/ split artifacts removed: >150% & rvol>=5 ran +0.63R, 8/14 months >=0.
|
||||
TRIGGER_GAP_SQUEEZE_MIN=150
|
||||
TRIGGER_GAP_SQUEEZE_RVOL=5.0
|
||||
WATCHER_PRICE_MIN=2 # raised $1->$2 2026-07-09: $1-2 bucket -0.031R over 609 unbiased entries
|
||||
TRADING_PRICE_MIN=2.0
|
||||
TRADING_PRICE_MAX=20.0
|
||||
WATCHER_PRICE_MAX=20 # RESTORED to the original $1-20 (2026-06-22). Was narrowed to 5 by a prior backtest ("$10-20 net losers") that the LIVE record contradicts + is n=2 noise; and $5 is why the bot missed NXTS ($9.21, today's +250% winner). Downside is now $500/trade-capped so the wider net is a bounded experiment — measure by price band on the clean sample before re-narrowing.
|
||||
ORB_BARS=5 # opening range = first N regular-hours minutes
|
||||
TRADING_TRIGGER_SLIP_PCT=0.004 # cap chase above the trigger on the fill
|
||||
WATCHER_STATIC_SYMBOLS=TSLA,NVDA,AMD,MSFT,AAPL
|
||||
ACCOUNT_SYNC_ENABLED=0
|
||||
ORDER_SYNC_ENABLED=0
|
||||
# --- LIVE PAPER TRADING (enabled for market open) ---
|
||||
ALPACA_PAPER_SYNC_ENABLED=1
|
||||
TRADING_EXECUTION_ENABLED=1
|
||||
TRADING_AUTO_APPROVE=1 # 1 = auto-arm bracket orders; 0 = approve each in the dashboard
|
||||
TRADING_MAX_DAILY_LOSS_PCT=0.05 # RE-ARMED 2026-06-22 (was 0.50 testing): circuit breaker halts entries + flattens at -5% equity (~-$4.5k on $90k). Caps a runaway day; per-trade risk is already $500-capped so ~all-stop-out ≈ -5%. Original design was 0.03 — tighten if you want a harder stop.
|
||||
TRADING_RISK_PER_TRADE_PCT=0.01 # risk 1% of equity per trade
|
||||
TRADING_MAX_RISK_DOLLARS=500 # HARD dollar-risk ceiling/trade: risk = min(1% equity, $500). On the ~$90k paper acct this caps the wide-stop ~$900-1k losers from the loss diagnosis at $500. 0=off (small live acct uses the % only)
|
||||
TRADING_MAX_FRESH_ENTRIES_PER_DAY=10 # EXPERIMENT 2026-07-02 (owner-authorized): raised 5->10 real FILLED entries/day (behavior fix counts fills not submits). Data says all-day/more-trades is -EV (afternoon -0.94R); paper A/B only. C+ quality gate + re-entry-on-winners still in force.
|
||||
TRADING_MAX_CONCURRENT_POSITIONS=10 # raised 6->10; the 60% gross-notional cap below still hard-bounds TOTAL exposure (so this = more, smaller positions, not more risk)
|
||||
TRADING_REQUIRE_ABOVE_VWAP=1 # VWAP entry gate: only take breakouts trading >= session VWAP. The ONE validated selection signal (above-VWAP reaches +1R ~1.5x as often, n=3,110 from the labeler). 0=shadow-only (logs below-VWAP signals as rule_type=vwap_below but still enters). Set 0 to revert to pre-gate behavior.
|
||||
TRADING_UNIFIED_ENTRY=1 # run the fast path's anti-chase gates (day-extension: don't enter a name whose breakout level is >30% above the session open; halt) on the LIVE auto path too. Closes the gap where those guards only ran on the rarely-used fast trigger path. Fail-open on missing data. 0=pre-unification (auto path skips these gates).
|
||||
TRADING_GATE_RECEIPTS=1 # emit one entry_gate_evaluated event per gate evaluation (both paths) with the full pass/reject gate map — turns 'why didn't it trade X?' into a query. High-volume diagnostic, purged after 14 days like other observability spam. 0=off (no receipts).
|
||||
TRADING_ENTRY_FILL_MODEL=resting # auto-path fill: 'resting' = rest a limit AT the breakout level (fills only on a pullback, misses runners that gap up). 'marketable' = limit a hair above the trigger so it fills as price breaks UP through it (catches runners, the fast path's behavior; the limit still caps the fill so no gap-chase). VALIDATED LIVE 2026-06-24 open (NOK/SOFI/SSPC filled near-trigger, no chase) then reverted to resting (safe default). Flip to 'marketable' to use it.
|
||||
# Account-aware sizing: positions size off REAL broker equity (auto — so a $300
|
||||
# live account sizes for $300, not the $100k default). One position's value is
|
||||
# capped to this fraction of equity so N positions fit buying power (~1/N).
|
||||
# 0.16 so 6 positions fit (6 x 16% ~ full account) — esp. the $300 cash account.
|
||||
TRADING_MAX_POSITION_PCT=0.16
|
||||
# Liquidity cap (shares <= this % of the symbol's day volume) — OFF at small
|
||||
# size; enable (e.g. 0.005) only when scaling to real money to avoid slippage.
|
||||
TRADING_LIQUIDITY_MAX_VOLUME_PCT=0.025
|
||||
# End-of-day auto-flatten: close everything ~5 min before the bell so the book
|
||||
# never carries overnight gap risk (a day-trading strategy shouldn't hold).
|
||||
TRADING_EOD_FLATTEN_ENABLED=1
|
||||
TRADING_EOD_FLATTEN_TIME=15:55
|
||||
# The process is one trading session: after a SUCCESSFUL EOD flatten the loop
|
||||
# exits 0 and compose restarts it fresh, so memory-only daily state (entry cap,
|
||||
# benches, session-closed flags) can never go stale across a date rollover.
|
||||
TRADING_EXIT_AFTER_EOD_FLATTEN=1
|
||||
TRADING_MAX_ORDERS_PER_TICK=1
|
||||
TRADING_ENTRY_ORDER_TYPE=limit # rest at the breakout trigger
|
||||
# --- managed exit (swept + out-of-sample verified: trailLow_a1) -------------
|
||||
# The exit sweep over 187 sessions with a held-out test split found that
|
||||
# trailing the stop under the prior bar's low (after +1R) nearly DOUBLES the
|
||||
# out-of-sample expectancy vs the old fixed +2R target (+0.128R vs +0.067R,
|
||||
# win 49% vs 38%) and is robust, not overfit. So: a far backstop target and let
|
||||
# the trailing stop do the work. See sweep_exits.py / strategy/exits.py.
|
||||
TRADING_REWARD_MULTIPLE=10.0 # far take-profit backstop; the trail is the real exit
|
||||
TRADING_EXIT_TRAIL_MODE=prior_low # trail the stop under each prior bar's low
|
||||
TRADING_EXIT_TRAIL_AFTER_R=1.0 # start trailing once the trade is +1R in profit
|
||||
# Profit-lock checkpoints (percentage of ENTRY): once a trade has been up gain%,
|
||||
# the stop is pinned so it can't close for less than lock% — you always keep a
|
||||
# minimum gain once it's working, and a fade back down sells you out in profit.
|
||||
# Format: gain:lock,gain:lock (whole %). Tunable; raise/lower to taste.
|
||||
TRADING_EXIT_PROFIT_TIERS=8:3,15:9,25:18,40:30
|
||||
TRADING_EXIT_BREAKEVEN_R=1.0 # ON: lock breakeven once +1R (forward safeguard)
|
||||
TRADING_EXIT_TRAIL_R_STEP=0.25 # 0.25R STEP-TRAIL from the buy price (2026-06-24, supersedes a standalone +2% breakeven): once +0.25R is reached the stop moves to entry (breakeven), +0.5R->+0.25R, +0.75R->+0.5R, ... It locks a small green pop from round-tripping to -1R (the SOFI +$281->-$118 case) AND keeps climbing under a runner. Coexists with the prior-low trail (stop = max of all). Tradeoff: scratches some winners at breakeven on noise (sim ~+0.05R net). Widen to 0.5 if too tight.
|
||||
# TRADING_EXIT_SCALE_OUT_R / _PCT # (off) scaling didn't help out-of-sample
|
||||
# TRADING_EXIT_FIRST_RED=0 # (off) first-red exits were NEGATIVE expectancy here
|
||||
# How far price may dip below the entry trigger before we CANCEL an unfilled
|
||||
# entry. 0.0 (the old default) cancels on any sub-cent wobble -> a breakout that
|
||||
# oscillates around the level gets cancelled before it can fill = pure churn,
|
||||
# never holds a trade. Give it room; the bracket STOP (at the opening-range low)
|
||||
# is the real protection, and risk is capped at 1% per trade by sizing.
|
||||
TRADING_ENTRY_INVALIDATE_PCT=0.015
|
||||
TRADING_ENTRY_TIMEOUT_BARS=2 # cancel an entry still unfilled after ~2 min
|
||||
|
||||
# ---- Database (PostgreSQL) — single datastore via local port-proxy to the LAN instance
|
||||
DATABASE_URL=<REDACTED> # direct to the LAN host; storage/db_pg.py auto-falls-back between 192.168.1.5 and the 127.0.0.1 netsh tunnel (whichever the WSL net can reach — it flips across reboots).
|
||||
|
||||
# anti-thrash: bench a symbol after it backs out (unfilled entry)
|
||||
TRADING_BACKOUT_COOLDOWN_SECONDS=180
|
||||
TRADING_MAX_BACKOUTS_PER_SYMBOL=3
|
||||
|
||||
# one round-trip per symbol per day: no re-entry after a position closes
|
||||
TRADING_BLOCK_REENTRY_AFTER_EXIT=1
|
||||
|
||||
# liquidity gate: skip breakouts on thin names (recent 5-bar $-volume floor)
|
||||
TRIGGER_MIN_DOLLAR_VOL=8000
|
||||
|
||||
# only bench re-entry after a real losing exit (>1%), not a scratch/win
|
||||
TRADING_REENTRY_MIN_LOSS_PCT=0.01
|
||||
TRADING_MAX_GROSS_NOTIONAL_PCT=0.60
|
||||
ALPACA_BACKFILL_FEED=sip
|
||||
# Freshness watchdog: warn (emit + display) when the newest bar is older than
|
||||
# this during RTH. INGEST_STALE_BLOCK=1 also refuses to ARM on stale data
|
||||
# (off by default so a mis-tuned threshold can't silently halt trading).
|
||||
INGEST_STALE_SECONDS=180
|
||||
INGEST_STALE_BLOCK=0
|
||||
|
||||
# Broker state sync
|
||||
ALPACA_PAPER_SYNC_ENABLED=1
|
||||
ALPACA_PAPER_SYNC_INTERVAL_SECONDS=60
|
||||
ACCOUNT_SYNC_ENABLED=0 # Schwab live account sync
|
||||
ORDER_SYNC_ENABLED=0 # Schwab live order sync
|
||||
|
||||
# Dashboard
|
||||
DASHBOARD_HOST=0.0.0.0
|
||||
DASHBOARD_PORT=8010
|
||||
|
||||
# ============================================================
|
||||
# Armed-trigger fast path (runtime/triggers.py) + entry quality
|
||||
# ============================================================
|
||||
TRIGGER_ENABLED=1
|
||||
TRIGGER_MAX_ARMED=10 # gappers kept ready to fire at once
|
||||
TRIGGER_INTERVAL_SECONDS=4 # fast path: live-price check / fire cadence
|
||||
TRIGGER_ARM_INTERVAL_SECONDS=20 # re-rank + recompute triggers cadence
|
||||
TRIGGER_GAP_MIN=3.0 # min overnight gap %
|
||||
# Cap the gap: blow-off gappers (>35%) fade — they're the worst-expectancy
|
||||
# bucket. Train/test validated on 14 SIP-backfilled sessions: lowering 60->35
|
||||
# lifted held-out TEST expectancy 0.19R -> 0.46R/trade AND raised total R.
|
||||
TRIGGER_GAP_MAX=35
|
||||
TRIGGER_RVOL_MIN=2.0 # min RELATIVE volume to fire
|
||||
# Absolute liquidity gate: rvol is relative, so 2x of near-nothing still
|
||||
# qualifies and we fill at the top of a thin one-bar spike that reverses. This
|
||||
# requires recent 5-bar $-volume >= the floor so thin names go WEAK, not armed.
|
||||
TRIGGER_MIN_DOLLAR_VOL=8000
|
||||
# Do not fire a trigger on a price older than this (seconds) — a stalled feed
|
||||
# must not fire a breakout on a frozen quote into an already-reverted market.
|
||||
TRIGGER_PRICE_MAX_AGE=12
|
||||
ORB_BARS=5 # opening range = first N regular-hours minutes
|
||||
TRADING_TRIGGER_SLIP_PCT=0.004 # cap the chase above the trigger on the fill
|
||||
# Account-aware sizing (matters most on a small REAL account):
|
||||
TRADING_MAX_POSITION_PCT=0.16 # one position's $ value <= this fraction of equity
|
||||
# PORTFOLIO gross-notional cap: total $ across ALL open+pending positions stays
|
||||
# under this fraction of equity. Without it, 6 concurrent x 16% = ~96% gross in
|
||||
# correlated small-cap gappers. With it, later entries shrink/blocked to fit.
|
||||
TRADING_MAX_GROSS_NOTIONAL_PCT=0.60
|
||||
# Liquidity-aware sizing: shares <= this fraction of the name's volume, so a
|
||||
# thin name is sized DOWN automatically instead of taking a full position it
|
||||
# can't absorb. 0 = off (fine at tiny size).
|
||||
TRADING_LIQUIDITY_MAX_VOLUME_PCT=0.025
|
||||
|
||||
# ============================================================
|
||||
# Risk discipline (added 2026-06-18 — see git history for the why)
|
||||
# Local LLM news/catalyst enrichment (Ollama) — TEST SPOT
|
||||
# Detected running at the Windows host (192.168.1.9:11434).
|
||||
# Phase-1 advisory only; trading behaviour does NOT change. If Ollama is down or
|
||||
# the model is missing it degrades to a no-op on its own thread (never impedes).
|
||||
# ============================================================
|
||||
# Anti-thrash: a breakout that won't fill on a marketable limit used to re-arm
|
||||
# every pass forever (observed: NEOV armed+backed-out 68x/session). Bench it.
|
||||
TRADING_BACKOUT_COOLDOWN_SECONDS=180 # bench this long after an unfilled backout
|
||||
TRADING_MAX_BACKOUTS_PER_SYMBOL=3 # after N backouts, bench for the session
|
||||
# Re-entry discipline: don't buy back a name that already round-tripped today...
|
||||
TRADING_BLOCK_REENTRY_AFTER_EXIT=1
|
||||
# ...but only if it exited at a real LOSS (>1%), so a scratch/win can re-enter
|
||||
# (the blunt version killed a +513 re-entry winner in replay). 0 = bench any close.
|
||||
TRADING_REENTRY_MIN_LOSS_PCT=0.01
|
||||
# End-of-day flatten — a day-trading book shouldn't carry overnight gap risk.
|
||||
TRADING_EOD_FLATTEN_ENABLED=1
|
||||
TRADING_EOD_FLATTEN_TIME=15:55 # flatten in the window [time, 16:00)
|
||||
OLLAMA_ENABLED=1
|
||||
OLLAMA_HOST=http://192.168.1.5:30068 # direct to host (WSL tunnel flips; client also falls back)
|
||||
OLLAMA_MODEL=qwen2.5:7b-instruct
|
||||
OLLAMA_TIMEOUT_SECONDS=60
|
||||
NEWS_ENRICH_ENABLED=1
|
||||
NEWS_ENRICH_INTERVAL_SECONDS=120
|
||||
NEWS_ENRICH_LOOKBACK_HOURS=12
|
||||
NEWS_ENRICH_BATCH_LIMIT=50
|
||||
# Phase-2 automation stays OFF until you've watched a clean advisory run:
|
||||
NEWS_CATALYST_SCORE_ENABLED=1
|
||||
NEWS_DILUTION_VETO_ENABLED=1
|
||||
OLLAMA_TEMPERATURE=0.1
|
||||
|
||||
# ============================================================
|
||||
# Managed exits (strategy/exits.py — swept + out-of-sample verified)
|
||||
# The far take-profit is a backstop; the TRAILING STOP is the real exit.
|
||||
# ============================================================
|
||||
TRADING_REWARD_MULTIPLE=10.0 # far take-profit backstop (NOT 2.0; trail does the work)
|
||||
TRADING_EXIT_TRAIL_MODE=prior_low # trail the stop under each prior bar's low
|
||||
TRADING_EXIT_TRAIL_AFTER_R=1.0 # start trailing once the trade is +1R
|
||||
TRADING_EXIT_BREAKEVEN_R=1.0 # lock breakeven once +1R (give-back safeguard)
|
||||
TRADING_EXIT_PROFIT_TIERS=8:3,15:9,25:18,40:30 # +gain% -> lock% profit-lock ladder
|
||||
TRADING_MIN_ENTRY_GAP_PCT=0.10 # POPULATION gate (year-long +EV boundary, n=3008/14mo): only real gappers (open vs prev close >= 10%) are tradeable; all rank x timing segments +0.05..+0.15R under the BE@+5% ratchet
|
||||
TRADING_MIN_DAY_VOLUME=300000 # population volume floor (shares, consolidated SIP): thin names whipsaw; fail-open on missing data
|
||||
TRADING_ENTRY_CONFIRM_BY_MINUTE=0 # OFF 2026-07-03: year-long replay (n=3008, 14mo) REFUTED the 15-min gate — late crosses beat early 12/14 months; population is the real +EV boundary (see below)
|
||||
TRADING_MIN_QUALITY_SCORE=0.60 # C or higher (owner 2026-07-07 after F-tier experiment; grade map: A>=.9 B>=.75 C>=.6 D>=.4). Old 0.50 was mislabeled C+ — actually mid-D
|
||||
|
||||
# NOTE: several defaults above intentionally override the simpler values earlier
|
||||
# in this file (e.g. MAX_CONCURRENT_POSITIONS, REWARD_MULTIPLE) — these reflect
|
||||
# the current live-verified config. Set MAX_CONCURRENT_POSITIONS=6 for the live book.
|
||||
TRADING_MAX_CONCURRENT_POSITIONS=6
|
||||
# !!! Before REAL-money trading, set TRADING_MAX_DAILY_LOSS_PCT back to 0.03 !!!
|
||||
# YouTube claim-extraction pipeline
|
||||
YOUTUBE_API_KEY=<REDACTED>
|
||||
YT_WATCHLIST=
|
||||
ALPACA_DATA_FEED=sip # Algo Trader Plus (paid 2026-07-02, entitlement verified 07-03): real-time consolidated tape. Flip back to iex if the plan lapses — the ratio estimator self-reactivates.
|
||||
@@ -0,0 +1,83 @@
|
||||
name: docker-image
|
||||
|
||||
# Build + push the app image to GHCR on every master push, so pull-only
|
||||
# hosts (TrueNAS SCALE custom apps) can deploy without a repo checkout.
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- "Dockerfile"
|
||||
- "requirements.txt"
|
||||
- "**.py"
|
||||
- "api/static/**"
|
||||
- ".github/workflows/docker.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
# Gate: no image reaches a registry unless the suite is green. Production
|
||||
# (TrueNAS) pulls :latest blindly, so this job is the only test run between
|
||||
# a master push and the live loop.
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18 # keep matched to the compose files' server image
|
||||
env:
|
||||
POSTGRES_USER: momentum
|
||||
POSTGRES_PASSWORD: momentum
|
||||
POSTGRES_DB: momentum
|
||||
ports: ["5432:5432"]
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U momentum" --health-interval 5s
|
||||
--health-timeout 5s --health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with: { python-version: "3.13" } # match Dockerfile's python:3.13-slim
|
||||
- run: pip install -r requirements.txt ruff
|
||||
# ruff F gate, scoped to the money path (tests/ and root-level scratch
|
||||
# scripts excluded). All per-file pins have been cleared as their owning
|
||||
# plans landed (api/main.py F821 os/hmac -> plan 007), so this is unpinned.
|
||||
- run: >-
|
||||
ruff check --select F
|
||||
run_live_paper.py trading_execution.py telegram_bot.py
|
||||
api/ alpaca_paper/ runtime/ storage/ strategy/ research/
|
||||
- run: python -m pytest -q
|
||||
env:
|
||||
DATABASE_URL: postgresql://momentum:momentum@localhost:5432/momentum
|
||||
# tests were written under an ET wall clock (the app image pins it too)
|
||||
TZ: America/New_York
|
||||
|
||||
build:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- id: ver
|
||||
run: echo "tag=$(date -u +%Y.%m.%d)-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
- uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
build-args: |
|
||||
GIT_SHA=${{ github.sha }}
|
||||
BUILD_TAG=${{ steps.ver.outputs.tag }}
|
||||
tags: |
|
||||
ghcr.io/fez5587/momentum-app:latest
|
||||
ghcr.io/fez5587/momentum-app:${{ github.sha }}
|
||||
ghcr.io/fez5587/momentum-app:${{ steps.ver.outputs.tag }}
|
||||
fez5587/momentum-app:latest
|
||||
fez5587/momentum-app:${{ github.sha }}
|
||||
fez5587/momentum-app:${{ steps.ver.outputs.tag }}
|
||||
@@ -58,3 +58,7 @@ momentum-rebuilt.zip
|
||||
*.zip
|
||||
archive/
|
||||
uv.lock
|
||||
.firecrawl/
|
||||
|
||||
# large research PDFs (local only)
|
||||
books/
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
# Momentum System — container image
|
||||
#
|
||||
# Single image for every run mode (the app is a monolith run from the repo root
|
||||
# with PYTHONPATH=/app, not a pip-installed wheel — see pyproject.toml
|
||||
# `package = false`). The Kubernetes workloads override CMD:
|
||||
# - app Deployment : python run_live_paper.py (loop + embedded dashboard)
|
||||
# - CronJobs : python eod_replay.py / nightly_tune.py / momentum_cli.py journal
|
||||
FROM python:3.13-slim
|
||||
|
||||
# psycopg2-binary ships its own libpq, so no build toolchain is needed; we only
|
||||
# add a tiny client for healthchecks/debugging and clean up the apt cache.
|
||||
# ffmpeg: yt-dlp audio extraction for the transcripts pipeline; curl: healthchecks
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libpq5 ffmpeg curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Image identity: CI passes the git sha + a date tag so the RUNNING app can
|
||||
# report exactly what it is (TrueNAS custom apps show only 'custom/1.0.0').
|
||||
ARG GIT_SHA=dev
|
||||
ARG BUILD_TAG=dev
|
||||
ENV APP_GIT_SHA=$GIT_SHA \
|
||||
APP_BUILD_TAG=$BUILD_TAG
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONPATH=/app \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
TZ=America/New_York
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install dependencies first so they cache across code-only changes.
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt \
|
||||
# transcripts pipeline (CPU whisper); heavy layer, cached separately
|
||||
&& pip install --no-cache-dir faster-whisper yt-dlp
|
||||
|
||||
# Application code.
|
||||
COPY . .
|
||||
|
||||
# Run as a non-root user that owns the writable data dir (DuckDB research file,
|
||||
# logs, learned_params.json, schwab tokens). In K8s this path is a mounted PVC.
|
||||
RUN useradd --create-home --uid 10001 app \
|
||||
&& mkdir -p /app/data \
|
||||
&& chown -R app:app /app/data
|
||||
USER app
|
||||
|
||||
EXPOSE 8765
|
||||
|
||||
# Default to the headless loop; the chart's app Deployment runs it WITH the
|
||||
# dashboard, and the CronJobs override this entirely.
|
||||
CMD ["python", "run_live_paper.py", "--no-dashboard"]
|
||||
@@ -1,15 +1,17 @@
|
||||
# Momentum
|
||||
|
||||
Event-sourced small-cap **momentum day-trading** system (Ross Cameron–style
|
||||
gap-and-go / opening-range breakout). It screens the day's most-active gappers
|
||||
($1–5, gapping ≥3% on ≥2× relative volume), arms an opening-range-breakout
|
||||
trigger the moment the range forms, fires a bracket order on the live cross,
|
||||
and manages the exit with a trailing stop + profit-lock ladder. Execution runs
|
||||
on an **Alpaca paper** account; everything is recorded in a **PostgreSQL**
|
||||
append-only event store and replayable.
|
||||
gap-and-go / opening-range breakout). It screens the day's top overnight
|
||||
gappers — the year-scale-validated population: **$2–20, static open gap
|
||||
10–35%, real volume**, plus a volume-verified **>150% squeeze exception** —
|
||||
arms an opening-range-breakout trigger the moment the range forms, fires a
|
||||
bracket order on the live cross (regular session only), and manages the exit
|
||||
with a breakeven ratchet + trailing profit-lock ladder. Execution runs on an
|
||||
**Alpaca paper** account over the **SIP consolidated feed**; everything is
|
||||
recorded in a **PostgreSQL** append-only event store and replayable.
|
||||
|
||||
```
|
||||
Alpaca IEX bars ─► research DBs (minute+daily) ─► screen gappers ─► ArmedTriggerBook
|
||||
Alpaca SIP bars ─► research DBs (minute+daily) ─► screen gappers ─► ArmedTriggerBook
|
||||
(1-min, live) │ live price
|
||||
▼ crosses trigger
|
||||
PostgreSQL event store ◄────────────────────── TradingExecutionService ─► Alpaca paper
|
||||
@@ -36,6 +38,15 @@ python run_live_paper.py # full loop (+ embedded dashboard)
|
||||
|
||||
No keys? `python run_live_paper.py --once` runs a single dry pass to verify wiring.
|
||||
|
||||
## Deployment
|
||||
|
||||
Three supported shapes — **Docker Compose is canonical** (production runs the
|
||||
full stack self-contained on TrueNAS SCALE: postgres, loop, dashboard :8765,
|
||||
admin panel :8767, telegram bot, transcripts, nightly scheduler). Local
|
||||
systemd units remain the development shape; a Helm refresh is planned.
|
||||
**See [deploy/README.md](deploy/README.md)** for all three, including the
|
||||
behavior flags every deployment must set explicitly.
|
||||
|
||||
## Commands
|
||||
|
||||
Everything is driven from the repo root. The `./momentum` wrapper runs the
|
||||
@@ -99,8 +110,8 @@ The `.sh` wrappers set the venv + cwd so cron can call them directly:
|
||||
|
||||
```bash
|
||||
DATABASE_URL=postgresql://user:pass@host:5432/momentum python -m pytest
|
||||
# ~130 tests: strategy, storage, triggers, execution (cooldown/re-entry/EOD
|
||||
# flatten/naked-stop guards), exits, journal P&L, schwab, api, research.
|
||||
# ~365 tests: strategy, storage, triggers, execution (cooldown/re-entry/EOD
|
||||
# flatten/naked-stop guards), exits, journal P&L, api, research.
|
||||
```
|
||||
|
||||
Tests isolate via throwaway `mem_*` Postgres schemas; `tests/conftest.py` sweeps
|
||||
@@ -111,8 +122,11 @@ them at session end.
|
||||
All behavior is environment-driven — see **`.env.example`** for every key with
|
||||
the *why*. The knobs that matter most:
|
||||
|
||||
- **Entry:** `WATCHER_PRICE_MIN/MAX` (1–5), `TRIGGER_GAP_MIN/MAX`, `TRIGGER_RVOL_MIN`,
|
||||
`TRIGGER_MIN_DOLLAR_VOL` (absolute liquidity floor — skip thin spikes).
|
||||
- **Entry population** (year-scale validated 2026-07): `TRADING_PRICE_MIN/MAX`
|
||||
(2–20), `TRADING_MIN_ENTRY_GAP_PCT` (0.10, static open gap), `TRIGGER_GAP_MAX`
|
||||
(35 — the 35-150% band tested dead), `TRIGGER_GAP_SQUEEZE_MIN/RVOL` (150/5 —
|
||||
re-admits real squeezes), `TRADING_MIN_DAY_VOLUME`, `TRADING_SESSION_ONLY`
|
||||
(bars after-hours entries).
|
||||
- **Sizing:** `TRADING_RISK_PER_TRADE_PCT` (1%), `TRADING_MAX_POSITION_PCT`,
|
||||
`TRADING_LIQUIDITY_MAX_VOLUME_PCT` (size down thin names).
|
||||
- **Exits:** `TRADING_EXIT_TRAIL_MODE=prior_low`, `TRADING_EXIT_TRAIL_AFTER_R`,
|
||||
@@ -124,8 +138,9 @@ the *why*. The knobs that matter most:
|
||||
|
||||
## Database & performance
|
||||
|
||||
The single datastore is **PostgreSQL** (`DATABASE_URL`); research bars live in
|
||||
DuckDB under `data/research/`. The event store is one append-only `events`
|
||||
The single datastore is **PostgreSQL** (`DATABASE_URL`) — bars, events,
|
||||
research tables, everything (the DuckDB era is over; `data/` holds only logs,
|
||||
transcripts, and boot-time state). The event store is one append-only `events`
|
||||
table that every projection reads.
|
||||
|
||||
**Hot-path index.** Projections overwhelmingly run
|
||||
@@ -149,10 +164,10 @@ For ongoing tuning:
|
||||
+ time partitioning + compression). Not required at current scale, and must be
|
||||
installed on the server.
|
||||
|
||||
**Maintenance.** ~80% of `events` rows are high-volume diagnostics
|
||||
(`criteria_evaluated`, `signal_blocked`, `module_tick`). A retention/prune of old
|
||||
diagnostic events keeps the table small and every query fast. Run `ANALYZE events`
|
||||
after large ingests so the planner has fresh stats.
|
||||
**Maintenance is automated.** The nightly job (scheduler service / systemd
|
||||
timer) prunes diagnostic events older than 14 days, rotates logs, refreshes the
|
||||
day's bars to SIP truth, and updates the float/dilution shadow caches.
|
||||
Autovacuum is tuned per-table; run `ANALYZE` manually only after bulk imports.
|
||||
|
||||
## Repository map
|
||||
|
||||
@@ -165,15 +180,19 @@ strategy/ pure strategy engine: evaluation, structure, exits, backte
|
||||
storage/ event schema + store (PostgreSQL), projections
|
||||
research/ research DBs, ingestion (bars, RSS, gappers), screeners
|
||||
alpaca_paper/ Alpaca paper client, executor, account/order sync
|
||||
schwab/ Schwab OAuth, market data, positions/orders, health
|
||||
api/ + dashboard_api.py dashboard JSON API + static UI
|
||||
api/ + dashboard_api.py dashboard JSON API + static UI (:8765)
|
||||
admin_api.py admin panel (:8767): health, config overrides, profiles, restarts
|
||||
youtube_claims/ trader-video transcript -> claim extraction pipeline
|
||||
controlroom/ NiceGUI control room (disabled by owner preference)
|
||||
deploy/ compose (canonical) / systemd (local) / helm (planned)
|
||||
eod_replay.py end-of-day whole-market replay
|
||||
docs/history/ point-in-time plans from earlier eras — don't follow them
|
||||
tests/ pytest suite (unit + integration)
|
||||
```
|
||||
|
||||
## Security
|
||||
|
||||
`.env` and all of `data/` (DBs, `data/schwab_tokens.json`) are git-ignored and
|
||||
`.env` and all of `data/` are git-ignored and
|
||||
never committed; token files are written `0600`. The `.env` holds **live broker /
|
||||
database / webhook credentials** — keep it local. If any secret was ever
|
||||
committed to a repo's history, **rotate it** (re-issue Alpaca keys / re-run the
|
||||
|
||||
+49
-209
@@ -1,227 +1,67 @@
|
||||
# RUNBOOK
|
||||
|
||||
Operational guide for running the momentum system. This file replaces the
|
||||
original MILESTONE*.md documents, which were unrecoverable during the
|
||||
rebuild.
|
||||
Operating the Momentum stack day to day. Deployment shapes and setup live in
|
||||
[deploy/README.md](deploy/README.md); this is "it's running — now what."
|
||||
|
||||
---
|
||||
## Daily rhythm (all times ET)
|
||||
|
||||
## 1. Why live paper trading never worked (root causes, now fixed)
|
||||
|
||||
1. **Minute bars were never ingested.** The watcher reads 1-minute bars from
|
||||
`data/research/market.duckdb`, but the old ingestion only wrote *daily*
|
||||
bars for a handful of mega-caps. Fix: `research/ingestion/market_data.py`
|
||||
ingests 1-minute Alpaca IEX bars continuously, and
|
||||
`run_live_paper.py` schedules it every `LIVE_BARS_INTERVAL_SECONDS`.
|
||||
|
||||
2. **Session flags were hardcoded.** Every old row had
|
||||
`is_regular_hours = TRUE`. Fix: flags are derived from the bar timestamp
|
||||
in US/Eastern (premarket 04:00–09:30, regular 09:30–16:00, afterhours
|
||||
16:00–20:00) — see `classify_session()` and its tests.
|
||||
|
||||
3. **The candidate filter excluded everything ingested.** The query had a
|
||||
hardcoded `close BETWEEN 1 AND 20` while only mega-caps (price ≫ 20) were
|
||||
in the database, so the watchlist was always empty. Fix: the band is
|
||||
configurable (`WATCHER_PRICE_MIN` / `WATCHER_PRICE_MAX`) and the default
|
||||
symbol list now actually gets ingested.
|
||||
|
||||
4. **No orchestrator.** Ingestion, watcher, broker sync, and execution were
|
||||
never wired into one loop. Fix: `run_live_paper.py`.
|
||||
|
||||
5. **Broken source files.** `runtime/providers/registry.py`,
|
||||
`schwab/market/models.py`, `option_chain_service.py`, and
|
||||
`schwab/streaming/client.py` had syntax errors that crashed imports.
|
||||
All repaired/rewritten.
|
||||
|
||||
6. **Schwab tokens were committed to git.** `data/schwab_tokens.json` is now
|
||||
git-ignored and written `chmod 0600`. **Action required:** if that file
|
||||
exists anywhere in your repo history, revoke the refresh token in the
|
||||
Schwab developer portal and re-run the OAuth flow. Scrub history with
|
||||
`git filter-repo --path data/schwab_tokens.json --invert-paths` if the
|
||||
repo was ever shared.
|
||||
|
||||
---
|
||||
|
||||
## 2. Environment setup
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Minimum for paper trading: `ALPACA_API_KEY`, `ALPACA_SECRET_KEY`
|
||||
(free keys are fine — the system uses the IEX feed). Everything else has a
|
||||
sensible default; see `.env.example` for the full catalog.
|
||||
|
||||
Key knobs:
|
||||
|
||||
| Variable | Default | Meaning |
|
||||
| When | What happens | Verify |
|
||||
|---|---|---|
|
||||
| `TRADING_EXECUTION_MODE` | `alpaca_paper` | `schwab_live` routes to real money — leave on paper |
|
||||
| `TRADING_AUTO_APPROVE` | `0` | `1` = orders submit without dashboard approval (or pass `--auto-approve`) |
|
||||
| `TRADING_MAX_ORDERS_PER_TICK` | `1` | throttle new entries |
|
||||
| `TRADING_MAX_CONCURRENT_POSITIONS` | `3` | cap on simultaneous open positions |
|
||||
| `TRADING_RISK_PER_TRADE_PCT` | `0.01` | 1% of equity risked per trade (sizing = risk / (entry − stop)) |
|
||||
| `TRADING_REWARD_MULTIPLE` | `2.0` | profit target = entry + this × risk (2R by default) |
|
||||
| `TRADING_ENTRY_ORDER_TYPE` | `limit` | `limit` rests at the trigger so an unfilled entry can be cancelled; `market` fills instantly |
|
||||
| `TRADING_ENTRY_TIMEOUT_BARS` | `2` | cancel an unfilled entry after this many minutes resting (wall-clock; `0` disables the time box) |
|
||||
| `TRADING_ENTRY_INVALIDATE_PCT` | `0.0` | cancel an unfilled entry if the **live last-trade price** trades this fraction below the trigger (`0.0` = any break below entry; `0.005` = 0.5% below; negative disables) |
|
||||
| `TRADING_ENTRY_GUARD_INTERVAL_SECONDS` | `5` | how often the fast guard re-checks armed entries against the live price — lower = faster break detection, more quote calls |
|
||||
| `TRADING_MAX_DAILY_LOSS_PCT` | `0.03` | daily-loss circuit breaker |
|
||||
| `WATCHER_SYMBOLS` | `AAPL,TSLA,…` | static watchlist; `--discover` adds screener actives |
|
||||
| `WATCHER_PRICE_MIN/MAX` | `1` / `20` | candidate price band |
|
||||
| `WATCHER_EVENT_DB_PATH` | `./data/momentum.duckdb` | the event store |
|
||||
| `DASHBOARD_HOST/PORT` | `127.0.0.1` / `8010` | UI address |
|
||||
| ~04:00+ | premarket ingest + discovery begin | trigger board fills with gappers |
|
||||
| 09:30 | session gate opens; entries allowed 09:30–15:55 | first `TRIGGER FIRED` / auto-approved orders |
|
||||
| 09:35 | opening ranges complete → triggers arm | board states flip waiting→armed |
|
||||
| 15:55 | fresh entries barred | — |
|
||||
| 16:00± | EOD flatten of open positions | positions = 0 |
|
||||
| 20:00 Mon–Fri | nightly job: SIP re-fetch, event retention, log rotation, shadow tracks, float/dilution caches | `appended:` line in scheduler log |
|
||||
|
||||
---
|
||||
|
||||
## 3. Start commands
|
||||
|
||||
**Full live paper loop (recommended):**
|
||||
## Health checks
|
||||
|
||||
**Compose (production):**
|
||||
```bash
|
||||
python run_live_paper.py
|
||||
# dashboard: http://127.0.0.1:8010
|
||||
# flags: --once --symbols SNDL,COSM --auto-approve --discover --no-dashboard
|
||||
docker ps # all momentum containers Up, loop (healthy)
|
||||
docker logs -f <loop> # discover/ingest/watch/execute cycle lines
|
||||
curl -s localhost:8765/api/snapshots | head -c 200 # dashboard serves
|
||||
# admin panel: http://<host>:8767 — services, DB, Alpaca, Ollama cards
|
||||
```
|
||||
|
||||
Each tick prints one status line:
|
||||
**Local (systemd):** `systemctl --user status 'momentum-*'` + the same curls.
|
||||
|
||||
**Database truth (any shape):**
|
||||
```sql
|
||||
SELECT max(timestamp) FROM events; -- heartbeat: < ~2 min during any hour
|
||||
SELECT count(*) FROM minute_bars; -- history intact: ~19M+ (2026-07)
|
||||
```
|
||||
[09:31:02] ingest: 240 minute rows | watch: evaluated=5 ready=['SNDL'] blocked=3 | sync: ok | execute: approvals_requested=['…'] auto_executed=0
|
||||
```
|
||||
Timestamps are **ET-naive**: compare against ET now, not UTC (a constant
|
||||
"4-hour stale" reading is a timezone artifact, not an outage).
|
||||
|
||||
**Standalone read-only dashboard** (e.g. to review a past session):
|
||||
## Failure modes seen in production (and the fix)
|
||||
|
||||
```bash
|
||||
python dashboard_api.py
|
||||
```
|
||||
| Symptom | Cause | Fix |
|
||||
|---|---|---|
|
||||
| Entries queue as "manual" in the dashboard | `TRADING_AUTO_APPROVE` missing from the loop's env | add `=1` to the loop service env; stale queue rows are zombies that clear with the session |
|
||||
| Orders submitted after hours | (fixed 2026-07-10) missing session gate | `TRADING_SESSION_ONLY=1` — default on; verify if reintroduced |
|
||||
| Duplicate resting orders after a loop crash/restart | armed/submitted state is in-memory | diff Alpaca open orders after any loop restart; cancel dupes (`get_orders` unscoped — `symbols=` can silently return empty) |
|
||||
| `PermissionError` on `/app/data/...` | host dir not owned by container uid | `chown -R 10001:10001 <data dir>` |
|
||||
| Postgres serving a tiny DB when history should exist | wrong pgdata mount (path typo/case, nested cluster, stale container) | `docker inspect <postgres> --format '{{range .Mounts}}...'`; verify by **size at destination**; recreate containers after volume edits (restart is not enough) |
|
||||
| Autovacuum cancelled every 60s on a big table | a frequent connector re-applying schema DDL | healthchecks/one-shot scripts must set `PG_SKIP_SCHEMA=1` |
|
||||
| Env edits in the YAML do nothing | TrueNAS expanded the anchors; `x-app` is decorative | put env in each **service's** block; verify with `docker exec <svc> sh -c 'echo $VAR'` |
|
||||
| Loop trading names it shouldn't | check gates in order: static open gap 10–35% (or ≥150% + rvol≥5), $2–20, day-vol, quality ≥0.60, VWAP, session window | `./momentum inspect criteria SYM` or the events `risk_rule_triggered` payloads |
|
||||
|
||||
Approve/reject/exit buttons require the orchestrator (it attaches the
|
||||
execution service to the same server); the standalone runner returns 503 on
|
||||
actions by design.
|
||||
## Safety posture
|
||||
|
||||
**Other tools:**
|
||||
- Paper account; bracket orders carry the protective stop; the exit manager
|
||||
enforces stops on anything naked, ratchets breakeven at +5%, catastrophe-
|
||||
stops at −10%/1.5×R.
|
||||
- **When in doubt, stop the loop container** — positions stay protected by
|
||||
broker-side stop legs; a missed day costs nothing (the no-trade day is a
|
||||
legitimate outcome). Never run two loops against one account.
|
||||
- Config experiments: use the admin panel's profiles; treat each switch as a
|
||||
hypothesis and compare the day against `validated-core` behavior.
|
||||
|
||||
```bash
|
||||
python fetch_minute_bars.py SNDL COSM --lookback 390 # ad-hoc ingestion
|
||||
python research_cli.py symbols --date 2026-06-11 # what's in the DB
|
||||
python research_cli.py gappers --date 2026-06-11 # gap scan → scanner_snapshots
|
||||
python backtest_cli.py --date 2026-06-10 # backtest a stored session
|
||||
python milestone3_verify.py # Schwab health walk
|
||||
python -m pytest # full test suite
|
||||
```
|
||||
## Where things live
|
||||
|
||||
---
|
||||
|
||||
## 4. The trading day, end to end
|
||||
|
||||
1. **Ingest** — every 60 s, 1-minute IEX bars for the watchlist land in
|
||||
`data/research/market.duckdb` with correct session flags. On boot, a
|
||||
30-day daily backfill gives gap % and relative volume a baseline.
|
||||
2. **Watch** — every 30 s the watcher pulls in-band candidates, evaluates
|
||||
nine weighted criteria (gap, RVOL, impulse, pullback, pullback volume,
|
||||
VWAP, candle quality, breakout, data sufficiency) plus structure
|
||||
detection, and drives `discovered → watching → ready | blocked | late`.
|
||||
A `signal_ready` is emitted at most once per symbol per session.
|
||||
3. **Sync** — Alpaca account, positions, and orders snapshot into the event
|
||||
store so projections and risk guards see broker truth.
|
||||
4. **Execute** — ready signals become orders sized at `risk_per_trade_pct`
|
||||
of equity, with the stop from the setup and the target at
|
||||
entry + `reward_multiple` × risk (2R by default). Manual mode parks them
|
||||
in the dashboard queue; **auto mode (`--auto-approve`) submits
|
||||
immediately, no approval.** Guards: max concurrent positions, no re-entry
|
||||
into held symbols, one request per symbol per session.
|
||||
5. **Manage** — approve/reject from the dashboard; **Exit** sends a market
|
||||
sell for the synced quantity.
|
||||
|
||||
### The entry lifecycle (auto-arm → fill or back out)
|
||||
|
||||
The earlier system would "see" a setup as something that had *already
|
||||
happened* rather than a trade about to happen, and never pull the trigger.
|
||||
The execution model now separates recognition from conviction from
|
||||
discipline:
|
||||
|
||||
1. **Recognize** — when all the criteria pass, a `signal_ready` fires. The
|
||||
setup is evaluated against the **last bar's timestamp**, not the wall
|
||||
clock, so a live 9:40 breakout is judged against the 9:40 cutoff instead
|
||||
of being dismissed as hours stale.
|
||||
2. **Arm with conviction** — in auto mode the order is submitted right away
|
||||
as a **bracket** (entry + stop + target placed atomically). With
|
||||
`TRADING_ENTRY_ORDER_TYPE=limit` (default) the entry rests at the trigger,
|
||||
so "unfilled" is a real, observable state.
|
||||
3. **Back out at a defined point** — a fast **guard loop**
|
||||
(`TRADING_ENTRY_GUARD_INTERVAL_SECONDS`, default 5 s) re-checks every
|
||||
armed-but-unfilled entry and cancels it if either:
|
||||
- the **live last-trade price** (pulled tick-by-tick from Alpaca's data
|
||||
API, not a bar close) trades back below the trigger by
|
||||
`TRADING_ENTRY_INVALIDATE_PCT`, or
|
||||
- it has rested longer than `TRADING_ENTRY_TIMEOUT_BARS` minutes
|
||||
(wall-clock — running the guard often does **not** shorten this window).
|
||||
A cancel frees the risk budget and the concurrent-position slot, and emits
|
||||
a `risk_rule_triggered` (`entry_backout`) you can see in the activity feed.
|
||||
Once the order **fills**, it stops being tracked and the bracket's stop and
|
||||
target manage the open position — no second-guessing.
|
||||
|
||||
Tuning cheat-sheet: react to a fakeout faster → lower
|
||||
`TRADING_ENTRY_GUARD_INTERVAL_SECONDS` (e.g. `2`); more patience on the fill →
|
||||
raise `TRADING_ENTRY_TIMEOUT_BARS`; give the breakout room to wobble → set
|
||||
`TRADING_ENTRY_INVALIDATE_PCT` to something like `0.005`; chase fills instead
|
||||
of resting → `TRADING_ENTRY_ORDER_TYPE=market` (note: market fills can't time
|
||||
out); bigger winners → raise `TRADING_REWARD_MULTIPLE`.
|
||||
|
||||
> **Quote rate limits.** The guard makes one latest-trade call per armed
|
||||
> symbol per pass (a ~2 s cache collapses duplicates). At the 5 s default with
|
||||
> a few open entries you're well inside Alpaca's free-tier 200 req/min. If you
|
||||
> drop the interval to 1 s with several armed entries, watch the budget —
|
||||
> bars ingestion and account sync also draw from it.
|
||||
|
||||
### Reading the dashboard
|
||||
|
||||
- **Badge** — blue `PAPER · ALPACA` vs red `LIVE · SCHWAB`. If it's red and
|
||||
you didn't mean it, stop and check `TRADING_EXECUTION_MODE`.
|
||||
- **P&L strip** (top) — session realized / unrealized / total, win rate,
|
||||
average R, open/closed counts. Green/red signed.
|
||||
- **Approval Queue** — pending entries with a **risk preview** ($ risk,
|
||||
shares, notional, stop %) and Approve/Reject. Keyboard: `A` / `R` act on the
|
||||
top row.
|
||||
- **Ready Signals / Open Positions** — each row has a live **sparkline** (last
|
||||
~40 one-minute closes) with dashed entry (blue) and stop (red) lines.
|
||||
- **Watch States** — click any row to expand the **nine-criteria breakdown**
|
||||
(✓ pass / ✕ fail / · not-evaluated). The add-symbol box (or `/`) injects a
|
||||
ticker onto the watchlist for the rest of the session.
|
||||
- **Activity Feed** — submissions and fills as they happen.
|
||||
- **Order Lifecycle** — per-order status from the event store.
|
||||
- **Connection dot** (top right) — green pulse = live SSE stream; it falls
|
||||
back to 4 s polling automatically if the stream drops.
|
||||
- **Sound** — toggle the button or press `S` for an audio + toast alert when a
|
||||
new signal goes ready. Press `?` for the shortcut list.
|
||||
|
||||
Streams from `GET /api/stream` (SSE); also exposes `/api/snapshots`,
|
||||
`/api/criteria?symbol=`, `/api/bars?symbol=&minutes=`, and
|
||||
`POST /api/watch/add`. Charts require the research DB, which the orchestrator
|
||||
wires in automatically; the standalone `dashboard_api.py` has no bars source,
|
||||
so sparklines there show "no bars".
|
||||
|
||||
---
|
||||
|
||||
## 5. Troubleshooting
|
||||
|
||||
| Symptom | Likely cause / fix |
|
||||
|---|---|
|
||||
| `watch: evaluated=0` all day | No minute bars for today. Check keys; run `python research_cli.py symbols`. Outside market hours there are no new bars — backfill with `fetch_minute_bars.py --lookback 1440`. |
|
||||
| Candidates exist but never `ready` | Score < 60%. Watch the `Watch States` panel score column; criteria details are in `criteria_evaluated` events. |
|
||||
| `ingest: … 403` errors | Free keys are IEX-only — ensure `ALPACA_DATA_FEED=iex` (default) and that the request window ends ≥30 s in the past (handled automatically). |
|
||||
| Approve button does nothing | You're on `dashboard_api.py` (read-only). Use `run_live_paper.py`. |
|
||||
| Port 8010 busy | Set `DASHBOARD_PORT`, or `--no-dashboard`. |
|
||||
| Schwab shows `SCHWAB-UNAUTH` | Expected without OAuth. Run the flow in `schwab/auth/oauth2_flow_manager.py`; verify with `milestone3_verify.py`. |
|
||||
| Start fresh | Delete `data/momentum.duckdb` (events) and/or `data/research/` (bars). Both rebuild automatically. |
|
||||
|
||||
---
|
||||
|
||||
## 6. Going live on Schwab (when ready)
|
||||
|
||||
Paper-trade until the stats earn it. Then: complete OAuth
|
||||
(`milestone3_verify.py` all green), set `TRADING_EXECUTION_MODE=schwab_live`,
|
||||
keep `TRADING_AUTO_APPROVE=0`, and start with the smallest viable
|
||||
`TRADING_RISK_PER_TRADE_PCT`. The dashboard badge turns red in live mode.
|
||||
- **Everything queryable**: PostgreSQL (`DATABASE_URL`) — events, bars,
|
||||
research, shadow tracks, caches.
|
||||
- **Local disk / app-data volume**: logs (rotated nightly), transcripts,
|
||||
`learned_params.json`, `overrides.env` (admin panel), `profiles/`.
|
||||
- **Historical plans/era docs**: `docs/history/` — don't follow them.
|
||||
+437
@@ -0,0 +1,437 @@
|
||||
"""Momentum ADMIN panel (:8767) — configure, validate, and control the stack.
|
||||
|
||||
LAN-only by intent; every mutating endpoint (and the page itself) requires the
|
||||
ADMIN_TOKEN (env). Capabilities:
|
||||
- service health: docker containers (via /var/run/docker.sock), DB, Alpaca,
|
||||
Ollama, loop heartbeat, market clock, data feed
|
||||
- config: read/edit the whitelisted strategy knobs via data/overrides.env
|
||||
(beats compose env on next boot; see runtime/env_overrides.py); secrets are
|
||||
shown as present/absent only, never values
|
||||
- profiles: named knob presets (data/profiles/*.json) applied in one click
|
||||
- restart: bounce individual containers so config changes take effect
|
||||
- paper/live switch: guarded — refuses unless ALPACA_LIVE_* keys are set
|
||||
|
||||
Run: python admin_api.py (ADMIN_HOST/ADMIN_PORT, default 0.0.0.0:8767)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Load the repo .env ONLY when this file is run as the entrypoint
|
||||
# (`python admin_api.py`). A bare module-level load_dotenv() here used to
|
||||
# fire on every import too -- including pytest collection -- leaking the
|
||||
# real repo .env (secrets included) into any process that merely imports
|
||||
# this module. This runs before the os.environ.get(...) constants below,
|
||||
# so `python admin_api.py` still behaves identically.
|
||||
load_dotenv()
|
||||
from runtime.env_overrides import load as _load_overrides # noqa: E402
|
||||
_load_overrides()
|
||||
from runtime import env_overrides # noqa: E402
|
||||
|
||||
DOCKER_SOCK = os.environ.get("DOCKER_SOCK", "/var/run/docker.sock")
|
||||
# Regime backtest (plan 011) on-demand trigger — logs to the app-data volume so
|
||||
# GET /api/regime/status can tail it (the panel had no way to fire this).
|
||||
REGIME_LOG = os.environ.get(
|
||||
"REGIME_RUN_LOG", os.path.join(os.path.dirname(__file__), "data", "regime_run.log"))
|
||||
_REGIME_DONE = "REGIME PIPELINE DONE"
|
||||
|
||||
|
||||
def run_regime_pipeline() -> int:
|
||||
"""Fire the full regime shadow pipeline DETACHED (it takes a few min): rebuild
|
||||
session labels -> walk-forward score both models -> replay + adjudicate the
|
||||
regime_gate experiments. Returns the child pid. Shadow only — never touches a
|
||||
live order. stdout+stderr -> REGIME_LOG (truncated each run so status shows
|
||||
the CURRENT run); a trailing marker lets status report completion."""
|
||||
py = sys.executable
|
||||
steps = " && ".join([
|
||||
f"{py} -m research.regime build-features",
|
||||
f"{py} -m research.regime score --model threshold_v1",
|
||||
f"{py} -m research.regime score --model meta_v1",
|
||||
f"{py} -m research.experiments run",
|
||||
f"{py} -m research.experiments adjudicate",
|
||||
f"echo {_REGIME_DONE}",
|
||||
])
|
||||
os.makedirs(os.path.dirname(REGIME_LOG), exist_ok=True)
|
||||
log = open(REGIME_LOG, "wb") # noqa: SIM115 — handed to the detached child
|
||||
proc = subprocess.Popen(["/bin/sh", "-c", steps], stdout=log,
|
||||
stderr=subprocess.STDOUT, start_new_session=True)
|
||||
return proc.pid
|
||||
|
||||
|
||||
def regime_status() -> dict:
|
||||
"""Tail REGIME_LOG + whether the last run finished (the marker) — no DB query
|
||||
needed; the adjudicate step prints each experiment's verdict into the log."""
|
||||
try:
|
||||
with open(REGIME_LOG, encoding="utf-8", errors="replace") as fh:
|
||||
lines = fh.read().splitlines()
|
||||
except FileNotFoundError:
|
||||
return {"log_tail": [], "running": False, "ran": False}
|
||||
return {"log_tail": lines[-40:], "ran": True,
|
||||
"running": _REGIME_DONE not in lines[-3:] if lines else False}
|
||||
PROFILES_DIR = os.environ.get(
|
||||
"PROFILES_DIR", os.path.join(os.path.dirname(__file__), "data", "profiles"))
|
||||
ADMIN_TOKEN = os.environ.get("ADMIN_TOKEN", "")
|
||||
|
||||
# knobs the UI may read+write — everything else is off-limits by construction
|
||||
EDITABLE_PREFIXES = ("TRADING_", "TRIGGER_", "WATCHER_", "LIVE_BARS_", "NEWS_",
|
||||
"OLLAMA_", "DISCOVER_", "SYMBOL_WEIGHTS")
|
||||
EDITABLE_EXACT = {"ALPACA_DATA_FEED"}
|
||||
SECRET_MARKERS = ("KEY", "SECRET", "TOKEN", "PASSWORD", "WEBHOOK", "URL")
|
||||
|
||||
SEED_PROFILES = {
|
||||
"validated-core": {
|
||||
"_note": "The year-validated population config (2026-07-09 state).",
|
||||
"TRADING_MIN_QUALITY_SCORE": "0.60", "TRADING_MAX_FRESH_ENTRIES_PER_DAY": "10",
|
||||
"TRADING_MIN_ENTRY_GAP_PCT": "0.10", "TRADING_MIN_DAY_VOLUME": "300000",
|
||||
"TRADING_PRICE_MIN": "2.0", "TRADING_PRICE_MAX": "20.0",
|
||||
"TRIGGER_GAP_MIN": "10.0", "TRIGGER_GAP_MAX": "35",
|
||||
"TRIGGER_GAP_SQUEEZE_MIN": "150", "TRIGGER_GAP_SQUEEZE_RVOL": "5.0",
|
||||
},
|
||||
"conservative": {
|
||||
"_note": "Fewer, stricter entries: B-grade floor, half the daily cap, no squeeze lottery.",
|
||||
"TRADING_MIN_QUALITY_SCORE": "0.75", "TRADING_MAX_FRESH_ENTRIES_PER_DAY": "5",
|
||||
"TRADING_MIN_ENTRY_GAP_PCT": "0.10", "TRADING_MIN_DAY_VOLUME": "300000",
|
||||
"TRADING_PRICE_MIN": "2.0", "TRADING_PRICE_MAX": "20.0",
|
||||
"TRIGGER_GAP_MIN": "10.0", "TRIGGER_GAP_MAX": "35",
|
||||
"TRIGGER_GAP_SQUEEZE_MIN": "0",
|
||||
},
|
||||
"squeeze-hunter": {
|
||||
"_note": "Experiment: lean into the >150% volume-verified squeeze band + looser quality.",
|
||||
"TRADING_MIN_QUALITY_SCORE": "0.50", "TRADING_MAX_FRESH_ENTRIES_PER_DAY": "10",
|
||||
"TRADING_MIN_ENTRY_GAP_PCT": "0.10", "TRADING_MIN_DAY_VOLUME": "300000",
|
||||
"TRADING_PRICE_MIN": "2.0", "TRADING_PRICE_MAX": "20.0",
|
||||
"TRIGGER_GAP_MIN": "10.0", "TRIGGER_GAP_MAX": "35",
|
||||
"TRIGGER_GAP_SQUEEZE_MIN": "150", "TRIGGER_GAP_SQUEEZE_RVOL": "3.0",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _safe_overrides() -> dict:
|
||||
"""Overrides for API responses with secret VALUES redacted — a /api/mode live
|
||||
switch writes ALPACA_LIVE_* into overrides.env; without this they'd echo back
|
||||
in every /api/status and /api/config response."""
|
||||
merged = {**env_overrides.read_overrides(), **env_overrides.read_db_overrides()}
|
||||
return {k: ("***" if any(m in k for m in SECRET_MARKERS) else v)
|
||||
for k, v in merged.items()}
|
||||
|
||||
|
||||
def _editable(key: str) -> bool:
|
||||
if key in EDITABLE_EXACT:
|
||||
return True
|
||||
return (key.startswith(EDITABLE_PREFIXES)
|
||||
and not any(m in key for m in SECRET_MARKERS))
|
||||
|
||||
|
||||
def _mask_value(key: str, value: str) -> str:
|
||||
"""Mask secret VALUES (*** + last 4). Uses the broader SECRET_MARKERS set so
|
||||
DATABASE_URL / *_WEBHOOK_URL are masked too — narrowing to KEY|SECRET|TOKEN|
|
||||
PASSWORD would leak the DB password in the config table."""
|
||||
if any(m in key.upper() for m in SECRET_MARKERS):
|
||||
tail = value[-4:] if value and len(value) >= 4 else ""
|
||||
return "***" + tail
|
||||
return value
|
||||
|
||||
|
||||
def _env_example_keys() -> set[str]:
|
||||
"""Keys documented in .env.example — the 'required' config surface used for
|
||||
drift detection (a documented key absent from the effective env is drift)."""
|
||||
path = os.path.join(os.path.dirname(__file__), ".env.example")
|
||||
keys: set[str] = set()
|
||||
try:
|
||||
with open(path) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
keys.add(line.split("=", 1)[0].strip())
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return keys
|
||||
|
||||
|
||||
def effective_config() -> dict:
|
||||
"""Per-key effective value + source for the admin config table.
|
||||
|
||||
Merge order baked < compose < overrides.env (< learned). Sources we can prove
|
||||
at runtime: ``overrides.env`` (from data/overrides.env) vs ``compose`` (the
|
||||
process env — baked and compose are indistinguishable inside the container, so
|
||||
both surface as ``compose``); a documented .env.example key that is absent
|
||||
surfaces as ``missing`` (drift). Secret values are masked."""
|
||||
overrides = env_overrides.read_overrides()
|
||||
db_overrides = env_overrides.read_db_overrides()
|
||||
example = _env_example_keys()
|
||||
keys = (set(example) | set(overrides) | set(db_overrides)
|
||||
| {k for k in os.environ if _editable(k)})
|
||||
out: dict[str, dict] = {}
|
||||
for key in sorted(keys):
|
||||
value = os.environ.get(key)
|
||||
if value is None:
|
||||
out[key] = {"value_masked": None,
|
||||
"source": "missing" if key in example else "unset",
|
||||
"drift": key in example}
|
||||
continue
|
||||
source = ("database" if key in db_overrides
|
||||
else "overrides.env" if key in overrides else "compose")
|
||||
out[key] = {"value_masked": _mask_value(key, str(value)), "source": source}
|
||||
return out
|
||||
|
||||
|
||||
# ---------- docker over the unix socket (stdlib only) ----------
|
||||
class _SockConn(http.client.HTTPConnection):
|
||||
def __init__(self):
|
||||
super().__init__("localhost")
|
||||
|
||||
def connect(self):
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.connect(DOCKER_SOCK)
|
||||
self.sock = s
|
||||
|
||||
|
||||
def docker(method: str, path: str) -> tuple[int, object]:
|
||||
try:
|
||||
c = _SockConn()
|
||||
c.request(method, path)
|
||||
r = c.getresponse()
|
||||
body = r.read()
|
||||
return r.status, (json.loads(body) if body else None)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return 0, {"error": str(exc)}
|
||||
|
||||
|
||||
def list_containers() -> list[dict]:
|
||||
st, data = docker("GET", "/containers/json?all=1")
|
||||
if st != 200 or not isinstance(data, list):
|
||||
return []
|
||||
out = []
|
||||
for c in data:
|
||||
name = (c.get("Names") or ["?"])[0].lstrip("/")
|
||||
if "momentum" not in name and "postgres" not in name:
|
||||
continue
|
||||
out.append({
|
||||
"name": name,
|
||||
"state": c.get("State"),
|
||||
"status": c.get("Status"),
|
||||
"health": ("healthy" if "(healthy)" in (c.get("Status") or "")
|
||||
else "unhealthy" if "(unhealthy)" in (c.get("Status") or "")
|
||||
else None),
|
||||
"image": c.get("Image"),
|
||||
"id": c.get("Id", "")[:12],
|
||||
})
|
||||
return sorted(out, key=lambda x: x["name"])
|
||||
|
||||
|
||||
# ---------- health checks ----------
|
||||
def checks() -> dict:
|
||||
out: dict = {}
|
||||
try:
|
||||
from research.multi_schema import open_research_db
|
||||
con = open_research_db("market")
|
||||
r = con.execute("SELECT max(timestamp) FROM events").fetchall()[0][0]
|
||||
age = (datetime.now() - r).total_seconds() if r else None
|
||||
out["db"] = {"ok": True, "newest_event_age_s": round(age) if age else None}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
out["db"] = {"ok": False, "error": str(exc)[:120]}
|
||||
try:
|
||||
# active execution broker (BROKER env) — JSON key stays "alpaca" for
|
||||
# UI compatibility; "name" carries the real broker label
|
||||
from tradier_broker.factory import make_trading_client
|
||||
c, label = make_trading_client()
|
||||
if c is None:
|
||||
out["alpaca"] = {"ok": False, "error": label}
|
||||
else:
|
||||
clock = c.get_clock()
|
||||
acct = c.get_account()
|
||||
out["alpaca"] = {"ok": True, "market_open": clock.get("is_open"),
|
||||
"equity": acct.get("equity"), "name": label,
|
||||
"mode": "LIVE" if label.endswith("LIVE") else "paper"}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
out["alpaca"] = {"ok": False, "error": str(exc)[:120]}
|
||||
try:
|
||||
host = os.environ.get("OLLAMA_HOST", "http://192.168.1.9:11434")
|
||||
with urllib.request.urlopen(host + "/api/tags", timeout=4) as r:
|
||||
out["ollama"] = {"ok": r.status == 200}
|
||||
except Exception: # noqa: BLE001
|
||||
out["ollama"] = {"ok": False, "note": "enrichment fails safe; trading unaffected"}
|
||||
out["feed"] = os.environ.get("ALPACA_DATA_FEED", "iex")
|
||||
return out
|
||||
|
||||
|
||||
# ---------- profiles ----------
|
||||
def seed_profiles() -> None:
|
||||
try:
|
||||
os.makedirs(PROFILES_DIR, exist_ok=True)
|
||||
for name, body in SEED_PROFILES.items():
|
||||
p = os.path.join(PROFILES_DIR, f"{name}.json")
|
||||
if not os.path.exists(p):
|
||||
json.dump(body, open(p, "w"), indent=2)
|
||||
except PermissionError:
|
||||
# host volume not writable by the container user (uid 10001) — degrade
|
||||
# instead of crash-looping; profiles/config editing will 500 until the
|
||||
# host dir is chown'd (see deploy/docker/README.md)
|
||||
print(f"[admin] WARNING: {PROFILES_DIR} not writable (chown -R 10001:10001 "
|
||||
"the data dir on the host); profiles/config editing disabled")
|
||||
|
||||
|
||||
def profiles() -> dict:
|
||||
seed_profiles()
|
||||
out = {}
|
||||
for f in sorted(os.listdir(PROFILES_DIR)):
|
||||
if f.endswith(".json"):
|
||||
try:
|
||||
out[f[:-5]] = json.load(open(os.path.join(PROFILES_DIR, f)))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
# ---------- HTTP ----------
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def _send(self, code: int, body, ctype="application/json"):
|
||||
raw = body if isinstance(body, bytes) else json.dumps(body, default=str).encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
def _authed(self) -> bool:
|
||||
# FAIL CLOSED: no token configured => deny (this panel fronts the docker
|
||||
# socket + the paper/live switch; an unset env var must not open it to the
|
||||
# LAN). Header only — a ?token= query would leak into proxy/access logs.
|
||||
if not ADMIN_TOKEN:
|
||||
return False
|
||||
return hmac.compare_digest(self.headers.get("X-Admin-Token", ""), ADMIN_TOKEN)
|
||||
|
||||
def do_GET(self): # noqa: N802
|
||||
path = self.path.split("?")[0]
|
||||
if path == "/":
|
||||
page = os.path.join(os.path.dirname(__file__), "api", "static", "admin.html")
|
||||
return self._send(200, open(page, "rb").read(), "text/html")
|
||||
if not self._authed():
|
||||
return self._send(401, {"error": "bad or missing X-Admin-Token"})
|
||||
if path == "/api/status":
|
||||
return self._send(200, {"containers": list_containers(), "checks": checks(),
|
||||
"overrides": _safe_overrides(),
|
||||
"version": os.environ.get("APP_BUILD_TAG", "dev"),
|
||||
"git_sha": os.environ.get("APP_GIT_SHA", "dev")[:12]})
|
||||
if path == "/api/config":
|
||||
eff = {k: v for k, v in sorted(os.environ.items()) if _editable(k)}
|
||||
secrets = {k: bool(os.environ.get(k)) for k in
|
||||
("ALPACA_API_KEY", "ALPACA_SECRET_KEY", "ALPACA_LIVE_API_KEY",
|
||||
"ALPACA_LIVE_SECRET_KEY", "TELEGRAM_BOT_TOKEN", "YOUTUBE_API_KEY",
|
||||
"FMP_API_KEY")}
|
||||
return self._send(200, {"effective": eff, "secrets_present": secrets,
|
||||
"overrides": _safe_overrides()})
|
||||
if path == "/api/profiles":
|
||||
return self._send(200, profiles())
|
||||
if path == "/api/config/effective":
|
||||
return self._send(200, {"config": effective_config()})
|
||||
if path == "/api/regime/status":
|
||||
return self._send(200, regime_status())
|
||||
return self._send(404, {"error": "unknown route"})
|
||||
|
||||
def do_POST(self): # noqa: N802
|
||||
if not self._authed():
|
||||
return self._send(401, {"error": "bad or missing X-Admin-Token"})
|
||||
n = int(self.headers.get("Content-Length") or 0)
|
||||
try:
|
||||
body = json.loads(self.rfile.read(n) or b"{}")
|
||||
except Exception: # noqa: BLE001
|
||||
return self._send(400, {"error": "bad json"})
|
||||
path = self.path.split("?")[0]
|
||||
|
||||
if path == "/api/config":
|
||||
updates = body.get("set") or {}
|
||||
deletes = body.get("delete") or []
|
||||
bad = [k for k in list(updates) + list(deletes) if not _editable(k)]
|
||||
if bad:
|
||||
return self._send(400, {"error": f"not editable: {bad}"})
|
||||
# knobs go to the DATABASE (owner-facing store; whitelist above
|
||||
# already refused secrets). overrides.env keeps secrets/bootstrap.
|
||||
try:
|
||||
merged = env_overrides.write_db_overrides(
|
||||
{**{k: str(v) for k, v in updates.items()},
|
||||
**{k: None for k in deletes}})
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return self._send(502, {"error": f"db write failed: {exc}"})
|
||||
return self._send(200, {"ok": True, "overrides": merged,
|
||||
"note": "restart affected services to apply"})
|
||||
|
||||
if path == "/api/profiles/apply":
|
||||
name = body.get("name", "")
|
||||
prof = profiles().get(name)
|
||||
if not prof:
|
||||
return self._send(404, {"error": f"no profile {name!r}"})
|
||||
knobs = {k: str(v) for k, v in prof.items() if not k.startswith("_")}
|
||||
merged = env_overrides.write_db_overrides(knobs)
|
||||
return self._send(200, {"ok": True, "applied": name, "overrides": merged,
|
||||
"note": "restart the loop to apply"})
|
||||
|
||||
if path == "/api/restart":
|
||||
name = body.get("service", "")
|
||||
target = next((c for c in list_containers() if c["name"] == name), None)
|
||||
if not target:
|
||||
return self._send(404, {"error": f"no container {name!r}"})
|
||||
st, _ = docker("POST", f"/containers/{target['id']}/restart?t=10")
|
||||
return self._send(200 if st in (200, 204) else 502,
|
||||
{"ok": st in (200, 204), "docker_status": st})
|
||||
|
||||
if path == "/api/mode":
|
||||
# paper/live switch — guarded: live requires ALPACA_LIVE_* keys present
|
||||
mode = body.get("mode")
|
||||
if mode == "live":
|
||||
if not (os.environ.get("ALPACA_LIVE_API_KEY")
|
||||
and os.environ.get("ALPACA_LIVE_SECRET_KEY")):
|
||||
return self._send(400, {"error": "ALPACA_LIVE_API_KEY/SECRET not set — "
|
||||
"refusing to switch to live"})
|
||||
merged = env_overrides.write_overrides({
|
||||
"ALPACA_BASE_URL": "https://api.alpaca.markets",
|
||||
"ALPACA_API_KEY": os.environ["ALPACA_LIVE_API_KEY"],
|
||||
"ALPACA_SECRET_KEY": os.environ["ALPACA_LIVE_SECRET_KEY"]})
|
||||
elif mode == "paper":
|
||||
merged = env_overrides.write_overrides({
|
||||
"ALPACA_BASE_URL": None, "ALPACA_API_KEY": None,
|
||||
"ALPACA_SECRET_KEY": None})
|
||||
else:
|
||||
return self._send(400, {"error": "mode must be paper|live"})
|
||||
return self._send(200, {"ok": True, "mode": mode, "overrides": merged,
|
||||
"note": "restart the loop to apply"})
|
||||
|
||||
if path == "/api/regime/run":
|
||||
try:
|
||||
pid = run_regime_pipeline()
|
||||
except Exception as e: # noqa: BLE001
|
||||
return self._send(500, {"ok": False, "error": str(e)[:200]})
|
||||
return self._send(200, {"ok": True, "pid": pid, "log": REGIME_LOG,
|
||||
"note": "regime shadow pipeline started (~a few min); "
|
||||
"poll GET /api/regime/status"})
|
||||
return self._send(404, {"error": "unknown route"})
|
||||
|
||||
def log_message(self, *a): # quiet
|
||||
pass
|
||||
|
||||
|
||||
def main() -> None:
|
||||
host = os.environ.get("ADMIN_HOST", "0.0.0.0")
|
||||
port = int(os.environ.get("ADMIN_PORT", "8767"))
|
||||
seed_profiles()
|
||||
print(f"[admin] serving http://{host}:{port} "
|
||||
f"(token {'SET' if ADMIN_TOKEN else 'NOT SET — open access'}; "
|
||||
f"docker sock {'ok' if os.path.exists(DOCKER_SOCK) else 'MISSING'})")
|
||||
ThreadingHTTPServer((host, port), Handler).serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+98
-6
@@ -137,8 +137,14 @@ class AlpacaPaperClient:
|
||||
continue
|
||||
raise AlpacaApiError(exc.code, exc.read().decode(errors="replace")) from exc
|
||||
except urllib.error.URLError:
|
||||
# transient network/DNS (e.g. gaierror) — retry a couple times
|
||||
if attempt >= self.max_retries:
|
||||
# transient network/DNS (e.g. gaierror) — retry a couple times.
|
||||
# NOT for POST: urllib wraps a lost RESPONSE (read timeout/reset)
|
||||
# in URLError too, so attempt 1 may have SUCCEEDED at the broker
|
||||
# with the reply lost; re-sending POST /orders then rests a
|
||||
# duplicate live order (or 422s and looks like a failure while a
|
||||
# real bracket rests untracked). Idempotent GET/DELETE retry; POST
|
||||
# raises so the caller reconciles against broker positions.
|
||||
if method == "POST" or attempt >= self.max_retries:
|
||||
raise
|
||||
_time.sleep(0.4 * (attempt + 1))
|
||||
raise RuntimeError("request retries exhausted") # unreachable
|
||||
@@ -162,6 +168,11 @@ class AlpacaPaperClient:
|
||||
def get_account(self) -> dict:
|
||||
return self._cached("account", lambda: self._trading("GET", "/account"))
|
||||
|
||||
def get_clock(self) -> dict:
|
||||
"""Broker market clock: {is_open, next_open, next_close, timestamp}. The
|
||||
authoritative open/closed check (handles weekends, holidays, half-days)."""
|
||||
return self._trading("GET", "/clock")
|
||||
|
||||
def get_positions(self, fresh: bool = False) -> list[dict]:
|
||||
# fresh=True bypasses the TTL cache — REQUIRED for the naked-stop guard,
|
||||
# which must never decide an order is unfilled off a stale snapshot.
|
||||
@@ -172,9 +183,23 @@ class AlpacaPaperClient:
|
||||
return val
|
||||
return self._cached("positions", lambda: self._trading("GET", "/positions"))
|
||||
|
||||
def get_order_by_client_order_id(self, client_order_id: str) -> dict | None:
|
||||
"""The single order carrying this client_order_id, or None if unknown.
|
||||
|
||||
Used to reconcile a LOST submit response (network drop after the POST
|
||||
left): the order may actually be resting at the broker, so we look it up
|
||||
by the client_order_id we sent before assuming it was cancelled. A 404
|
||||
(never landed) surfaces as an AlpacaApiError the caller treats as 'not
|
||||
found'. `_trading` already prefixes /v2; the literal ':' in the path is
|
||||
Alpaca's by-client-id sub-resource, not a typo."""
|
||||
return self._trading(
|
||||
"GET", "/orders:by_client_order_id",
|
||||
params={"client_order_id": client_order_id},
|
||||
)
|
||||
|
||||
def get_orders(
|
||||
self, status: str = "all", limit: int = 500, nested: bool = True,
|
||||
symbols: list[str] | None = None,
|
||||
symbols: list[str] | None = None, after: str | None = None,
|
||||
) -> list[dict]:
|
||||
# nested=true includes bracket child legs (stop/target) so working risk
|
||||
# isn't under-reported in the orders snapshot.
|
||||
@@ -188,6 +213,8 @@ class AlpacaPaperClient:
|
||||
# the exit manager / safety checks must do.
|
||||
params: dict = {"status": status, "limit": min(limit, 500),
|
||||
"nested": str(nested).lower(), "direction": "desc"}
|
||||
if after: # only orders submitted after this ISO timestamp (exclusive) — the cage-count reconcile scopes to today
|
||||
params["after"] = after
|
||||
if symbols:
|
||||
# scoped to a few names — well under the cap, one page is complete
|
||||
params["symbols"] = ",".join(symbols)
|
||||
@@ -224,6 +251,7 @@ class AlpacaPaperClient:
|
||||
order_type: str = "market",
|
||||
time_in_force: str = "day",
|
||||
limit_price: float | None = None,
|
||||
stop_price: float | None = None,
|
||||
stop_loss_price: float | None = None,
|
||||
take_profit_price: float | None = None,
|
||||
client_order_id: str | None = None,
|
||||
@@ -238,6 +266,8 @@ class AlpacaPaperClient:
|
||||
}
|
||||
if limit_price is not None:
|
||||
body["limit_price"] = str(round(limit_price, 2))
|
||||
if stop_price is not None: # standalone stop / stop-limit trigger
|
||||
body["stop_price"] = str(round(stop_price, 2))
|
||||
if client_order_id:
|
||||
body["client_order_id"] = client_order_id
|
||||
if stop_loss_price is not None or take_profit_price is not None:
|
||||
@@ -329,6 +359,22 @@ class AlpacaPaperClient:
|
||||
trades.update(payload.get("trades") or {})
|
||||
return trades
|
||||
|
||||
def get_latest_quotes(self, symbols: list[str]) -> dict[str, dict]:
|
||||
"""Latest NBBO quote per symbol (bid/ask). Mirrors ``get_latest_trades``;
|
||||
each quote dict carries Alpaca's raw keys — ``bp``/``ap`` (bid/ask price)
|
||||
and ``bs``/``as`` (sizes). On the free IEX feed quotes are thinner than
|
||||
SIP, so a symbol may be absent; callers must treat a missing entry as
|
||||
"unknown spread", not a tight one."""
|
||||
quotes: dict[str, dict] = {}
|
||||
for chunk in self._chunked(symbols, 100):
|
||||
payload = self._data(
|
||||
"GET",
|
||||
"/v2/stocks/quotes/latest",
|
||||
params={"symbols": ",".join(chunk), "feed": self.settings.feed},
|
||||
)
|
||||
quotes.update(payload.get("quotes") or {})
|
||||
return quotes
|
||||
|
||||
def get_most_actives(self, top: int = 20, by: str = "volume") -> list[dict]:
|
||||
payload = self._data(
|
||||
"GET",
|
||||
@@ -337,10 +383,56 @@ class AlpacaPaperClient:
|
||||
)
|
||||
return payload.get("most_actives") or []
|
||||
|
||||
def get_assets(self, status: str = "active") -> list[dict]:
|
||||
"""All tradable assets (trading API). Used by the gapper-history reconstruction
|
||||
to enumerate the listed US-equity universe (~10-11k symbols)."""
|
||||
payload = self._trading("GET", "/assets",
|
||||
params={"status": status, "asset_class": "us_equity"})
|
||||
return payload if isinstance(payload, list) else []
|
||||
|
||||
def get_movers(self, top: int = 15) -> list[dict]:
|
||||
"""Top %-change gainers (the 'wildcard' movers). Ranked by PERCENT change, not
|
||||
share volume — a thin reverse-split name squeezing on a headline (CX on
|
||||
2026-07-02: $3->$6, the source trader's whole day, invisible to
|
||||
most-actives-by-volume) shows up here and nowhere else.
|
||||
Items: {symbol, percent_change, change, price}."""
|
||||
payload = self._data(
|
||||
"GET",
|
||||
"/v1beta1/screener/stocks/movers",
|
||||
params={"top": top},
|
||||
)
|
||||
return payload.get("gainers") or []
|
||||
|
||||
def get_news(
|
||||
self,
|
||||
symbols: list[str] | None = None,
|
||||
limit: int = 50,
|
||||
start_iso: str | None = None,
|
||||
sort: str = "desc",
|
||||
) -> list[dict]:
|
||||
"""Latest market news (Benzinga). Each item carries a stable ``id`` and an
|
||||
authoritative ``symbols`` list, so no regex ticker-scraping is needed.
|
||||
Optional symbol filter (capped at 50 symbols/req); Alpaca caps ``limit`` at 50."""
|
||||
params: dict = {
|
||||
"limit": max(1, min(int(limit), 50)),
|
||||
"sort": sort,
|
||||
"exclude_contentless": "true",
|
||||
}
|
||||
if symbols:
|
||||
params["symbols"] = ",".join(symbols[:50])
|
||||
if start_iso:
|
||||
params["start"] = start_iso
|
||||
payload = self._data("GET", "/v1beta1/news", params=params)
|
||||
return payload.get("news") or []
|
||||
|
||||
def get_daily_bars(
|
||||
self, symbols: list[str], start_iso: str, end_iso: str | None = None
|
||||
self, symbols: list[str], start_iso: str, end_iso: str | None = None,
|
||||
feed: str | None = None,
|
||||
) -> dict[str, list[dict]]:
|
||||
"""Daily bars, batching symbols (cap-safe) + following pagination."""
|
||||
"""Daily bars, batching symbols (cap-safe) + following pagination.
|
||||
|
||||
`feed`: None -> settings.feed (iex on free keys, ~2-5% of true volume).
|
||||
Pass "sip" for consolidated-tape volume (free keys allow historical SIP)."""
|
||||
all_bars: dict[str, list[dict]] = {}
|
||||
for chunk in self._chunked(symbols, 100):
|
||||
page_token = None
|
||||
@@ -354,7 +446,7 @@ class AlpacaPaperClient:
|
||||
"start": start_iso,
|
||||
"end": end_iso,
|
||||
"limit": 10_000,
|
||||
"feed": self.settings.feed,
|
||||
"feed": feed or self.settings.feed,
|
||||
"adjustment": "raw",
|
||||
"page_token": page_token,
|
||||
},
|
||||
|
||||
@@ -93,6 +93,9 @@ class AlpacaPaperExecutor:
|
||||
):
|
||||
self.store = store
|
||||
self.client = client or AlpacaPaperClient()
|
||||
# honest broker label in journaled events when a mirror client
|
||||
# (e.g. TradierClient) is injected
|
||||
self.broker_name = getattr(self.client, "broker_name", "alpaca_paper")
|
||||
self.session_id = session_id
|
||||
self.mode = mode
|
||||
|
||||
@@ -122,12 +125,47 @@ class AlpacaPaperExecutor:
|
||||
ok=False, order_id=request.order_id, error=str(exc)
|
||||
)
|
||||
except Exception as exc:
|
||||
# LOST RESPONSE: the POST may have LANDED before the connection dropped,
|
||||
# leaving a real order resting at the broker. Reconcile by the
|
||||
# client_order_id we sent before assuming it was cancelled — journalling
|
||||
# a live order as 'cancelled' is how a duplicate/naked order is born
|
||||
# (the 2026-07-06 incident). Only the generic/transport error path
|
||||
# reconciles; an AlpacaApiError above is a genuine broker rejection.
|
||||
logger.exception("alpaca order failed")
|
||||
self._emit_cancelled(request, f"submit error: {exc}")
|
||||
recovered = self._reconcile_lost_submit(request)
|
||||
if recovered is not None:
|
||||
logger.warning("recovered lost submit for %s via client_order_id "
|
||||
"(broker status=%s)", request.symbol, recovered.get("status"))
|
||||
return self._finalize(request, recovered)
|
||||
self._emit_cancelled(request, f"submit error (reconcile: not found): {exc}")
|
||||
return ExecutionResult(
|
||||
ok=False, order_id=request.order_id, error=str(exc)
|
||||
)
|
||||
|
||||
return self._finalize(request, raw)
|
||||
|
||||
_LIVE_STATUSES = {"new", "accepted", "pending_new", "partially_filled",
|
||||
"filled", "held", "accepted_for_bidding"}
|
||||
|
||||
def _reconcile_lost_submit(self, request: "ExecutionRequest") -> dict | None:
|
||||
"""Look up the order by the client_order_id we sent; return it only if it
|
||||
is actually LIVE at the broker (so a lost response isn't journaled as a
|
||||
cancel). Returns None on 404/lookup error — then the caller emits cancelled."""
|
||||
getter = getattr(self.client, "get_order_by_client_order_id", None)
|
||||
if getter is None:
|
||||
return None
|
||||
try:
|
||||
found = getter(request.order_id[:48])
|
||||
except Exception: # noqa: BLE001 — 404 (never landed) or transport error
|
||||
return None
|
||||
if found and str(found.get("status")) in self._LIVE_STATUSES:
|
||||
return found
|
||||
return None
|
||||
|
||||
def _finalize(self, request: "ExecutionRequest", raw: dict) -> ExecutionResult:
|
||||
"""Record a submission's outcome from the broker response (or a recovered
|
||||
order) and return the ExecutionResult — one event sequence for the normal
|
||||
and the reconcile-recovered paths alike."""
|
||||
broker_order_id = raw.get("id")
|
||||
status = raw.get("status")
|
||||
|
||||
@@ -147,7 +185,7 @@ class AlpacaPaperExecutor:
|
||||
correlation_id=self.session_id,
|
||||
message=(
|
||||
f"Submitted {request.side} {request.quantity} {request.symbol} "
|
||||
f"to alpaca_paper ({status})"
|
||||
f"to {self.broker_name} ({status})"
|
||||
),
|
||||
symbol=request.symbol,
|
||||
order_id=request.order_id,
|
||||
@@ -181,11 +219,13 @@ class AlpacaPaperExecutor:
|
||||
order_id=request.order_id,
|
||||
fill_price=fill_price or 0.0,
|
||||
fill_quantity=filled_qty,
|
||||
side=request.side,
|
||||
payload={
|
||||
"symbol": request.symbol,
|
||||
"order_id": request.order_id,
|
||||
"fill_price": fill_price,
|
||||
"fill_quantity": filled_qty,
|
||||
"side": request.side,
|
||||
},
|
||||
)
|
||||
)
|
||||
@@ -211,16 +251,57 @@ class AlpacaPaperExecutor:
|
||||
def cancel_entry(
|
||||
self, order_id: str, broker_order_id: str | None, symbol: str, reason: str
|
||||
) -> ExecutionResult:
|
||||
"""Cancel a resting (unfilled) entry order at the broker and record it."""
|
||||
"""Cancel a resting (unfilled) entry order at the broker and record it.
|
||||
|
||||
CANCEL-vs-FILL RACE: the entry can fill at the broker before (or as)
|
||||
the cancel lands — the cancel then no-ops but journaling only a cancel
|
||||
orphans a REAL position (untracked shares, the 2026-07-06 class;
|
||||
happened live on Tradier sandbox 2026-07-28: lazy fills + entry
|
||||
timeouts). After cancelling, re-check the broker's final word and emit
|
||||
the fill for any executed shares so the position is tracked."""
|
||||
try:
|
||||
if broker_order_id:
|
||||
self.client.cancel_order(broker_order_id)
|
||||
except AlpacaApiError as exc:
|
||||
# already filled/cancelled at the broker, or transient — log and
|
||||
# still emit the cancel so our state stops tracking it
|
||||
# continue to the fill-race check below
|
||||
logger.warning("cancel of %s returned: %s", order_id, exc)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("cancel of %s errored: %s", order_id, exc)
|
||||
|
||||
filled_qty, fill_price, side = 0, None, "buy"
|
||||
getter = getattr(self.client, "get_order_by_client_order_id", None)
|
||||
if getter is not None:
|
||||
try:
|
||||
found = getter(order_id[:48]) or {}
|
||||
filled_qty = int(float(found.get("filled_qty") or 0))
|
||||
fill_price = float(found.get("filled_avg_price") or 0) or None
|
||||
side = found.get("side") or "buy"
|
||||
except Exception: # noqa: BLE001 — lookup failure: keep cancel-only path
|
||||
pass
|
||||
|
||||
if filled_qty > 0:
|
||||
logger.warning("cancel of %s raced a fill: %d %s executed @ %s — "
|
||||
"journaling the fill", order_id, filled_qty, symbol, fill_price)
|
||||
self.store.emit(
|
||||
OrderFilledEvent(
|
||||
timestamp=datetime.now(),
|
||||
mode=self.mode,
|
||||
correlation_id=self.session_id,
|
||||
message=f"Filled {symbol} {filled_qty} @ {fill_price} "
|
||||
f"(cancel raced fill)",
|
||||
symbol=symbol,
|
||||
order_id=order_id,
|
||||
fill_price=fill_price or 0.0,
|
||||
fill_quantity=filled_qty,
|
||||
side=side,
|
||||
payload={"symbol": symbol, "order_id": order_id,
|
||||
"fill_price": fill_price, "fill_quantity": filled_qty,
|
||||
"side": side, "cancel_raced_fill": True},
|
||||
)
|
||||
)
|
||||
return ExecutionResult(ok=True, order_id=order_id, status="filled")
|
||||
|
||||
self._emit_cancelled(
|
||||
ExecutionRequest(symbol=symbol, side="buy", quantity=0, order_id=order_id),
|
||||
reason,
|
||||
|
||||
+130
-14
@@ -9,6 +9,8 @@ from __future__ import annotations
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
import json
|
||||
|
||||
from alpaca_paper.client import AlpacaPaperClient
|
||||
from storage.event_schema import (
|
||||
AccountOrdersUpdatedEvent,
|
||||
@@ -16,9 +18,16 @@ from storage.event_schema import (
|
||||
AccountSummaryUpdatedEvent,
|
||||
BrokerHealthChangedEvent,
|
||||
EventMode,
|
||||
PositionClosedEvent,
|
||||
)
|
||||
from storage.event_store import EventStore
|
||||
|
||||
# Alpaca order type -> trade-journal exit reason
|
||||
_EXIT_REASON = {
|
||||
"stop": "stop_loss", "stop_limit": "stop_loss", "trailing_stop": "trailing_stop",
|
||||
"limit": "take_profit", "market": "market_exit",
|
||||
}
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BROKER_NAME = "alpaca_paper"
|
||||
@@ -36,6 +45,17 @@ class AlpacaPaperSync:
|
||||
self.client = client or AlpacaPaperClient()
|
||||
self.session_id = session_id
|
||||
self.mode = mode
|
||||
# Label events by the ACTUAL execution broker (BROKER=tradier swaps the
|
||||
# client): the executor filters position snapshots by client.broker_name,
|
||||
# so a hardcoded "alpaca_paper" here made those queries return empty on
|
||||
# the Tradier stack (position cap / re-entry block saw zero positions).
|
||||
self.broker_name = getattr(self.client, "broker_name", BROKER_NAME)
|
||||
self.account_desc = (
|
||||
"Tradier" if self.broker_name == "tradier" else "Alpaca Paper")
|
||||
# last seen open positions, to detect closes by diffing snapshots. None
|
||||
# until the first sync, when it's seeded from the last PERSISTED snapshot
|
||||
# so a close that happened while the loop was down is still journaled.
|
||||
self._prev_positions: list[dict] | None = None
|
||||
|
||||
def _emit_health(self, reason: str) -> None:
|
||||
"""Make a broker sync failure VISIBLE (else stale snapshots look current)."""
|
||||
@@ -44,8 +64,8 @@ class AlpacaPaperSync:
|
||||
timestamp=datetime.now(),
|
||||
mode=self.mode,
|
||||
correlation_id=self.session_id,
|
||||
message=f"alpaca_paper sync degraded: {reason}",
|
||||
broker_name=BROKER_NAME,
|
||||
message=f"{self.broker_name} sync degraded: {reason}",
|
||||
broker_name=self.broker_name,
|
||||
previous_health="healthy",
|
||||
new_health="degraded",
|
||||
health_reason=reason,
|
||||
@@ -65,10 +85,10 @@ class AlpacaPaperSync:
|
||||
timestamp=datetime.now(),
|
||||
mode=self.mode,
|
||||
correlation_id=self.session_id,
|
||||
message=f"Alpaca paper equity {account.get('equity')}",
|
||||
broker_name=BROKER_NAME,
|
||||
message=f"{self.account_desc} equity {account.get('equity')}",
|
||||
broker_name=self.broker_name,
|
||||
account_id=account_id,
|
||||
account_desc="Alpaca Paper",
|
||||
account_desc=self.account_desc,
|
||||
total_equity=float(account.get("equity") or 0),
|
||||
cash_balance=float(account.get("cash") or 0),
|
||||
buying_power=float(account.get("buying_power") or 0),
|
||||
@@ -103,8 +123,8 @@ class AlpacaPaperSync:
|
||||
timestamp=datetime.now(),
|
||||
mode=self.mode,
|
||||
correlation_id=self.session_id,
|
||||
message=f"Alpaca paper positions: {len(positions)}",
|
||||
broker_name=BROKER_NAME,
|
||||
message=f"{self.account_desc} positions: {len(positions)}",
|
||||
broker_name=self.broker_name,
|
||||
account_id="paper",
|
||||
positions=positions,
|
||||
)
|
||||
@@ -153,17 +173,113 @@ class AlpacaPaperSync:
|
||||
timestamp=datetime.now(),
|
||||
mode=self.mode,
|
||||
correlation_id=self.session_id,
|
||||
message=f"Alpaca paper orders: {len(orders)}",
|
||||
broker_name=BROKER_NAME,
|
||||
message=f"{self.account_desc} orders: {len(orders)}",
|
||||
broker_name=self.broker_name,
|
||||
account_id="paper",
|
||||
orders=orders,
|
||||
)
|
||||
)
|
||||
return orders
|
||||
|
||||
def _load_last_persisted_positions(self) -> list[dict]:
|
||||
"""The most recent persisted positions snapshot (prior session/run), so
|
||||
the first reconcile after a restart can still catch a close that happened
|
||||
while we were down. [] if none / on any fault."""
|
||||
try:
|
||||
rows = self.store.con.execute(
|
||||
"SELECT payload_json FROM events "
|
||||
"WHERE event_type = 'account_positions_updated' "
|
||||
"ORDER BY timestamp DESC, created_at DESC LIMIT 1"
|
||||
).fetchall()
|
||||
if rows:
|
||||
return json.loads(rows[0][0] or "{}").get("positions") or []
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return []
|
||||
|
||||
def _find_exit(self, symbol: str, is_short: bool, orders: list[dict]):
|
||||
"""(exit_price, exit_reason, stop_level, exit_order_id) for a now-closed
|
||||
symbol, read from the filled exit leg in the orders snapshot. The exit of
|
||||
a long is a filled SELL (a short, a filled BUY). Also surfaces the bracket
|
||||
stop level (filled or not) for the R-multiple. Any None -> caller falls
|
||||
back to the last mark."""
|
||||
exit_side = "buy" if is_short else "sell"
|
||||
stop_level = None
|
||||
fills = []
|
||||
for o in orders:
|
||||
if o.get("symbol") != symbol:
|
||||
continue
|
||||
otype = str(o.get("type") or "")
|
||||
if otype in ("stop", "stop_limit") and o.get("stop_price") not in (None, ""):
|
||||
try:
|
||||
stop_level = float(o.get("stop_price"))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if (str(o.get("side") or "").lower() == exit_side
|
||||
and str(o.get("status") or "").lower() == "filled"
|
||||
and o.get("filled_avg_price") not in (None, "")):
|
||||
fills.append(o)
|
||||
if not fills:
|
||||
return None, None, stop_level, None
|
||||
fills.sort(key=lambda o: str(o.get("submitted_at") or ""), reverse=True)
|
||||
f = fills[0]
|
||||
try:
|
||||
px = float(f.get("filled_avg_price"))
|
||||
except (TypeError, ValueError):
|
||||
px = None
|
||||
reason = _EXIT_REASON.get(str(f.get("type") or ""), "closed")
|
||||
return px, reason, stop_level, f.get("broker_order_id")
|
||||
|
||||
def reconcile_closed(self, prev_positions, new_positions, orders) -> int:
|
||||
"""Emit a position_closed event for every symbol that was open in the
|
||||
previous snapshot and is gone (or flat) in the new one. This is the ONLY
|
||||
place the trade journal learns about exits — bracket stops/targets fill
|
||||
broker-side and EOD flatten closes via market orders, so a snapshot diff
|
||||
is the one signal that catches them all. Realized $ stays broker-
|
||||
authoritative in the P&L strip; these events feed wins/losses/avg-R/the
|
||||
trade list. Returns the number emitted."""
|
||||
new_open = {p.get("symbol") for p in (new_positions or [])
|
||||
if abs(float(p.get("quantity") or 0)) > 0}
|
||||
emitted = 0
|
||||
for prev in prev_positions or []:
|
||||
sym = prev.get("symbol")
|
||||
qty = abs(float(prev.get("quantity") or 0))
|
||||
if not sym or qty <= 0 or sym in new_open:
|
||||
continue
|
||||
entry = float(prev.get("avg_entry_price") or 0)
|
||||
is_short = str(prev.get("side") or "long").lower() == "short"
|
||||
exit_price, reason, stop_level, oid = self._find_exit(sym, is_short, orders or [])
|
||||
if exit_price is None: # no fill seen yet -> last mark
|
||||
cp = prev.get("current_price")
|
||||
exit_price = float(cp) if cp not in (None, "") else entry
|
||||
reason = reason or "closed"
|
||||
realized = (entry - exit_price) * qty if is_short else (exit_price - entry) * qty
|
||||
ts = datetime.now()
|
||||
self.store.emit(PositionClosedEvent(
|
||||
timestamp=ts, mode=self.mode, correlation_id=self.session_id,
|
||||
message=f"{sym} closed @ {exit_price:.4f} ({reason}) pnl={realized:+.2f}",
|
||||
position_id=str(oid or f"{sym}-{ts.isoformat()}"),
|
||||
symbol=sym, exit_price=round(exit_price, 4), exit_reason=reason,
|
||||
realized_pnl=round(realized, 2), entry_price=round(entry, 4),
|
||||
stop_loss_price=round(stop_level, 4) if stop_level else None,
|
||||
side=("sell" if is_short else "buy"), quantity=qty,
|
||||
))
|
||||
emitted += 1
|
||||
return emitted
|
||||
|
||||
def sync_all(self) -> dict:
|
||||
return {
|
||||
"account": self.sync_account(),
|
||||
"positions": self.sync_positions(),
|
||||
"orders": self.sync_orders(),
|
||||
}
|
||||
# seed the close-detection baseline from the last persisted snapshot
|
||||
# BEFORE this cycle emits a fresh one (else prev == current => no diff).
|
||||
if self._prev_positions is None:
|
||||
self._prev_positions = self._load_last_persisted_positions()
|
||||
account = self.sync_account()
|
||||
positions = self.sync_positions()
|
||||
orders = self.sync_orders()
|
||||
# only reconcile on a good positions read; keep the last baseline otherwise
|
||||
if positions is not None:
|
||||
try:
|
||||
self.reconcile_closed(self._prev_positions, positions, orders or [])
|
||||
except Exception as exc: # noqa: BLE001 — never let journaling break sync
|
||||
logger.warning("position-close reconcile failed: %s", exc)
|
||||
self._prev_positions = positions
|
||||
return {"account": account, "positions": positions, "orders": orders}
|
||||
+646
-15
@@ -8,10 +8,17 @@ GET /api/stream Server-Sent Events: pushes snapshots on
|
||||
GET /api/approval-queue pending approvals only
|
||||
GET /api/criteria?symbol=XYZ per-criterion pass/fail breakdown
|
||||
GET /api/bars?symbol=XYZ[&minutes=60] recent 1-min bars for a sparkline
|
||||
GET /api/chart?symbol=XYZ[&hours=24][&date=YYYY-MM-DD] candles + fill markers + levels;
|
||||
date= anchors the window to that session (else "now")
|
||||
GET /api/decisions[?symbol=&limit=&date=] entry_gate_evaluated decision receipts (plan 014)
|
||||
GET /backtest theoretical-replay dashboard HTML (plan 015)
|
||||
GET /api/backtest[?days=10|since=&until=] theoretical trades under CURRENT settings (plan 015)
|
||||
GET /static/vendor/... vendored JS libs (e.g. Lightweight Charts)
|
||||
POST /api/trading/approvals/approve {"order_id": ..., "approved_by"?: ...}
|
||||
POST /api/trading/approvals/reject {"order_id": ..., "reason"?: ...}
|
||||
POST /api/trading/exit-order {"symbol": ...}
|
||||
POST /api/watch/add {"symbol": ...} (manual watchlist inject)
|
||||
POST /api/command {"command": ..., "symbol"?: ...} (operator kill switch; requires DASHBOARD_TOKEN)
|
||||
|
||||
The server holds an EventStore for reads; trade actions route through a
|
||||
TradingExecutionService when one is attached (the orchestrator does this).
|
||||
@@ -21,23 +28,32 @@ dashboard still works read-only.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from storage.event_schema import EventMode, OperatorCommandEvent, OPERATOR_COMMANDS
|
||||
from storage.event_store import EventStore
|
||||
from storage.projections import (
|
||||
query_account_orders_snapshot,
|
||||
query_account_positions_snapshot,
|
||||
query_account_summary_snapshot,
|
||||
query_alltime_score,
|
||||
query_approval_queue,
|
||||
query_catalyst_advisory,
|
||||
query_catalyst_feed,
|
||||
query_daily_performance,
|
||||
query_fills_feed,
|
||||
query_gate_receipts,
|
||||
query_armed_triggers,
|
||||
query_loop_session,
|
||||
query_equity_curve,
|
||||
query_order_lifecycle_snapshot,
|
||||
query_ready_signals_snapshot,
|
||||
@@ -48,9 +64,46 @@ from storage.projections import (
|
||||
query_watch_states_snapshot,
|
||||
)
|
||||
|
||||
import logging
|
||||
|
||||
from research.multi_schema import open_research_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
STATIC_DIR = Path(__file__).parent / "static"
|
||||
|
||||
|
||||
def _command_bus_enabled() -> bool:
|
||||
"""The operator command bus is live only when a token guards it AND it is
|
||||
explicitly enabled — so a stray COMMAND_BUS_ENABLED never opens an
|
||||
unauthenticated kill switch."""
|
||||
return bool(os.environ.get("DASHBOARD_TOKEN")) and (
|
||||
os.environ.get("COMMAND_BUS_ENABLED", "").strip().lower()
|
||||
in {"1", "true", "yes", "on"}
|
||||
)
|
||||
|
||||
|
||||
def _broker_block() -> dict:
|
||||
"""Broker truth for the LIVE badge, derived from the base URL the services
|
||||
already have (not the execution_mode literal, which lied). Alpaca's paper host
|
||||
is ``paper-api.alpaca.markets``; Tradier's sandbox host starts with
|
||||
``sandbox.``; anything else is live money."""
|
||||
from urllib.parse import urlparse as _urlparse
|
||||
|
||||
if os.environ.get("BROKER", "alpaca").strip().lower() == "tradier":
|
||||
url = os.environ.get("TRADIER_BASE_URL") or (
|
||||
"https://api.tradier.com/v1"
|
||||
if os.environ.get("TRADIER_ENV", "sandbox").strip().lower() == "live"
|
||||
else "https://sandbox.tradier.com/v1")
|
||||
host = _urlparse(url).hostname or url
|
||||
return {"name": "TRADIER", "base_url_host": host,
|
||||
"is_paper": host.startswith("sandbox.")}
|
||||
url = os.environ.get("ALPACA_BASE_URL") or "https://paper-api.alpaca.markets"
|
||||
host = _urlparse(url).hostname or url
|
||||
return {"name": "ALPACA", "base_url_host": host,
|
||||
"is_paper": host.startswith("paper-api")}
|
||||
|
||||
|
||||
def _eastern_session_date():
|
||||
"""Today's date in US/Eastern — matches the ingestion pipeline's flags."""
|
||||
from zoneinfo import ZoneInfo
|
||||
@@ -58,6 +111,19 @@ def _eastern_session_date():
|
||||
return datetime.now(ZoneInfo("America/New_York")).date()
|
||||
|
||||
|
||||
def _to_epoch_et(ts) -> int:
|
||||
"""Naive US/Eastern timestamp (datetime or ISO string) -> UTC epoch seconds.
|
||||
|
||||
Event timestamps (and ``minute_bars``) are stored naive-ET; both /api/chart
|
||||
(plan 013) and /api/decisions (plan 014) hand the browser epoch seconds so
|
||||
Lightweight Charts — and the JS merging the two marker sets — never does
|
||||
timezone math."""
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
dt = ts if isinstance(ts, datetime) else datetime.fromisoformat(str(ts))
|
||||
return int(dt.replace(tzinfo=ZoneInfo("America/New_York")).timestamp())
|
||||
|
||||
|
||||
def _available_dates(store) -> list[str]:
|
||||
"""Distinct session dates with account data — populates the date picker so
|
||||
you can switch from LIVE to review a past day."""
|
||||
@@ -72,6 +138,117 @@ def _available_dates(store) -> list[str]:
|
||||
return []
|
||||
|
||||
|
||||
def _catalyst_engine(store, catalyst: dict) -> dict:
|
||||
"""Catalyst-engine status for the dashboard: config (read from the same .env)
|
||||
plus live activity (advisory + dilution counts, and how many entries the
|
||||
dilution veto has blocked today)."""
|
||||
try:
|
||||
from config import OllamaConfig
|
||||
cfg = OllamaConfig.from_env()
|
||||
on = bool(cfg.enabled and cfg.enrichment_enabled)
|
||||
model = cfg.model
|
||||
score_blend = bool(cfg.catalyst_score_enabled)
|
||||
veto = bool(cfg.dilution_veto_enabled)
|
||||
except Exception: # noqa: BLE001
|
||||
on, model, score_blend, veto = False, "", False, False
|
||||
adv = catalyst or {}
|
||||
dilutive = sum(1 for a in adv.values() if isinstance(a, dict) and a.get("is_dilutive"))
|
||||
vetoes = 0
|
||||
try:
|
||||
row = store.con.execute(
|
||||
"SELECT count(*) FROM events WHERE event_type='risk_rule_triggered' "
|
||||
"AND timestamp::date = CURRENT_DATE "
|
||||
"AND payload_json LIKE '%catalyst_dilution_veto%'"
|
||||
).fetchone()
|
||||
vetoes = int(row[0]) if row and row[0] else 0
|
||||
except Exception: # noqa: BLE001
|
||||
vetoes = 0
|
||||
return {"enabled": on, "model": model, "score_blend": score_blend,
|
||||
"dilution_veto": veto, "advisories": len(adv),
|
||||
"dilutive": dilutive, "vetoes_today": vetoes}
|
||||
|
||||
|
||||
class _ChangeSignal:
|
||||
"""One process-wide event-change signal shared by ALL SSE clients.
|
||||
|
||||
Before this, every SSE client ran ``COUNT(*)`` on a FRESH EventStore
|
||||
connection every ~1s (re-applying the 473-line schema unless
|
||||
``PG_SKIP_SCHEMA=1``). Now a single long-lived connection polls
|
||||
``max(timestamp)`` (an index-tail seek) at most once per second GLOBALLY,
|
||||
behind a lock — N clients cost one query/sec, not N. ``reader`` is injectable
|
||||
so tests can count underlying queries without a DB.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str | None = None, *, reader=None, min_interval: float = 1.0):
|
||||
self._db_path = db_path
|
||||
self._reader = reader
|
||||
self._min_interval = float(min_interval)
|
||||
self._lock = threading.Lock()
|
||||
self._store = None
|
||||
self._stamp = None
|
||||
self._checked_at = 0.0
|
||||
self.query_count = 0 # test hook: number of underlying reads performed
|
||||
|
||||
def _read(self):
|
||||
self.query_count += 1
|
||||
if self._reader is not None:
|
||||
return self._reader()
|
||||
if self._store is None:
|
||||
self._store = EventStore(self._db_path) # long-lived; schema skipped via PG_SKIP_SCHEMA
|
||||
return self._store.max_timestamp()
|
||||
|
||||
def current(self):
|
||||
"""Latest change token (max event timestamp), refreshed at most once per
|
||||
``min_interval`` seconds across all callers."""
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
if self._checked_at == 0.0 or (now - self._checked_at) >= self._min_interval:
|
||||
try:
|
||||
self._stamp = self._read()
|
||||
except Exception: # noqa: BLE001 — a transient read must not kill the stream
|
||||
pass
|
||||
self._checked_at = now
|
||||
return self._stamp
|
||||
|
||||
|
||||
def _maybe_json(v):
|
||||
"""JSONB columns come back from psycopg2 as str OR already-parsed dict; make it
|
||||
a dict either way so the ledger never double-encodes."""
|
||||
if isinstance(v, str):
|
||||
try:
|
||||
return json.loads(v)
|
||||
except (ValueError, TypeError):
|
||||
return {}
|
||||
return v or {}
|
||||
|
||||
|
||||
def _experiment_ledger(store) -> list[dict]:
|
||||
"""Latest verdict per experiment for the ledger panel (plan 006 tables).
|
||||
|
||||
Returns [] on ANY error — the experiments/experiment_verdicts tables live in a
|
||||
separate schema file applied by the registry, so on a bare/test database they
|
||||
simply don't exist and the panel shows its empty state."""
|
||||
try:
|
||||
rows = store.con.execute(
|
||||
"SELECT v.experiment_id, v.as_of, v.n, v.verdict, v.metrics, "
|
||||
" e.hypothesis, e.bars, e.status "
|
||||
"FROM (SELECT DISTINCT ON (experiment_id) experiment_id, as_of, n, verdict, metrics "
|
||||
" FROM experiment_verdicts ORDER BY experiment_id, as_of DESC) v "
|
||||
"LEFT JOIN experiments e ON e.id = v.experiment_id "
|
||||
"ORDER BY v.as_of DESC"
|
||||
).fetchall()
|
||||
except Exception: # noqa: BLE001 — tables absent on a bare DB
|
||||
return []
|
||||
out = []
|
||||
for r in rows:
|
||||
out.append({
|
||||
"experiment": r[0], "as_of": str(r[1]), "n": r[2], "verdict": r[3],
|
||||
"metrics": _maybe_json(r[4]), "hypothesis": r[5],
|
||||
"bars": _maybe_json(r[6]), "status": r[7],
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
class DashboardState:
|
||||
"""Shared state: event store path + optional live execution service."""
|
||||
|
||||
@@ -90,6 +267,41 @@ class DashboardState:
|
||||
self.research_con = research_con
|
||||
self.watch_inject = watch_inject
|
||||
self._lock = threading.Lock()
|
||||
self._research_lock = threading.Lock()
|
||||
# lazily-built, process-wide SSE change signal (shared by all clients)
|
||||
self._change_signal: _ChangeSignal | None = None
|
||||
# backtest replay cache (plan 015) — a DEDICATED lock, not self._lock,
|
||||
# so a slow replay (10-30s) never blocks the live snapshot/SSE path.
|
||||
# key = (since, until, repr(settings)) -> (result, monotonic timestamp).
|
||||
self._backtest_lock = threading.Lock()
|
||||
self._backtest_cache: dict = {}
|
||||
|
||||
def _research(self):
|
||||
"""The research DB connection, opened LAZILY and self-healing. It used
|
||||
to be opened once at boot; if the DB was unreachable then (container
|
||||
started before Postgres, or a blip), it stayed None forever and every
|
||||
chart/sparkline/backtest returned 'no research db attached' — which
|
||||
read as broken TradingView charts when the real cause was an empty data
|
||||
pipe. Now a None connection is (re)opened on demand; the facade itself
|
||||
auto-reconnects a dropped-but-open one."""
|
||||
if self.research_con is not None:
|
||||
return self.research_con
|
||||
with self._research_lock:
|
||||
if self.research_con is None:
|
||||
try:
|
||||
self.research_con = open_research_db("market")
|
||||
except Exception as exc: # noqa: BLE001 — degrade to None, retry next call
|
||||
logger.warning("research DB open failed (charts empty): %s", exc)
|
||||
self.research_con = None
|
||||
return self.research_con
|
||||
|
||||
def change_stamp(self):
|
||||
"""SSE change token from the shared once-per-second signal (see _ChangeSignal)."""
|
||||
if self._change_signal is None:
|
||||
with self._lock:
|
||||
if self._change_signal is None:
|
||||
self._change_signal = _ChangeSignal(self.event_db_path)
|
||||
return self._change_signal.current()
|
||||
|
||||
def open_store(self) -> EventStore:
|
||||
return EventStore(self.event_db_path)
|
||||
@@ -113,13 +325,29 @@ class DashboardState:
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"execution_mode": self.execution_mode,
|
||||
"has_execution": self.execution_service is not None,
|
||||
# broker truth (LIVE badge) + operator command-bus availability
|
||||
"broker": _broker_block(),
|
||||
"command_bus": _command_bus_enabled(),
|
||||
# experiment ledger (plan 006 tables) — [] until they exist
|
||||
"experiments": _experiment_ledger(store) if is_live else [],
|
||||
"view_date": None if is_live else for_date,
|
||||
"available_dates": _available_dates(store),
|
||||
"pnl": query_session_pnl(store, for_date=for_date),
|
||||
# all-time scorecard (every closed round-trip, not date-scoped)
|
||||
"alltime": query_alltime_score(store),
|
||||
# one point per market day for the day-over-day comparison curve
|
||||
"daily": query_daily_performance(store),
|
||||
"risk": query_risk_state(store, for_date=for_date),
|
||||
"triggers": query_armed_triggers(store) if is_live else {"triggers": [], "armed": 0},
|
||||
# loop session identity + day-2 staleness alarm (memory-only
|
||||
# daily flags survive midnight only to silently block entries)
|
||||
"loop_session": query_loop_session(store) if is_live else {},
|
||||
"equity_curve": query_equity_curve(store, for_date=for_date),
|
||||
"catalyst": catalyst,
|
||||
"catalyst_engine": _catalyst_engine(store, catalyst) if is_live else {},
|
||||
# The raw stream of what the LLM read + decided (newest first)
|
||||
# so the operator can see the WHY, not just the engine totals.
|
||||
"catalyst_feed": query_catalyst_feed(store) if is_live else [],
|
||||
"approval_queue": approval_queue,
|
||||
"ready_signals": query_ready_signals_snapshot(store) if is_live else [],
|
||||
# watch_states is a LIVE concept (and the heaviest projection
|
||||
@@ -167,22 +395,23 @@ class DashboardState:
|
||||
pipeline). If today has no bars yet (pre-market, weekend, or replay of
|
||||
an older session), falls back to the most recent session that does.
|
||||
"""
|
||||
if self.research_con is None:
|
||||
con = self._research()
|
||||
if con is None:
|
||||
return {"symbol": symbol, "bars": [], "reason": "no research db attached"}
|
||||
from research.query import query_minute_bars
|
||||
|
||||
with self._lock:
|
||||
try:
|
||||
session_date = _eastern_session_date()
|
||||
df = query_minute_bars(self.research_con, symbol, session_date)
|
||||
df = query_minute_bars(con, symbol, session_date)
|
||||
if df is None or df.empty:
|
||||
# fall back to the latest session with bars for this symbol
|
||||
row = self.research_con.execute(
|
||||
row = con.execute(
|
||||
"SELECT MAX(session_date) FROM minute_bars WHERE symbol = ?",
|
||||
[symbol],
|
||||
).fetchone()
|
||||
if row and row[0] is not None:
|
||||
df = query_minute_bars(self.research_con, symbol, row[0])
|
||||
df = query_minute_bars(con, symbol, row[0])
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {"symbol": symbol, "bars": [], "reason": str(exc)}
|
||||
if df is None or df.empty:
|
||||
@@ -201,6 +430,266 @@ class DashboardState:
|
||||
]
|
||||
return {"symbol": symbol, "bars": bars}
|
||||
|
||||
def chart(self, symbol: str, hours: int = 24, anchor_date: str | None = None) -> dict:
|
||||
"""24h candlestick bars + trade markers + entry/stop/target levels for
|
||||
the symbol drill-in chart (plan 013). Bars come from the research DB;
|
||||
markers from the fills feed; levels from the same account snapshots
|
||||
``snapshots()`` already loads. Timestamps in ``minute_bars``/``events``
|
||||
are naive US/Eastern — every timestamp returned here is a UTC epoch
|
||||
second (what Lightweight Charts wants).
|
||||
|
||||
``anchor_date`` (plan 015): anchors the window to that session's close
|
||||
instead of "now" — the live modal never passes it (unchanged
|
||||
behavior), but the backtest replay's drill-in needs bars from
|
||||
arbitrary past sessions the plain ``hours``-back-from-now window can
|
||||
never reach (it's capped at 72h).
|
||||
"""
|
||||
con = self._research()
|
||||
if con is None:
|
||||
return {"symbol": symbol, "bars": [], "reason": "no research db attached"}
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
et = ZoneInfo("America/New_York")
|
||||
hours = max(1, min(int(hours or 24), 72))
|
||||
if anchor_date:
|
||||
d = date.fromisoformat(str(anchor_date))
|
||||
now_et = datetime(d.year, d.month, d.day, 20, 0, tzinfo=et) # well after the close
|
||||
else:
|
||||
now_et = datetime.now(et)
|
||||
cutoff_et = now_et - timedelta(hours=hours)
|
||||
cutoff_naive = cutoff_et.replace(tzinfo=None) # naive ET, matches storage
|
||||
cutoff_epoch = int(cutoff_et.timestamp())
|
||||
to_epoch = _to_epoch_et
|
||||
|
||||
with self._lock:
|
||||
try:
|
||||
rows = con.execute(
|
||||
"SELECT timestamp, open, high, low, close, volume "
|
||||
"FROM minute_bars WHERE symbol = ? AND timestamp >= ? "
|
||||
"ORDER BY timestamp",
|
||||
[symbol, cutoff_naive],
|
||||
).fetchall()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {"symbol": symbol, "bars": [], "reason": str(exc)}
|
||||
|
||||
store = self.open_store()
|
||||
try:
|
||||
raw_fills = query_fills_feed(store, limit=200, symbol=symbol)
|
||||
positions = query_account_positions_snapshot(store)
|
||||
orders = query_account_orders_snapshot(store)
|
||||
try: # exits for the marker loop (store is open only in here)
|
||||
closed_events = store.query_events(event_type="position_closed", limit=200)
|
||||
except Exception: # noqa: BLE001
|
||||
closed_events = []
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
bars = [
|
||||
{"time": to_epoch(r[0]), "open": float(r[1]), "high": float(r[2]),
|
||||
"low": float(r[3]), "close": float(r[4]), "volume": int(r[5] or 0)}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
# OrderFilledEvent carries no `side` field (schema gap, not this plan's
|
||||
# to fix — alpaca_paper/execution.py is out of scope) — backfill a
|
||||
# fill's side from its sibling order_submitted row via order_id so the
|
||||
# marker can still be colored buy/sell.
|
||||
side_by_order = {r["order_id"]: r["side"] for r in raw_fills if r.get("side")}
|
||||
markers = []
|
||||
for r in raw_fills:
|
||||
ts = r.get("timestamp")
|
||||
if not ts:
|
||||
continue
|
||||
epoch = to_epoch(ts)
|
||||
if epoch < cutoff_epoch:
|
||||
continue
|
||||
markers.append({
|
||||
"time": epoch,
|
||||
"side": r.get("side") or side_by_order.get(r.get("order_id")),
|
||||
"kind": r.get("kind"),
|
||||
"price": r.get("price"),
|
||||
"quantity": r.get("quantity"),
|
||||
})
|
||||
# EXIT markers — the fills feed carries entries (order_submitted); exits
|
||||
# live in position_closed (the loop fills brackets via position-diffing,
|
||||
# so no order_filled ever fires). Without this the chart showed entries
|
||||
# but never where a trade closed. kind='exit' so the UI can mark it
|
||||
# distinctly (X / down-triangle) with the exit reason + P&L.
|
||||
for e in closed_events:
|
||||
p = json.loads(e.get("payload_json", "{}"))
|
||||
if p.get("symbol") != symbol:
|
||||
continue
|
||||
ts = e.get("timestamp")
|
||||
epoch = to_epoch(ts) if ts else None
|
||||
if epoch is None or epoch < cutoff_epoch:
|
||||
continue
|
||||
markers.append({
|
||||
"time": epoch, "kind": "exit", "side": "sell",
|
||||
"price": p.get("exit_price"), "quantity": p.get("quantity"),
|
||||
"exit_reason": p.get("exit_reason"),
|
||||
"pnl": p.get("realized_pnl"),
|
||||
})
|
||||
markers.sort(key=lambda m: m["time"])
|
||||
|
||||
levels = {"avg_entry": None, "stop": None, "target": None, "entry_limit": None}
|
||||
for snap in positions:
|
||||
for pos in snap.get("positions", []):
|
||||
if pos.get("symbol") != symbol:
|
||||
continue
|
||||
avg = pos.get("avg_entry_price")
|
||||
if avg is None:
|
||||
avg = pos.get("average_price")
|
||||
if avg is not None:
|
||||
levels["avg_entry"] = float(avg)
|
||||
working = {"new", "accepted", "pending_new", "partially_filled"}
|
||||
for snap in orders:
|
||||
for o in snap.get("orders", []):
|
||||
if o.get("symbol") != symbol or o.get("status") not in working:
|
||||
continue
|
||||
if (o.get("side") == "sell" and o.get("type") in ("stop", "stop_limit")
|
||||
and o.get("stop_price") is not None and levels["stop"] is None):
|
||||
levels["stop"] = float(o["stop_price"])
|
||||
elif (o.get("side") == "sell" and o.get("type") == "limit"
|
||||
and o.get("limit_price") is not None and levels["target"] is None):
|
||||
levels["target"] = float(o["limit_price"])
|
||||
elif (o.get("side") == "buy" and o.get("type") == "limit"
|
||||
and o.get("limit_price") is not None and levels["entry_limit"] is None):
|
||||
levels["entry_limit"] = float(o["limit_price"])
|
||||
|
||||
return {"symbol": symbol, "bars": bars, "markers": markers, "levels": levels}
|
||||
|
||||
def decisions(self, symbol: str | None = None, limit: int = 100,
|
||||
for_date: str | None = None) -> list[dict]:
|
||||
"""Live decision-trace feed (plan 014): entry_gate_evaluated receipts
|
||||
with a server-computed ``epoch`` per row (naive-ET storage; the chart
|
||||
overlay and the panel never do timezone math client-side)."""
|
||||
with self._lock:
|
||||
store = self.open_store()
|
||||
try:
|
||||
rows = query_gate_receipts(store, symbol=symbol, limit=limit,
|
||||
for_date=for_date)
|
||||
finally:
|
||||
store.close()
|
||||
for row in rows:
|
||||
ts = row.get("timestamp")
|
||||
row["epoch"] = _to_epoch_et(ts) if ts else None
|
||||
return rows
|
||||
|
||||
def backtest(self, days: int | None = 10, since: str | None = None,
|
||||
until: str | None = None, regime_override: bool | None = None) -> dict:
|
||||
"""Theoretical trades under CURRENT live settings (plan 015) —
|
||||
assembly only, the simulation lives in research.experiments.replay.
|
||||
|
||||
``days`` (default path): the most recent N trading sessions with
|
||||
setup_events, counted backwards. ``since``/``until``: an explicit
|
||||
date range instead. Either way capped at 45 sessions (``clamped``
|
||||
reports when the request exceeded it). Cached 10 min behind a
|
||||
DEDICATED lock (not self._lock) so a slow replay never blocks the
|
||||
live snapshot/SSE path — two concurrent identical requests compute
|
||||
the replay exactly once.
|
||||
"""
|
||||
from strategy.exits import ExitConfig
|
||||
from strategy.population import PopulationConfig
|
||||
|
||||
# cap protects the (serialized) request thread; 300 spans the full stored
|
||||
# minute-bar history (~273 sessions) so months-scale ranges work via
|
||||
# since/until — a full-history request runs minutes, operator's choice.
|
||||
MAX_SESSIONS = int(os.environ.get("BACKTEST_MAX_SESSIONS", "300"))
|
||||
pop = PopulationConfig.from_env()
|
||||
regime_on = os.environ.get("TRADING_REGIME_GATE_ENABLED", "0").strip().lower() in {
|
||||
"1", "true", "yes", "on"}
|
||||
if regime_override is not None:
|
||||
# /backtest UX: let the page show RAW-criteria trades when the gate
|
||||
# sits out the whole window (owner: 'no data in backtest').
|
||||
regime_on = regime_override
|
||||
fidelity = {"atr_frac": 0.25, "spread_mult": 3, "adverse_ticks": 1}
|
||||
settings = {
|
||||
"population": {"price_min": pop.price_min, "price_max": pop.price_max,
|
||||
"min_entry_gap_pct": pop.min_entry_gap_pct,
|
||||
"min_day_volume": pop.min_day_volume},
|
||||
"regime_gate": regime_on,
|
||||
"fidelity": fidelity,
|
||||
"target_r": ExitConfig.from_env().target_r,
|
||||
}
|
||||
empty = {"trades": [], "daily": [], "skipped": 0,
|
||||
"summary": {"n": 0, "sum_r": 0.0, "avg_r": 0.0, "hit_rate": 0.0, "sessions": 0},
|
||||
"screened": {"population": 0, "regime": 0},
|
||||
"live_split": {"taken": {"n": 0, "sum_r": 0.0, "avg_r": 0.0},
|
||||
"not_taken": {"n": 0, "sum_r": 0.0, "avg_r": 0.0}}}
|
||||
|
||||
store = self.open_store()
|
||||
try:
|
||||
base_q = ("SELECT DISTINCT session_date FROM setup_events WHERE "
|
||||
"entry_reference_price IS NOT NULL AND invalidation_price IS NOT NULL")
|
||||
if since and until:
|
||||
rows = store.con.execute(
|
||||
base_q + " AND session_date >= ? AND session_date <= ? "
|
||||
"ORDER BY session_date DESC", [since, until]).fetchall()
|
||||
all_dates = sorted(r[0] for r in rows)
|
||||
clamped = len(all_dates) > MAX_SESSIONS
|
||||
dates = all_dates[-MAX_SESSIONS:] if clamped else all_dates
|
||||
sessions_requested = len(all_dates)
|
||||
else:
|
||||
requested = int(days or 10)
|
||||
clamped = requested > MAX_SESSIONS
|
||||
n_sessions = min(max(requested, 1), MAX_SESSIONS)
|
||||
rows = store.con.execute(
|
||||
base_q + " ORDER BY session_date DESC LIMIT ?", [n_sessions]).fetchall()
|
||||
dates = sorted(r[0] for r in rows)
|
||||
sessions_requested = n_sessions
|
||||
|
||||
if not dates:
|
||||
return {**empty, "settings": settings, "clamped": clamped,
|
||||
"sessions_requested": sessions_requested}
|
||||
win_since, win_until = str(dates[0]), str(dates[-1])
|
||||
|
||||
key = (win_since, win_until, repr(settings))
|
||||
with self._backtest_lock:
|
||||
cached = self._backtest_cache.get(key)
|
||||
if cached and (time.monotonic() - cached[1]) < 600:
|
||||
result = cached[0]
|
||||
else:
|
||||
from research.experiments.replay import replay_window
|
||||
result = replay_window(store, win_since, win_until, population_cfg=pop,
|
||||
regime_gate=regime_on, fidelity=fidelity)
|
||||
for t in result["trades"]:
|
||||
t["entry_epoch"] = _to_epoch_et(t["entry_ts"])
|
||||
t["exit_epoch"] = _to_epoch_et(t["exit_ts"])
|
||||
self._backtest_cache[key] = (result, time.monotonic())
|
||||
|
||||
# Live-taken vs skipped split (2026-07-25): which replay fires did the
|
||||
# live loop actually trade? Closes for past sessions are immutable, so
|
||||
# re-annotating the cached result on every request is idempotent.
|
||||
rows = store.con.execute(
|
||||
"SELECT timestamp, payload_json FROM events "
|
||||
"WHERE event_type = 'position_closed' AND timestamp >= ? AND timestamp < ?",
|
||||
[win_since,
|
||||
(date.fromisoformat(win_until) + timedelta(days=1)).isoformat()],
|
||||
).fetchall()
|
||||
live_pnl: dict = {}
|
||||
for ts, pj in rows:
|
||||
p = pj if isinstance(pj, dict) else json.loads(pj or "{}")
|
||||
sym = p.get("symbol")
|
||||
if sym and p.get("realized_pnl") is not None:
|
||||
k = (str(ts.date()), sym)
|
||||
live_pnl[k] = round(live_pnl.get(k, 0.0) + float(p["realized_pnl"]), 2)
|
||||
live_split = {"taken": {"n": 0, "sum_r": 0.0},
|
||||
"not_taken": {"n": 0, "sum_r": 0.0}}
|
||||
for t in result["trades"]:
|
||||
pnl = live_pnl.get((str(t["session_date"]), t["symbol"]))
|
||||
t["live_pnl"] = pnl
|
||||
grp = live_split["taken" if pnl is not None else "not_taken"]
|
||||
grp["n"] += 1
|
||||
grp["sum_r"] += t["r_multiple"]
|
||||
for grp in live_split.values():
|
||||
grp["sum_r"] = round(grp["sum_r"], 3)
|
||||
grp["avg_r"] = round(grp["sum_r"] / grp["n"], 3) if grp["n"] else 0.0
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
return {**result, "settings": settings, "clamped": clamped,
|
||||
"live_split": live_split, "sessions_requested": sessions_requested}
|
||||
|
||||
def _with_service(self, fn):
|
||||
if self.execution_service is None:
|
||||
return None
|
||||
@@ -220,6 +709,28 @@ class DashboardState:
|
||||
def exit_position(self, symbol: str) -> dict | None:
|
||||
return self._with_service(lambda svc: svc.submit_exit_order(symbol))
|
||||
|
||||
def emit_command(self, command: str, symbol: str | None = None,
|
||||
requested_by: str = "dashboard") -> dict:
|
||||
"""Emit a guarded operator command event (the loop consumes + executes it).
|
||||
|
||||
Works even when execution_service is None — the standalone dashboard is a
|
||||
thin emitter; the loop process is the single execution authority. Callers
|
||||
gate on DASHBOARD_TOKEN before reaching here (commands never run
|
||||
unauthenticated)."""
|
||||
nonce = uuid.uuid4().hex
|
||||
with self._lock:
|
||||
store = self.open_store()
|
||||
try:
|
||||
store.emit(OperatorCommandEvent(
|
||||
timestamp=datetime.now(), mode=EventMode.PAPER,
|
||||
message=f"operator command {command}"
|
||||
+ (f" {symbol}" if symbol else ""),
|
||||
command=command, symbol=(symbol or None),
|
||||
requested_by=requested_by, nonce=nonce))
|
||||
finally:
|
||||
store.close()
|
||||
return {"ok": True, "command": command, "symbol": symbol, "nonce": nonce}
|
||||
|
||||
def add_watch(self, symbol: str) -> dict | None:
|
||||
"""Manually inject a symbol onto the watchlist (if supported)."""
|
||||
if not self.watch_inject or not symbol:
|
||||
@@ -230,6 +741,33 @@ class DashboardState:
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {"ok": False, "error": str(exc)}
|
||||
|
||||
def add_pick(self, body: dict) -> dict:
|
||||
"""Morning pick journal (research/pick_journal.py) — shadow forecast
|
||||
experiment; never read by the live loop."""
|
||||
from research.pick_journal import add_pick as _add
|
||||
store = self.open_store()
|
||||
try:
|
||||
row = _add(store.con, str(body.get("symbol") or ""),
|
||||
forecaster=str(body.get("forecaster") or "phil"),
|
||||
direction=str(body.get("direction") or "long"),
|
||||
conviction=int(body.get("conviction") or 2),
|
||||
note=body.get("note"))
|
||||
return {"ok": True, "valid": row["valid"], "symbol": row["symbol"],
|
||||
"picked_at": row["picked_at"].isoformat()}
|
||||
except ValueError as exc:
|
||||
return {"ok": False, "error": str(exc)}
|
||||
finally:
|
||||
store.con.close()
|
||||
|
||||
def picks(self, for_date: str | None = None) -> list[dict]:
|
||||
from research.pick_journal import list_picks
|
||||
store = self.open_store()
|
||||
try:
|
||||
d = date.fromisoformat(for_date) if for_date else None
|
||||
return list_picks(store.con, d)
|
||||
finally:
|
||||
store.con.close()
|
||||
|
||||
|
||||
def _json_default(obj):
|
||||
if isinstance(obj, datetime):
|
||||
@@ -252,10 +790,27 @@ class DashboardHandler(BaseHTTPRequestHandler):
|
||||
|
||||
def _send_json(self, payload, status: int = 200) -> None:
|
||||
body = _dump(payload)
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
try:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
# the client hung up while we were rendering (slow query + browser
|
||||
# timeout) — nothing to send it; don't double-traceback the log
|
||||
pass
|
||||
|
||||
def _send_static(self, path: Path, content_type: str) -> None:
|
||||
"""Serve a static asset (e.g. the vendored chart lib) — cacheable, unlike
|
||||
the dashboard shell."""
|
||||
if not path.exists():
|
||||
self._send_json({"error": "not found"}, 404)
|
||||
return
|
||||
body = path.read_bytes()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
@@ -300,18 +855,18 @@ class DashboardHandler(BaseHTTPRequestHandler):
|
||||
self.send_header("Content-Type", "text/event-stream")
|
||||
self.send_header("Cache-Control", "no-cache")
|
||||
self.send_header("Connection", "keep-alive")
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.end_headers()
|
||||
|
||||
shutdown = getattr(type(self), "shutdown_event", None)
|
||||
last_count = -1
|
||||
_sentinel = object()
|
||||
last_stamp = _sentinel
|
||||
last_heartbeat = 0.0
|
||||
try:
|
||||
while shutdown is None or not shutdown.is_set():
|
||||
count = self.state.event_count()
|
||||
stamp = self.state.change_stamp()
|
||||
now = time.monotonic()
|
||||
if count != last_count:
|
||||
last_count = count
|
||||
if stamp != last_stamp: # last_stamp starts as a unique sentinel
|
||||
last_stamp = stamp
|
||||
payload = _dump(self.state.snapshots()).decode()
|
||||
self.wfile.write(f"event: snapshot\ndata: {payload}\n\n".encode())
|
||||
self.wfile.flush()
|
||||
@@ -335,6 +890,17 @@ class DashboardHandler(BaseHTTPRequestHandler):
|
||||
route = urlparse(self.path).path
|
||||
if route in ("/", "/index.html", "/dashboard"):
|
||||
self._send_html(STATIC_DIR / "dashboard.html")
|
||||
elif route == "/backtest":
|
||||
self._send_html(STATIC_DIR / "backtest.html")
|
||||
elif route.startswith("/static/vendor/"):
|
||||
# path-traversal guard: resolve then require it stays inside
|
||||
# STATIC_DIR/vendor (e.g. "/static/vendor/../dashboard.html" 404s).
|
||||
vendor_dir = (STATIC_DIR / "vendor").resolve()
|
||||
requested = (vendor_dir / route[len("/static/vendor/"):]).resolve()
|
||||
if requested.is_relative_to(vendor_dir) and requested.is_file():
|
||||
self._send_static(requested, "application/javascript")
|
||||
else:
|
||||
self._send_json({"error": "not found"}, 404)
|
||||
elif route == "/api/stream":
|
||||
self._stream()
|
||||
elif route == "/api/snapshots":
|
||||
@@ -350,14 +916,54 @@ class DashboardHandler(BaseHTTPRequestHandler):
|
||||
symbol = (q.get("symbol") or [""])[0]
|
||||
minutes = int((q.get("minutes") or ["60"])[0])
|
||||
self._send_json(self.state.bars(symbol, minutes))
|
||||
elif route == "/api/chart":
|
||||
q = self._query()
|
||||
symbol = (q.get("symbol") or [""])[0]
|
||||
hours = int((q.get("hours") or ["24"])[0])
|
||||
anchor = (q.get("date") or [""])[0] or None
|
||||
self._send_json(self.state.chart(symbol, hours, anchor_date=anchor))
|
||||
elif route == "/api/decisions":
|
||||
q = self._query()
|
||||
symbol = (q.get("symbol") or [""])[0] or None
|
||||
limit = min(int((q.get("limit") or ["100"])[0]), 500)
|
||||
d = (q.get("date") or [""])[0] or None
|
||||
self._send_json({"decisions":
|
||||
self.state.decisions(symbol=symbol, limit=limit, for_date=d)})
|
||||
elif route == "/api/backtest":
|
||||
q = self._query()
|
||||
since = (q.get("since") or [""])[0] or None
|
||||
until = (q.get("until") or [""])[0] or None
|
||||
days = (q.get("days") or [""])[0]
|
||||
rg = (q.get("regime") or [""])[0]
|
||||
self._send_json(self.state.backtest(
|
||||
days=int(days) if days else 10, since=since, until=until,
|
||||
regime_override=(rg == "1") if rg in ("0", "1") else None))
|
||||
elif route == "/api/attribution":
|
||||
# per-component stage ratings (research/attribution.py); computed on
|
||||
# demand — a handful of indexed event queries, fine per click
|
||||
from research.attribution import compute_attribution
|
||||
days = int((self._query().get("days") or ["7"])[0])
|
||||
self._send_json(compute_attribution(days=days))
|
||||
elif route == "/api/picks":
|
||||
d = (self._query().get("date") or [""])[0] or None
|
||||
self._send_json({"picks": self.state.picks(d)})
|
||||
elif route == "/api/health":
|
||||
self._send_json({"ok": True, "mode": self.state.execution_mode})
|
||||
self._send_json({"ok": True, "mode": self.state.execution_mode,
|
||||
"version": os.environ.get("APP_BUILD_TAG", "dev"),
|
||||
"git_sha": os.environ.get("APP_GIT_SHA", "dev")[:12]})
|
||||
else:
|
||||
self._send_json({"error": "not found"}, 404)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._send_json({"error": str(exc)}, 500)
|
||||
|
||||
def do_POST(self): # noqa: N802
|
||||
# optional shared-secret gate on trade actions (approve/reject/exit/watch).
|
||||
# Off by default (autonomous stack auto-approves); set DASHBOARD_TOKEN to
|
||||
# require X-Dashboard-Token on every POST. CORS wildcard was removed so a
|
||||
# cross-origin LAN page can't drive these either way.
|
||||
_tok = os.environ.get("DASHBOARD_TOKEN")
|
||||
if _tok and not hmac.compare_digest(self.headers.get("X-Dashboard-Token", ""), _tok):
|
||||
return self._send_json({"error": "bad or missing X-Dashboard-Token"}, 401)
|
||||
try:
|
||||
route = urlparse(self.path).path
|
||||
body = self._read_body()
|
||||
@@ -377,6 +983,32 @@ class DashboardHandler(BaseHTTPRequestHandler):
|
||||
result = self.state.exit_position(str(body.get("symbol") or ""))
|
||||
elif route == "/api/watch/add":
|
||||
result = self.state.add_watch(str(body.get("symbol") or ""))
|
||||
elif route == "/api/picks":
|
||||
self._send_json(self.state.add_pick(body))
|
||||
return
|
||||
elif route == "/api/command":
|
||||
# The command bus is token-gated by construction: without
|
||||
# DASHBOARD_TOKEN a command could run unauthenticated, so refuse
|
||||
# outright (the top-of-do_POST gate only bites WHEN a token is set).
|
||||
if not os.environ.get("DASHBOARD_TOKEN"):
|
||||
self._send_json(
|
||||
{"ok": False, "error": "command bus disabled: DASHBOARD_TOKEN not set"},
|
||||
503)
|
||||
return
|
||||
command = str(body.get("command") or "")
|
||||
if command not in OPERATOR_COMMANDS:
|
||||
self._send_json(
|
||||
{"ok": False, "error": f"unknown command {command!r}"}, 400)
|
||||
return
|
||||
symbol = body.get("symbol")
|
||||
symbol = str(symbol).upper() if symbol else None
|
||||
if command == "flatten_symbol" and not symbol:
|
||||
self._send_json(
|
||||
{"ok": False, "error": "flatten_symbol requires a symbol"}, 400)
|
||||
return
|
||||
self._send_json(
|
||||
self.state.emit_command(command, symbol=symbol), 200)
|
||||
return
|
||||
else:
|
||||
self._send_json({"error": "not found"}, 404)
|
||||
return
|
||||
@@ -391,7 +1023,6 @@ class DashboardHandler(BaseHTTPRequestHandler):
|
||||
|
||||
def do_OPTIONS(self): # noqa: N802
|
||||
self.send_response(204)
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
self.send_header("Access-Control-Allow-Headers", "Content-Type")
|
||||
self.end_headers()
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Momentum Admin</title>
|
||||
<style>
|
||||
:root{--bg:#f6f7fb;--card:#fff;--ink:#1a2233;--mut:#68738a;--line:#e5e8f0;
|
||||
--ok:#16a34a;--warn:#d97706;--bad:#dc2626;--acc:#2563eb;--accbg:#eff4ff;--r:14px}
|
||||
@media (prefers-color-scheme:dark){:root{--bg:#0f1420;--card:#171e2e;--ink:#e8ecf6;
|
||||
--mut:#8b96ad;--line:#232c42;--accbg:#1a2540}}
|
||||
*{box-sizing:border-box;margin:0}
|
||||
body{font:15px/1.5 -apple-system,'Segoe UI',Roboto,sans-serif;background:var(--bg);color:var(--ink);padding:24px}
|
||||
.wrap{max-width:1100px;margin:0 auto}
|
||||
header{display:flex;align-items:center;gap:14px;margin-bottom:22px;flex-wrap:wrap}
|
||||
h1{font-size:20px;font-weight:700}
|
||||
.badge{padding:3px 12px;border-radius:99px;font-size:12px;font-weight:700;letter-spacing:.4px}
|
||||
.paper{background:#e0edff;color:#1d4ed8}.live{background:#fee2e2;color:#b91c1c}
|
||||
.spacer{flex:1}
|
||||
input[type=password],input[type=text]{background:var(--card);border:1px solid var(--line);
|
||||
border-radius:8px;padding:7px 10px;color:var(--ink);font-size:14px}
|
||||
h2{font-size:13px;text-transform:uppercase;letter-spacing:.8px;color:var(--mut);margin:26px 0 10px}
|
||||
.grid{display:grid;gap:12px}.g3{grid-template-columns:repeat(auto-fill,minmax(240px,1fr))}
|
||||
.g4{grid-template-columns:repeat(auto-fill,minmax(200px,1fr))}
|
||||
.card{background:var(--card);border:1px solid var(--line);border-radius:var(--r);padding:14px 16px}
|
||||
.row{display:flex;align-items:center;gap:10px}
|
||||
.dot{width:9px;height:9px;border-radius:99px;flex:none}
|
||||
.ok{background:var(--ok)}.warn{background:var(--warn)}.bad{background:var(--bad)}
|
||||
.mut{color:var(--mut);font-size:13px}
|
||||
button{background:var(--acc);color:#fff;border:0;border-radius:8px;padding:7px 14px;
|
||||
font-size:13px;font-weight:600;cursor:pointer}
|
||||
button.sec{background:var(--accbg);color:var(--acc)}
|
||||
button.danger{background:var(--bad)}
|
||||
button:disabled{opacity:.45;cursor:default}
|
||||
table{width:100%;border-collapse:collapse;font-size:14px}
|
||||
td,th{padding:8px 10px;border-bottom:1px solid var(--line);text-align:left}
|
||||
th{color:var(--mut);font-size:12px;text-transform:uppercase;letter-spacing:.5px}
|
||||
td input{width:130px}
|
||||
.tag{font-size:11px;background:var(--accbg);color:var(--acc);border-radius:6px;padding:1px 7px;font-weight:700}
|
||||
#toast{position:fixed;bottom:22px;left:50%;transform:translateX(-50%);background:var(--ink);
|
||||
color:var(--bg);padding:10px 20px;border-radius:10px;font-size:14px;opacity:0;transition:.3s}
|
||||
</style>
|
||||
</head>
|
||||
<body><div class="wrap">
|
||||
<header>
|
||||
<h1>Momentum Admin</h1>
|
||||
<span id="mode" class="badge paper">PAPER</span>
|
||||
<span id="mkt" class="mut"></span>
|
||||
<div class="spacer"></div>
|
||||
<input type="password" id="token" placeholder="admin token" size="14">
|
||||
<button class="sec" onclick="saveToken()">Connect</button>
|
||||
</header>
|
||||
|
||||
<h2>Health</h2>
|
||||
<div class="grid g4" id="health"></div>
|
||||
|
||||
<h2>Services</h2>
|
||||
<div class="grid g3" id="services"></div>
|
||||
|
||||
<h2>Strategy profiles</h2>
|
||||
<div class="grid g3" id="profiles"></div>
|
||||
|
||||
<h2>Configuration <span class="mut" style="text-transform:none;letter-spacing:0">— edits go to overrides.env; restart the loop to apply. <span class="tag">override</span> = differs from deployed env</span></h2>
|
||||
<div class="card"><table id="cfg"><thead><tr><th>key</th><th>value</th><th></th></tr></thead><tbody></tbody></table>
|
||||
<div class="row" style="margin-top:12px"><button onclick="saveCfg()">Save changes</button>
|
||||
<span class="mut" id="cfgnote"></span></div></div>
|
||||
|
||||
<h2>Effective config <span class="mut" style="text-transform:none;letter-spacing:0">— resolved value + source per key; <span class="tag">overrides.env</span> beats compose. Secrets masked. A red row = a documented .env.example key is missing (drift).</span></h2>
|
||||
<div class="card"><table id="effcfg"><thead><tr><th>key</th><th>effective value</th><th>source</th></tr></thead><tbody></tbody></table></div>
|
||||
|
||||
<h2>Trading mode</h2>
|
||||
<div class="card row">
|
||||
<div style="flex:1"><b>Paper ⇄ Live switch</b><div class="mut">Live requires ALPACA_LIVE_API_KEY / ALPACA_LIVE_SECRET_KEY in the deployment env. Switching restarts nothing by itself — restart the loop after.</div></div>
|
||||
<button class="sec" onclick="setMode('paper')">Paper</button>
|
||||
<button class="danger" onclick="if(confirm('Switch the loop to LIVE trading with real money on next restart. Are you sure?'))setMode('live')">Go Live</button>
|
||||
</div>
|
||||
|
||||
<h2>Research jobs</h2>
|
||||
<div class="card row">
|
||||
<div style="flex:1"><b>Regime backtest <span class="mut" style="text-transform:none;letter-spacing:0">— plan 011, SHADOW</span></b>
|
||||
<div class="mut">Rebuild session labels → walk-forward score both models → replay + adjudicate the regime_gate experiments. Runs a few min; does NOT touch live trading.</div></div>
|
||||
<button class="sec" onclick="runRegime()">Run</button>
|
||||
<button class="sec" onclick="regimeStatus()">Status</button>
|
||||
</div>
|
||||
<div class="card" id="regimeout" style="display:none"><pre id="regimelog" class="mut" style="white-space:pre-wrap;max-height:240px;overflow:auto;margin:0;font-size:12px"></pre></div>
|
||||
|
||||
<h2>Secrets present</h2>
|
||||
<div class="card" id="secrets"></div>
|
||||
<div id="toast"></div>
|
||||
</div>
|
||||
<script>
|
||||
const $=q=>document.querySelector(q);let staged={};
|
||||
const esch=s=>String(s==null?'':s).replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
||||
const tok=()=>localStorage.getItem('admintok')||'';
|
||||
function saveToken(){localStorage.setItem('admintok',$('#token').value);load()}
|
||||
async function api(p,opt={}){opt.headers={...(opt.headers||{}),'X-Admin-Token':tok(),'Content-Type':'application/json'};
|
||||
const r=await fetch(p,opt);if(r.status===401){toast('bad token');throw 0}return r.json()}
|
||||
function toast(m){const t=$('#toast');t.textContent=m;t.style.opacity=1;setTimeout(()=>t.style.opacity=0,2600)}
|
||||
async function runRegime(){toast('starting regime pipeline…');
|
||||
try{const r=await api('/api/regime/run',{method:'POST',body:'{}'});
|
||||
toast(r.ok?('started (pid '+r.pid+') — poll Status'):('failed: '+(r.error||'')));
|
||||
setTimeout(regimeStatus,2000)}catch(e){toast('failed')}}
|
||||
async function regimeStatus(){const r=await api('/api/regime/status');
|
||||
$('#regimeout').style.display='block';
|
||||
$('#regimelog').textContent=(r.log_tail&&r.log_tail.length?r.log_tail.join('\n'):'(no run yet — click Run)')
|
||||
+(r.running?'\n\n… running …':(r.ran?'\n\n[finished]':''));
|
||||
if(r.running)setTimeout(regimeStatus,4000)}
|
||||
function dot(ok){return `<span class="dot ${ok===true?'ok':ok===false?'bad':'warn'}"></span>`}
|
||||
async function load(){
|
||||
const s=await api('/api/status');
|
||||
const c=s.checks||{};
|
||||
$('#health').innerHTML=[
|
||||
`<div class="card row">${dot(c.db?.ok)}<div><b>Database</b><div class="mut">${c.db?.ok?`event ${c.db.newest_event_age_s}s ago`:c.db?.error||''}</div></div></div>`,
|
||||
`<div class="card row">${dot(c.alpaca?.ok)}<div><b>${c.alpaca?.name||'Broker'}</b><div class="mut">${c.alpaca?.ok?`$${(+c.alpaca.equity).toLocaleString()} · market ${c.alpaca.market_open?'open':'closed'}`:c.alpaca?.error||''}</div></div></div>`,
|
||||
`<div class="card row">${dot(c.ollama?.ok)}<div><b>Ollama</b><div class="mut">${c.ollama?.ok?'enrichment up':'down (fails safe)'}</div></div></div>`,
|
||||
`<div class="card row">${dot(c.feed==='sip')}<div><b>Data feed</b><div class="mut">${c.feed}</div></div></div>`].join('');
|
||||
$('#mode').className='badge '+(c.alpaca?.mode==='LIVE'?'live':'paper');
|
||||
$('#mode').textContent=c.alpaca?.mode==='LIVE'?'LIVE':'PAPER';
|
||||
$('#mkt').textContent=c.alpaca?.market_open?'market open':'market closed';
|
||||
$('#services').innerHTML=(s.containers||[]).map(x=>`<div class="card row">
|
||||
${dot(x.health?x.health==='healthy':x.state==='running')}
|
||||
<div style="flex:1"><b>${x.name.replace(/^ix-|-1$/g,'')}</b><div class="mut">${x.status||x.state}</div></div>
|
||||
<button class="sec" onclick="restart('${x.name}')">restart</button></div>`).join('');
|
||||
const cfg=await api('/api/config');staged={};
|
||||
$('#cfg tbody').innerHTML=Object.entries(cfg.effective).map(([k,v])=>`<tr>
|
||||
<td>${k} ${cfg.overrides[k]!==undefined?'<span class="tag">override</span>':''}</td>
|
||||
<td><input type="text" value="${v}" oninput="staged['${k}']=this.value"></td>
|
||||
<td>${cfg.overrides[k]!==undefined?`<button class="sec" onclick="delKey('${k}')">reset</button>`:''}</td></tr>`).join('');
|
||||
const eff=await api('/api/config/effective');
|
||||
$('#effcfg tbody').innerHTML=Object.entries(eff.config||{}).map(([k,v])=>`<tr${v.drift?' style="background:rgba(220,38,38,.12)"':''}>
|
||||
<td>${esch(k)}</td>
|
||||
<td>${v.value_masked==null?'<span class="mut">—</span>':esch(v.value_masked)}</td>
|
||||
<td><span class="tag">${esch(v.source)}</span></td></tr>`).join('');
|
||||
$('#secrets').innerHTML=Object.entries(cfg.secrets_present).map(([k,v])=>
|
||||
`<span style="margin-right:18px">${v?'✅':'❌'} ${k}</span>`).join('');
|
||||
const prof=await api('/api/profiles');
|
||||
$('#profiles').innerHTML=Object.entries(prof).map(([n,p])=>`<div class="card">
|
||||
<b>${n}</b><div class="mut" style="min-height:40px">${p._note||''}</div>
|
||||
<button onclick="applyProfile('${n}')">Apply</button></div>`).join('');
|
||||
}
|
||||
async function saveCfg(){if(!Object.keys(staged).length)return toast('nothing changed');
|
||||
await api('/api/config',{method:'POST',body:JSON.stringify({set:staged})});
|
||||
toast('saved — restart the loop to apply');load()}
|
||||
async function delKey(k){await api('/api/config',{method:'POST',body:JSON.stringify({delete:[k]})});load()}
|
||||
async function applyProfile(n){await api('/api/profiles/apply',{method:'POST',body:JSON.stringify({name:n})});
|
||||
toast(`profile ${n} staged — restart the loop to apply`);load()}
|
||||
async function restart(n){toast('restarting '+n+'…');
|
||||
const r=await api('/api/restart',{method:'POST',body:JSON.stringify({service:n})});
|
||||
toast(r.ok?'restarted '+n:'restart failed');setTimeout(load,3000)}
|
||||
async function setMode(m){const r=await api('/api/mode',{method:'POST',body:JSON.stringify({mode:m})});
|
||||
toast(r.error||('mode -> '+m+' — restart the loop to apply'));load()}
|
||||
$('#token').value=tok();load().catch(()=>{});setInterval(()=>load().catch(()=>{}),15000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,434 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Momentum — Backtest Replay</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #080a10;
|
||||
--panel: #111622;
|
||||
--panel-2: #161c2a;
|
||||
--panel-hi: #1c2435;
|
||||
--border: #232c41;
|
||||
--border-soft: #18202f;
|
||||
--text: #e3e8f2;
|
||||
--muted: #7884a2;
|
||||
--accent: #5ba3ff;
|
||||
--accent-dim: #2a4a78;
|
||||
--green: #34d17e;
|
||||
--red: #ff5a60;
|
||||
--amber: #f7b13b;
|
||||
--purple: #c2a6e2;
|
||||
--glow-green: rgba(52,209,126,.5);
|
||||
--glow-red: rgba(255,90,96,.45);
|
||||
--shadow: 0 8px 24px -12px rgba(0,0,0,.7), 0 1px 0 0 rgba(255,255,255,.035) inset;
|
||||
--mono: "SF Mono", "Cascadia Code", "JetBrains Mono", Menlo, Consolas, monospace;
|
||||
}
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body {
|
||||
background:
|
||||
radial-gradient(1100px 560px at 82% -12%, rgba(30,58,110,.28), transparent 60%),
|
||||
radial-gradient(820px 480px at -5% -5%, rgba(60,40,120,.16), transparent 55%),
|
||||
var(--bg);
|
||||
background-attachment: fixed;
|
||||
color: var(--text);
|
||||
font: 14px/1.5 -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||
min-height: 100vh;
|
||||
}
|
||||
a { color: var(--accent); }
|
||||
header { display: flex; align-items: center; gap: 14px; padding: 14px 20px;
|
||||
border-bottom: 1px solid var(--border); flex-wrap: wrap; }
|
||||
header h1 { font-size: 16px; letter-spacing: .3px; }
|
||||
header h1 a { color: var(--muted); text-decoration: none; font-size: 12px; font-weight: 600;
|
||||
border: 1px solid var(--border); border-radius: 4px; padding: 3px 8px; margin-left: 10px; }
|
||||
header h1 a:hover { color: var(--text); border-color: var(--accent); }
|
||||
.picker { display: flex; align-items: center; gap: 6px; margin-left: auto; flex-wrap: wrap; }
|
||||
.picker button { background: var(--panel-2); border: 1px solid var(--border); color: var(--text);
|
||||
border-radius: 5px; padding: 6px 12px; font: 600 12px var(--mono); cursor: pointer; }
|
||||
.picker button:hover { border-color: var(--accent); }
|
||||
.picker button.active { background: var(--accent-dim); border-color: var(--accent); color: #fff; }
|
||||
.picker input[type=date] { background: var(--panel-2); border: 1px solid var(--border); color: var(--text);
|
||||
border-radius: 5px; padding: 5px 8px; font: 600 12px var(--mono); }
|
||||
.picker .run { background: var(--accent-dim); border-color: var(--accent); }
|
||||
|
||||
.banner { margin: 14px 20px 0; padding: 10px 14px; border-radius: 8px;
|
||||
background: rgba(247,177,59,.10); border: 1px solid rgba(247,177,59,.35);
|
||||
color: var(--amber); font: 700 12px/1.4 var(--mono); letter-spacing: .2px; }
|
||||
|
||||
.chips { display: flex; gap: 8px; flex-wrap: wrap; margin: 12px 20px 0; }
|
||||
.chip { background: var(--panel-2); border: 1px solid var(--border); border-radius: 20px;
|
||||
padding: 4px 12px; font: 600 11px var(--mono); color: var(--muted); }
|
||||
.chip.on { color: var(--green); border-color: rgba(52,209,126,.4); }
|
||||
.chip.off { color: var(--muted); }
|
||||
|
||||
.loading { margin: 40px 20px; text-align: center; color: var(--muted); font: 600 13px var(--mono); }
|
||||
.loading .spin { width: 30px; height: 30px; margin: 0 auto 14px; border: 3px solid var(--border);
|
||||
border-top-color: var(--accent); border-radius: 50%; animation: spin .8s linear infinite; }
|
||||
@keyframes spin { to { transform: rotate(360deg); } }
|
||||
.error { margin: 20px; padding: 14px; border-radius: 8px; background: rgba(255,90,96,.08);
|
||||
border: 1px solid rgba(255,90,96,.35); color: var(--red); font: 600 12px var(--mono); }
|
||||
.hidden { display: none !important; }
|
||||
|
||||
main { padding: 16px 20px 40px; }
|
||||
|
||||
.summary { display: grid; grid-template-columns: repeat(8, 1fr); gap: 1px; background: var(--border);
|
||||
border: 1px solid var(--border); border-radius: 10px; overflow: hidden; margin-bottom: 16px; }
|
||||
.summary .cell { background: var(--panel); padding: 12px 14px; }
|
||||
.summary .label { font: 600 10px var(--mono); color: var(--muted); text-transform: uppercase; letter-spacing: .5px; }
|
||||
.summary .value { font: 700 20px var(--mono); margin-top: 4px; }
|
||||
.summary .value.pos { color: var(--green); }
|
||||
.summary .value.neg { color: var(--red); }
|
||||
|
||||
.panel { background: var(--panel); border: 1px solid var(--border); border-radius: 10px;
|
||||
box-shadow: var(--shadow); margin-bottom: 16px; }
|
||||
.panel h2 { font-size: 13px; padding: 12px 16px; border-bottom: 1px solid var(--border-soft);
|
||||
display: flex; align-items: center; gap: 8px; }
|
||||
.panel h2 .sub { font-weight: 500; color: var(--muted); font-size: 11px; }
|
||||
.panel-body { padding: 12px 16px; overflow-x: auto; }
|
||||
.empty { color: var(--muted); font: 600 12px var(--mono); padding: 20px 0; display: block; text-align: center; }
|
||||
|
||||
.daily-svg { display: block; }
|
||||
.daily-col { cursor: pointer; }
|
||||
.daily-hit { fill: transparent; }
|
||||
.daily-col:hover .daily-hit { fill: rgba(91,163,255,.06); }
|
||||
.daily-lbl { font: 600 9px var(--mono); fill: var(--muted); text-anchor: middle; }
|
||||
.daily-lbl.daily-sel { fill: var(--accent); }
|
||||
.daily-col.daily-sel .daily-hit { fill: rgba(91,163,255,.10); }
|
||||
|
||||
table { width: 100%; border-collapse: collapse; font-size: 12px; }
|
||||
th { text-align: left; font: 600 10px var(--mono); color: var(--muted); text-transform: uppercase;
|
||||
letter-spacing: .4px; padding: 6px 10px; border-bottom: 1px solid var(--border-soft); position: sticky; top: 0; background: var(--panel); }
|
||||
td { padding: 7px 10px; border-bottom: 1px solid var(--border-soft); font: 500 12px var(--mono); white-space: nowrap; }
|
||||
tbody tr { cursor: pointer; }
|
||||
tbody tr:hover { background: var(--panel-hi); }
|
||||
td.sym { font-weight: 700; color: var(--accent); }
|
||||
td.r.pos { color: var(--green); font-weight: 700; }
|
||||
td.r.neg { color: var(--red); font-weight: 700; }
|
||||
|
||||
footer { padding: 10px 20px 30px; color: var(--muted); font: 500 10px/1.6 var(--mono); }
|
||||
|
||||
/* chart modal — imitates dashboard.html's plan-013 drill-in */
|
||||
.chart-modal { position: fixed; inset: 0; background: rgba(6,8,12,.82); display: none; place-items: center; z-index: 70; }
|
||||
.chart-modal.show { display: grid; }
|
||||
.chart-card { background: var(--panel); border: 1px solid var(--border); border-radius: 10px;
|
||||
width: min(920px, 94vw); height: min(560px, 88vh); display: flex; flex-direction: column; overflow: hidden; }
|
||||
.chart-hdr { display: flex; align-items: center; gap: 10px; padding: 10px 16px;
|
||||
border-bottom: 1px solid var(--border); flex-shrink: 0; }
|
||||
.chart-hdr .sym { font: 700 15px var(--mono); letter-spacing: .3px; }
|
||||
.chart-hdr .meta { color: var(--muted); font: 600 11px var(--mono); }
|
||||
.chart-hdr .close { margin-left: auto; background: transparent; border: 1px solid var(--border);
|
||||
color: var(--muted); border-radius: 4px; padding: 4px 10px; cursor: pointer; font: 600 11px var(--mono); }
|
||||
.chart-hdr .close:hover { color: var(--text); border-color: var(--accent); }
|
||||
#chart-container { flex: 1; min-height: 0; }
|
||||
.chart-ftr { padding: 6px 16px; border-top: 1px solid var(--border-soft); font: 500 10px var(--mono);
|
||||
color: var(--muted); flex-shrink: 0; }
|
||||
|
||||
@media (max-width: 900px) { .summary { grid-template-columns: repeat(3, 1fr); } }
|
||||
@media (max-width: 640px) { .summary { grid-template-columns: repeat(2, 1fr); } header { padding: 10px 14px; } }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<h1>Backtest Replay <a href="/">← live dashboard</a></h1>
|
||||
<div class="picker">
|
||||
<button data-days="1">1d</button>
|
||||
<button data-days="5">5d</button>
|
||||
<button data-days="10">10d</button>
|
||||
<button data-days="20">20d</button>
|
||||
<button data-days="45">45d</button>
|
||||
<input type="date" id="since-input">
|
||||
<input type="date" id="until-input">
|
||||
<button class="run" id="run-custom">Run range</button>
|
||||
<label style="font: 600 11px var(--mono); color: var(--muted); display:flex; align-items:center; gap:5px; margin-left:8px">
|
||||
<input type="checkbox" id="regime-toggle" checked> apply regime gate (as deployed)
|
||||
</label>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="banner">THEORETICAL — simulated fills (adverse-tick fidelity), setup-sourced entries; not live P&L.</div>
|
||||
<div class="chips" id="chips"></div>
|
||||
|
||||
<div class="loading hidden" id="loading"><div class="spin"></div><span id="loading-msg">replaying…</span></div>
|
||||
<div class="error hidden" id="error"></div>
|
||||
|
||||
<main class="hidden" id="results">
|
||||
<div class="summary" id="summary"></div>
|
||||
|
||||
<div class="panel">
|
||||
<h2>Equity curve (R) <span class="sub">click a day to filter the trade table</span></h2>
|
||||
<div class="panel-body"><div id="daily-curve"></div></div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<h2>Trades <span class="sub" id="trades-sub"></span></h2>
|
||||
<div class="panel-body">
|
||||
<table>
|
||||
<thead><tr><th>Date</th><th>Symbol</th><th>Entry</th><th>Exit</th><th>Reason</th><th>R</th><th>Live P&L</th></tr></thead>
|
||||
<tbody id="trades-body"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<footer id="footer" class="hidden">
|
||||
<div>entries are setup_events-sourced ORB triggers (the labeler's population); entry fill = trigger + adverse ticks — optimistic vs real slippage on thin names.</div>
|
||||
<div>per-trade sim ≠ per-trade reality (minute-bar granularity + adverse fills can flip individual rows, e.g. a live +$200 winner simming as a −1R stop) — trust the aggregates and the live-taken/not-taken split, not single rows.</div>
|
||||
<div id="coverage-line"></div>
|
||||
</footer>
|
||||
|
||||
<div id="chart-modal" class="chart-modal">
|
||||
<div class="chart-card">
|
||||
<div class="chart-hdr">
|
||||
<span class="sym" id="chart-sym"></span>
|
||||
<span class="meta" id="chart-meta"></span>
|
||||
<button class="close" id="chart-close">CLOSE (Esc)</button>
|
||||
</div>
|
||||
<div id="chart-container"></div>
|
||||
<div class="chart-ftr">blue = theoretical entry · purple = theoretical exit (reason labeled) · Charts by TradingView</div>
|
||||
</div>
|
||||
</div>
|
||||
<script src="/static/vendor/lightweight-charts.standalone.production.js" defer></script>
|
||||
|
||||
<script>
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const esc = (s) => String(s ?? "").replace(/[&<>"']/g, c => ({"&":"&","<":"<",">":">",'"':""","'":"'"}[c]));
|
||||
const fmtR = (v) => (v == null || isNaN(+v)) ? "—" : (v >= 0 ? "+" : "") + (+v).toFixed(2) + "R";
|
||||
const fmtPx = (v) => (v == null || isNaN(+v)) ? "—" : (+v).toFixed(2);
|
||||
const fmtTs = (iso) => { try { return iso.replace("T", " ").slice(5, 16); } catch { return iso || "—"; } };
|
||||
|
||||
let state = { data: null, viewDate: null };
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// fetch + orchestration
|
||||
// -------------------------------------------------------------------------
|
||||
function setLoading(msg) {
|
||||
$("loading-msg").textContent = msg;
|
||||
$("loading").classList.remove("hidden");
|
||||
$("error").classList.add("hidden");
|
||||
$("results").classList.add("hidden");
|
||||
$("footer").classList.add("hidden");
|
||||
}
|
||||
|
||||
async function runBacktest(params, activeBtn) {
|
||||
document.querySelectorAll(".picker button[data-days]").forEach(b => b.classList.toggle("active", b === activeBtn));
|
||||
state.lastParams = params; state.lastBtn = activeBtn;
|
||||
// the gate toggle rides every request: unchecked = raw-criteria trades
|
||||
params = { ...params, regime: $("regime-toggle").checked ? "1" : "0" };
|
||||
const n = params.days || "the requested";
|
||||
setLoading("replaying " + n + " session" + (n === 1 ? "" : "s") + "… (can take up to 30s)");
|
||||
try {
|
||||
const qs = new URLSearchParams(params).toString();
|
||||
const r = await fetch("/api/backtest?" + qs);
|
||||
const data = await r.json();
|
||||
if (!r.ok) throw new Error(data.error || ("HTTP " + r.status));
|
||||
state.data = data;
|
||||
state.viewDate = null;
|
||||
render(data);
|
||||
} catch (e) {
|
||||
$("loading").classList.add("hidden");
|
||||
$("error").textContent = "backtest failed: " + e.message;
|
||||
$("error").classList.remove("hidden");
|
||||
}
|
||||
}
|
||||
|
||||
document.querySelectorAll(".picker button[data-days]").forEach(btn => {
|
||||
btn.onclick = () => runBacktest({ days: btn.dataset.days }, btn);
|
||||
});
|
||||
$("regime-toggle").onchange = () => runBacktest(state.lastParams || { days: 10 }, state.lastBtn || null);
|
||||
$("run-custom").onclick = () => {
|
||||
const since = $("since-input").value, until = $("until-input").value;
|
||||
if (!since || !until) return;
|
||||
runBacktest({ since, until }, null);
|
||||
};
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// render
|
||||
// -------------------------------------------------------------------------
|
||||
function render(data) {
|
||||
$("loading").classList.add("hidden");
|
||||
$("results").classList.remove("hidden");
|
||||
$("footer").classList.remove("hidden");
|
||||
renderChips(data.settings, data.clamped);
|
||||
renderSummary(data.summary, data.skipped, data.screened, data.live_split);
|
||||
renderDaily(data.daily);
|
||||
renderTrades(data.trades, null);
|
||||
const y = data.sessions_requested;
|
||||
const x = (data.daily || []).length;
|
||||
$("coverage-line").textContent = "sessions replayed: " + x + " of " + y + " requested" +
|
||||
(data.clamped ? " (clamped to the 45-session cap)" : "") +
|
||||
(x < y ? " — a nightly setup_events build gap may explain the shortfall" : "");
|
||||
}
|
||||
|
||||
function renderChips(s, clamped) {
|
||||
if (!s) { $("chips").innerHTML = ""; return; }
|
||||
const gap = s.population.min_entry_gap_pct > 0 ? "gap≥" + (s.population.min_entry_gap_pct * 100).toFixed(0) + "%" : "gap gate off";
|
||||
const chips = [
|
||||
"target_r=" + s.target_r,
|
||||
"price $" + s.population.price_min + "-" + s.population.price_max,
|
||||
gap,
|
||||
"regime gate " + (s.regime_gate ? "ON" : "OFF"),
|
||||
"fidelity atr×" + s.fidelity.atr_frac + " spread×" + s.fidelity.spread_mult + " adverse=" + s.fidelity.adverse_ticks + "tick",
|
||||
];
|
||||
if (clamped) chips.push("CLAMPED to 45 sessions");
|
||||
$("chips").innerHTML = chips.map(c => '<span class="chip ' + (c.includes("ON") ? "on" : "") + '">' + esc(c) + '</span>').join("");
|
||||
}
|
||||
|
||||
function renderSummary(sum, skipped, screened, split) {
|
||||
sum = sum || { n: 0, sum_r: 0, avg_r: 0, hit_rate: 0, sessions: 0 };
|
||||
const cell = (label, val, cls) => '<div class="cell"><div class="label">' + label +
|
||||
'</div><div class="value ' + (cls || "") + '">' + val + '</div></div>';
|
||||
const sp = (g) => g ? fmtR(g.sum_r) + ' <span style="font-size:10px;color:var(--muted)">avg ' +
|
||||
fmtR(g.avg_r) + '</span>' : "—";
|
||||
const taken = split && split.taken, not_taken = split && split.not_taken;
|
||||
$("summary").innerHTML =
|
||||
cell("Trades", sum.n) +
|
||||
cell("Sum R", fmtR(sum.sum_r), sum.sum_r >= 0 ? "pos" : "neg") +
|
||||
cell("Avg R", fmtR(sum.avg_r), sum.avg_r >= 0 ? "pos" : "neg") +
|
||||
cell("Hit Rate", (sum.hit_rate * 100).toFixed(0) + "%") +
|
||||
cell("Sessions", sum.sessions) +
|
||||
cell("Skipped", skipped + " <span style=\"font-size:10px;color:var(--muted)\">/ pop " +
|
||||
(screened ? screened.population : 0) + " · regime " + (screened ? screened.regime : 0) + "</span>") +
|
||||
cell("Live-taken (" + (taken ? taken.n : 0) + ")", sp(taken),
|
||||
taken ? (taken.sum_r >= 0 ? "pos" : "neg") : "") +
|
||||
cell("Not taken (" + (not_taken ? not_taken.n : 0) + ")", sp(not_taken),
|
||||
not_taken ? (not_taken.sum_r >= 0 ? "pos" : "neg") : "");
|
||||
}
|
||||
|
||||
// day-over-day R curve — waterfall adapted from dashboard.html's renderDaily
|
||||
// (cum_pnl -> cum_r, day_pnl -> sum_r); click a day filters the trade table.
|
||||
function renderDaily(days) {
|
||||
const host = $("daily-curve");
|
||||
if (!days || !days.length) {
|
||||
const reg = ((state.data || {}).screened || {}).regime || 0;
|
||||
host.innerHTML = '<span class="empty">' + (reg > 0
|
||||
? 'No trades: the regime gate sat out every replayed session in this window — the bot would not have traded these days at all. Uncheck \u201capply regime gate\u201d above to see what the raw criteria would have done.'
|
||||
: 'No trades in this window (no qualifying setups).') + '</span>';
|
||||
return;
|
||||
}
|
||||
const n = days.length;
|
||||
const W = Math.max(host.clientWidth || 680, n * 48);
|
||||
const H = 184, padT = 16, padB = 30, padX = 12, plotH = H - padT - padB;
|
||||
const cums = days.map(d => +d.cum_r || 0);
|
||||
const lo = Math.min(0, ...cums), hi = Math.max(0, ...cums), rng = (hi - lo) || 1;
|
||||
const Y = v => padT + plotH * (1 - (v - lo) / rng);
|
||||
const slot = (W - 2 * padX) / n, cx = i => padX + slot * (i + 0.5);
|
||||
const barW = Math.min(slot * 0.5, 34);
|
||||
const z = Y(0).toFixed(1);
|
||||
let bars = "", labels = "", dots = "", prev = 0;
|
||||
days.forEach((d, i) => {
|
||||
const cur = cums[i], up = (+d.sum_r || 0) >= 0, col = up ? "#34d17e" : "#ff5a60";
|
||||
const y1 = Y(prev), y2 = Y(cur), top = Math.min(y1, y2), h = Math.max(1, Math.abs(y1 - y2));
|
||||
const x = (cx(i) - barW / 2).toFixed(1);
|
||||
const sel = state.viewDate === d.date ? " daily-sel" : "";
|
||||
const tip = d.date + " · " + d.n + " trades · sum " + fmtR(d.sum_r) + " · cum " + fmtR(cur);
|
||||
bars += '<g class="daily-col' + sel + '" onclick="filterByDay(\'' + d.date + '\')"><title>' + esc(tip) + '</title>'
|
||||
+ '<rect class="daily-hit" x="' + (padX + slot * i).toFixed(1) + '" y="' + padT + '" width="' + slot.toFixed(1) + '" height="' + plotH + '"/>'
|
||||
+ '<rect x="' + x + '" y="' + top.toFixed(1) + '" width="' + barW.toFixed(1) + '" height="' + h.toFixed(1) + '" rx="2" fill="' + col + '" opacity=".5"/>'
|
||||
+ '</g>';
|
||||
labels += '<text class="daily-lbl' + sel + '" x="' + cx(i).toFixed(1) + '" y="' + (H - 10) + '">' + d.date.slice(5) + '</text>';
|
||||
dots += '<circle cx="' + cx(i).toFixed(1) + '" cy="' + y2.toFixed(1) + '" r="2.8" fill="#5ba3ff"/>';
|
||||
prev = cur;
|
||||
});
|
||||
const line = days.map((d, i) => cx(i).toFixed(1) + "," + Y(cums[i]).toFixed(1)).join(" ");
|
||||
host.innerHTML = '<svg width="' + W + '" height="' + H + '" viewBox="0 0 ' + W + ' ' + H + '" class="daily-svg">'
|
||||
+ '<line x1="' + padX + '" y1="' + z + '" x2="' + (W - padX) + '" y2="' + z + '" stroke="#30363d" stroke-width="1" stroke-dasharray="3 3"/>'
|
||||
+ bars + '<polyline points="' + line + '" fill="none" stroke="#5ba3ff" stroke-width="1.6"/>' + dots + labels + '</svg>';
|
||||
}
|
||||
|
||||
function filterByDay(date) {
|
||||
state.viewDate = (state.viewDate === date) ? null : date;
|
||||
renderDaily(state.data.daily);
|
||||
renderTrades(state.data.trades, state.viewDate);
|
||||
}
|
||||
|
||||
function renderTrades(trades, viewDate) {
|
||||
trades = trades || [];
|
||||
const rows = viewDate ? trades.filter(t => t.session_date === viewDate) : trades;
|
||||
$("trades-sub").textContent = viewDate ? (rows.length + " on " + viewDate + " — click the day again to clear") :
|
||||
(rows.length + " total");
|
||||
if (!rows.length) { $("trades-body").innerHTML = '<tr><td colspan="7" class="empty">No trades.</td></tr>'; return; }
|
||||
$("trades-body").innerHTML = rows.map((t, i) => {
|
||||
const idx = trades.indexOf(t);
|
||||
const cls = t.r_multiple >= 0 ? "pos" : "neg";
|
||||
let live = '<span style="color:var(--muted)">—</span>';
|
||||
if (t.live_pnl != null) {
|
||||
const lcls = t.live_pnl >= 0 ? "pos" : "neg";
|
||||
// sign disagreement = the per-trade caveat made visible (minute-bar
|
||||
// granularity + adverse fills); aggregates are the trustworthy view
|
||||
const diverged = (t.live_pnl >= 0) !== (t.r_multiple >= 0)
|
||||
? ' <span style="color:var(--amber)" title="sim and live disagree on this trade — minute-bar granularity + adverse fills; trust aggregates, not rows">≠</span>' : '';
|
||||
live = '<span class="r ' + lcls + '">' + (t.live_pnl >= 0 ? "+$" : "−$") +
|
||||
Math.abs(t.live_pnl).toFixed(0) + '</span>' + diverged;
|
||||
}
|
||||
return '<tr onclick="openTradeChart(' + idx + ')">' +
|
||||
'<td>' + esc(t.session_date) + '</td>' +
|
||||
'<td class="sym">' + esc(t.symbol) + '</td>' +
|
||||
'<td>' + fmtTs(t.entry_ts) + ' @ ' + fmtPx(t.entry_fill) + '</td>' +
|
||||
'<td>' + fmtTs(t.exit_ts) + ' @ ' + fmtPx(t.fills.length ? t.fills[t.fills.length - 1].price : null) + '</td>' +
|
||||
'<td>' + esc(t.exit_reason) + '</td>' +
|
||||
'<td class="r ' + cls + '">' + fmtR(t.r_multiple) + '</td>' +
|
||||
'<td>' + live + '</td>' +
|
||||
'</tr>';
|
||||
}).join("");
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// chart drill-in — imitates dashboard.html's openChart/refreshChart (plan 013)
|
||||
// -------------------------------------------------------------------------
|
||||
let chartState = { chart: null, series: null, markersApi: null };
|
||||
|
||||
function closeChart() {
|
||||
$("chart-modal").classList.remove("show");
|
||||
if (chartState.chart) chartState.chart.remove();
|
||||
chartState = { chart: null, series: null, markersApi: null };
|
||||
}
|
||||
|
||||
async function openTradeChart(idx) {
|
||||
if (!window.LightweightCharts) return;
|
||||
const t = state.data.trades[idx];
|
||||
if (!t) return;
|
||||
if (chartState.chart) closeChart();
|
||||
$("chart-sym").textContent = t.symbol;
|
||||
$("chart-meta").textContent = t.session_date + " · " + t.exit_reason + " · " + fmtR(t.r_multiple);
|
||||
$("chart-modal").classList.add("show");
|
||||
const container = $("chart-container");
|
||||
container.innerHTML = "";
|
||||
const chart = LightweightCharts.createChart(container, {
|
||||
autoSize: true,
|
||||
layout: { background: { color: "#111622" }, textColor: "#e3e8f2" },
|
||||
grid: { vertLines: { color: "#18202f" }, horzLines: { color: "#18202f" } },
|
||||
timeScale: { timeVisible: true, secondsVisible: false },
|
||||
});
|
||||
const series = chart.addSeries(LightweightCharts.CandlestickSeries, {
|
||||
upColor: "#2fbf71", downColor: "#e5484d", borderVisible: false,
|
||||
wickUpColor: "#2fbf71", wickDownColor: "#e5484d",
|
||||
});
|
||||
const markersApi = LightweightCharts.createSeriesMarkers(series, []);
|
||||
chartState = { chart, series, markersApi };
|
||||
|
||||
try {
|
||||
const r = await fetch("/api/chart?symbol=" + encodeURIComponent(t.symbol) +
|
||||
"&hours=24&date=" + encodeURIComponent(t.session_date));
|
||||
const data = await r.json();
|
||||
series.setData((data.bars || []).map(b => ({ time: b.time, open: b.open, high: b.high, low: b.low, close: b.close })));
|
||||
const markers = [
|
||||
{ time: t.entry_epoch, position: "belowBar", shape: "arrowUp", color: "#5ba3ff", text: "entry " + fmtPx(t.entry_fill) },
|
||||
{ time: t.exit_epoch, position: "aboveBar", shape: "arrowDown", color: "#c2a6e2", text: t.exit_reason },
|
||||
].sort((a, b) => a.time - b.time);
|
||||
markersApi.setMarkers(markers);
|
||||
} catch (e) { /* leave the empty chart on screen */ }
|
||||
}
|
||||
|
||||
document.addEventListener("click", (e) => { if (e.target.id === "chart-modal") closeChart(); });
|
||||
document.addEventListener("keydown", (e) => { if (e.key === "Escape") closeChart(); });
|
||||
$("chart-close").onclick = closeChart;
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// init — default 10-session window
|
||||
// -------------------------------------------------------------------------
|
||||
runBacktest({ days: 10 }, document.querySelector('.picker button[data-days="10"]'));
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
+1267
-119
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,2 @@
|
||||
TradingView Lightweight Charts™
|
||||
Copyright (с) 2025 TradingView, Inc. https://www.tradingview.com/
|
||||
@@ -0,0 +1,7 @@
|
||||
/*!
|
||||
* @license
|
||||
* TradingView Lightweight Charts™ v5.2.0
|
||||
* Copyright (c) 2026 TradingView, Inc.
|
||||
* Licensed under Apache License 2.0 https://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
!function(){"use strict";const t={title:"",visible:!0,hitTestTolerance:3,lastValueVisible:!0,priceLineVisible:!0,priceLineSource:0,priceLineWidth:1,priceLineColor:"",priceLineStyle:2,baseLineVisible:!0,baseLineWidth:1,baseLineColor:"#B2B5BE",baseLineStyle:0,priceFormat:{type:"price",precision:2,minMove:.01}};var i,n;function s(t,i){const n=function(t,i){switch(t){case 0:default:return[];case 1:return[i,i];case 2:return[2*i,2*i];case 3:return[6*i,6*i];case 4:return[i,4*i]}}(i,t.lineWidth);return t.setLineDash(n),n}function e(t,i,n,s){t.beginPath();const e=t.lineWidth%2?.5:0;t.moveTo(n,i+e),t.lineTo(s,i+e),t.stroke()}function r(t,i){if(!t)throw new Error("Assertion failed"+(i?": "+i:""))}function h(t){if(void 0===t)throw new Error("Value is undefined");return t}function a(t){if(null===t)throw new Error("Value is null");return t}function l(t){return a(h(t))}!function(t){t[t.Simple=0]="Simple",t[t.WithSteps=1]="WithSteps",t[t.Curved=2]="Curved"}(i||(i={})),function(t){t[t.Solid=0]="Solid",t[t.Dotted=1]="Dotted",t[t.Dashed=2]="Dashed",t[t.LargeDashed=3]="LargeDashed",t[t.SparseDotted=4]="SparseDotted"}(n||(n={}));class o{constructor(){this.t=[]}i(t,i,n){const s={h:t,l:i,o:!0===n};this.t.push(s)}_(t){const i=this.t.findIndex((i=>t===i.h));i>-1&&this.t.splice(i,1)}u(t){this.t=this.t.filter((i=>i.l!==t))}p(t,i,n){const s=[...this.t];this.t=this.t.filter((t=>!t.o)),s.forEach((s=>s.h(t,i,n)))}v(){return this.t.length>0}m(){this.t=[]}}function _(t,...i){for(const n of i)for(const i in n)void 0!==n[i]&&Object.prototype.hasOwnProperty.call(n,i)&&!["__proto__","constructor","prototype"].includes(i)&&("object"!=typeof n[i]||void 0===t[i]||Array.isArray(n[i])?t[i]=n[i]:_(t[i],n[i]));return t}function u(t){return"number"==typeof t&&isFinite(t)}function c(t){return"number"==typeof t&&t%1==0}function d(t){return"string"==typeof t}function f(t){return"boolean"==typeof t}function p(t){const i=t;if(!i||"object"!=typeof i)return i;let n,s,e;for(s in n=Array.isArray(i)?[]:{},i)i.hasOwnProperty(s)&&(e=i[s],n[s]=e&&"object"==typeof e?p(e):e);return n}function v(t){return null!==t}function m(t){return null===t?void 0:t}const w="-apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif";function g(t,i,n){return void 0===i&&(i=w),`${n=void 0!==n?`${n} `:""}${t}px ${i}`}class M{constructor(t){this.M={S:1,C:5,k:NaN,P:"",T:"",R:"",D:"",I:0,V:0,B:0,A:0,L:0},this.O=t}N(){const t=this.M,i=this.F(),n=this.W();return t.k===i&&t.T===n||(t.k=i,t.T=n,t.P=g(i,n),t.A=2.5/12*i,t.I=t.A,t.V=i/12*t.C,t.B=i/12*t.C,t.L=0),t.R=this.H(),t.D=this.U(),this.M}H(){return this.O.N().layout.textColor}U(){return this.O.$()}F(){return this.O.N().layout.fontSize}W(){return this.O.N().layout.fontFamily}}function b(t){return t<0?0:t>255?255:Math.round(t)||0}function S(t){return.199*t[0]+.687*t[1]+.114*t[2]}class x{constructor(t,i){this.j=new Map,this.q=t,i&&(this.j=i)}Y(t,i){if("transparent"===t)return t;const n=this.K(t),s=n[3];return`rgba(${n[0]}, ${n[1]}, ${n[2]}, ${i*s})`}Z(t){const i=this.K(t);return{G:`rgb(${i[0]}, ${i[1]}, ${i[2]})`,X:S(i)>160?"black":"white"}}J(t){return S(this.K(t))}tt(t,i,n){const[s,e,r,h]=this.K(t),[a,l,o,_]=this.K(i),u=[b(s+n*(a-s)),b(e+n*(l-e)),b(r+n*(o-r)),(c=h+n*(_-h),c<=0||c>1?Math.min(Math.max(c,0),1):Math.round(1e4*c)/1e4)];var c;return`rgba(${u[0]}, ${u[1]}, ${u[2]}, ${u[3]})`}K(t){const i=this.j.get(t);if(i)return i;const n=function(t){const i=document.createElement("div");i.style.display="none",document.body.appendChild(i),i.style.color=t;const n=window.getComputedStyle(i).color;return document.body.removeChild(i),n}(t),s=n.match(/^rgba?\s*\((\d+),\s*(\d+),\s*(\d+)(?:,\s*(\d*\.?\d+))?\)$/);if(!s){if(this.q.length)for(const i of this.q){const n=i(t);if(n)return this.j.set(t,n),n}throw new Error(`Failed to parse color: ${t}`)}const e=[parseInt(s[1],10),parseInt(s[2],10),parseInt(s[3],10),s[4]?parseFloat(s[4]):1];return this.j.set(t,e),e}}class C{constructor(){this.it=[]}nt(t){this.it=t}st(t,i,n){this.it.forEach((s=>{s.st(t,i,n)}))}}class y{st(t,i,n){t.useBitmapCoordinateSpace((t=>this.et(t,i,n)))}}class k extends y{constructor(){super(...arguments),this.rt=null}ht(t){this.rt=t}et({context:t,horizontalPixelRatio:i,verticalPixelRatio:n}){if(null===this.rt||null===this.rt.lt)return;const s=this.rt.lt,e=this.rt,r=Math.max(1,Math.floor(i))%2/2,h=h=>{t.beginPath();for(let a=s.to-1;a>=s.from;--a){const s=e.ot[a],l=Math.round(s._t*i)+r,o=s.ut*n,_=h*n+r;t.moveTo(l,o),t.arc(l,o,_,0,2*Math.PI)}t.fill()};e.ct>0&&(t.fillStyle=e.dt,h(e.ft+e.ct)),t.fillStyle=e.vt,h(e.ft)}}function P(){return{ot:[{_t:0,ut:0,wt:0,gt:0}],vt:"",dt:"",ft:0,ct:0,lt:null}}const T={from:0,to:1};class R{constructor(t,i,n){this.Mt=new C,this.bt=[],this.St=[],this.xt=!0,this.O=t,this.Ct=i,this.yt=n,this.Mt.nt(this.bt)}kt(t){this.Pt(),this.xt=!0}Tt(){return this.xt&&(this.Rt(),this.xt=!1),this.Mt}Pt(){const t=this.yt.Dt();t.length!==this.bt.length&&(this.St=t.map(P),this.bt=this.St.map((t=>{const i=new k;return i.ht(t),i})),this.Mt.nt(this.bt))}Rt(){consLine truncated
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Targeted historical backfill: minute bars (premarket included, SIP) for the
|
||||
year of recorded gapper-days in ``historical_gappers``.
|
||||
|
||||
Purpose: fatten n for the sample-starved queues — premarket-structure H1/H2,
|
||||
pyramid_1r_v1, orderflow experiments — by giving eod_replay and the premarket
|
||||
labeler real bar paths for ~260 historical sessions. Pulls ONLY each session's
|
||||
recorded gappers (the population we trade), never the whole market.
|
||||
|
||||
PYTHONPATH=. python backfill_gapper_days.py --limit 20 # newest 20 days
|
||||
PYTHONPATH=. python backfill_gapper_days.py # all recorded days
|
||||
|
||||
Per session: fetch 04:00-20:00 ET SIP minute bars for that day's gappers,
|
||||
upsert, fetch ~40 calendar days of daily bars ending that session (the daily
|
||||
screen needs prior-20-day volume), then build premarket_structure for the day.
|
||||
Idempotent: sessions whose gappers already have premarket bars are skipped.
|
||||
Replay/experiments consume the bars via their own runs afterwards:
|
||||
|
||||
for d in $(dates); do PYTHONPATH=. python eod_replay.py $d; done
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from runtime.env_overrides import load as _load_cfg_overrides
|
||||
|
||||
_load_cfg_overrides()
|
||||
load_dotenv()
|
||||
|
||||
from alpaca_paper.client import AlpacaPaperClient # noqa: E402
|
||||
from alpaca_paper.settings import AlpacaPaperSettings # noqa: E402
|
||||
from research.ingestion.market_data import ( # noqa: E402
|
||||
upsert_daily_bars,
|
||||
upsert_minute_bars,
|
||||
)
|
||||
from research.multi_schema import open_research_db # noqa: E402
|
||||
from research.premarket_structure import build as build_pm # noqa: E402
|
||||
|
||||
ET = ZoneInfo("America/New_York")
|
||||
|
||||
|
||||
def backfill_session(con, client, sess, symbols: list[str], feed: str) -> dict:
|
||||
start = datetime(sess.year, sess.month, sess.day, 4, 0, tzinfo=ET)
|
||||
end = datetime(sess.year, sess.month, sess.day, 20, 0, tzinfo=ET)
|
||||
by_sym = client.get_minute_bars(symbols, start_iso=start.isoformat(),
|
||||
end_iso=end.isoformat(), feed=feed)
|
||||
minute_rows = sum(upsert_minute_bars(con, s, b, source_provider="alpaca_sip_backfill")
|
||||
for s, b in by_sym.items() if b)
|
||||
d_start = (start - timedelta(days=40)).isoformat()
|
||||
daily = client.get_daily_bars(symbols, start_iso=d_start,
|
||||
end_iso=end.isoformat(), feed=feed)
|
||||
daily_rows = sum(upsert_daily_bars(con, s, b) for s, b in daily.items() if b)
|
||||
pm = build_pm(con, session_date=str(sess))
|
||||
return {"minute": minute_rows, "daily": daily_rows, "pm_rows": pm["rows"]}
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--limit", type=int, help="only the newest N recorded sessions")
|
||||
ap.add_argument("--force", action="store_true", help="re-fetch even if bars exist")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
con = open_research_db("market")
|
||||
client = AlpacaPaperClient(AlpacaPaperSettings.from_env())
|
||||
feed = client.settings.backfill_feed
|
||||
lim = f" LIMIT {int(args.limit)}" if args.limit else ""
|
||||
days = con.execute(
|
||||
"SELECT session_date, array_agg(DISTINCT symbol) FROM historical_gappers "
|
||||
f"GROUP BY session_date ORDER BY session_date DESC{lim}").fetchall()
|
||||
print(f"{len(days)} recorded gapper sessions; feed={feed}")
|
||||
done = skipped = 0
|
||||
for sess, symbols in days:
|
||||
symbols = [s for s in (symbols or []) if s]
|
||||
if not symbols:
|
||||
continue
|
||||
if not args.force:
|
||||
have = con.execute(
|
||||
"SELECT count(DISTINCT symbol) FROM minute_bars "
|
||||
"WHERE session_date = ? AND is_premarket AND symbol = ANY(?)",
|
||||
[str(sess), symbols]).fetchall()[0][0]
|
||||
if have >= max(1, len(symbols) // 2): # most already present -> skip
|
||||
skipped += 1
|
||||
continue
|
||||
try:
|
||||
out = backfill_session(con, client, sess, symbols, feed)
|
||||
done += 1
|
||||
print(f"{sess}: {len(symbols)} syms, {out['minute']} minute rows, "
|
||||
f"{out['daily']} daily rows, {out['pm_rows']} pm-structure rows", flush=True)
|
||||
except Exception as exc: # noqa: BLE001 — one bad day must not kill the sweep
|
||||
print(f"{sess}: FAILED {exc}", flush=True)
|
||||
print(f"done: {done} sessions backfilled, {skipped} already present")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-time backfill of the trade journal from Alpaca's filled-order history.
|
||||
|
||||
The live journal (position_closed events from the sync snapshot-diff) is
|
||||
forward-only, so past market days show 0 trades. This reconstructs every
|
||||
completed round-trip from order history and emits the matching position_closed
|
||||
events, timestamped (ET) at the exit fill so each lands on its own market day —
|
||||
populating per-day win-rate / avg-R / the trade list retroactively.
|
||||
|
||||
Idempotent: re-running deletes prior backfill events first. Live-reconciled
|
||||
closes are left untouched (deduped by symbol + exit-minute).
|
||||
|
||||
python backfill_journal.py --dry-run # preview, write nothing
|
||||
python backfill_journal.py # write the events
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
ET = ZoneInfo("America/New_York")
|
||||
BACKFILL_TAG = "journal_backfill"
|
||||
|
||||
|
||||
def _to_et_naive(iso: str):
|
||||
"""Alpaca filled_at is UTC; store as naive ET to match the loop's clock and
|
||||
land each close on the correct market day."""
|
||||
try:
|
||||
dt = datetime.fromisoformat(str(iso).replace("Z", "+00:00"))
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(ET).replace(tzinfo=None)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--dry-run", action="store_true", help="preview, write nothing")
|
||||
args = ap.parse_args()
|
||||
|
||||
load_dotenv()
|
||||
from storage.event_schema import EventMode, PositionClosedEvent
|
||||
from storage.event_store import EventStore
|
||||
from storage.journal_backfill import flatten_fills, reconstruct_round_trips
|
||||
from tradier_broker.factory import make_trading_client
|
||||
|
||||
store = EventStore("momentum")
|
||||
client, label = make_trading_client()
|
||||
if client is None:
|
||||
raise SystemExit(f"[backfill_journal] {label} — aborting")
|
||||
|
||||
raw = client.get_orders(status="closed", limit=500, nested=True)
|
||||
print(f"fetched {len(raw)} historical orders from {label}")
|
||||
trips = reconstruct_round_trips(flatten_fills(raw))
|
||||
print(f"reconstructed {len(trips)} completed round-trips")
|
||||
|
||||
# existing closes: collect prior-backfill ids (to clear) and live keys (to skip)
|
||||
existing = store.query_events(event_type="position_closed", limit=None)
|
||||
prior_backfill = [e["id"] for e in existing if e.get("correlation_id") == BACKFILL_TAG]
|
||||
live_keys = set()
|
||||
for e in existing:
|
||||
if e.get("correlation_id") == BACKFILL_TAG:
|
||||
continue
|
||||
p = json.loads(e["payload_json"])
|
||||
live_keys.add((p.get("symbol"), str(e["timestamp"])[:16]))
|
||||
|
||||
pending, by_day, skipped = [], {}, 0
|
||||
for t in trips:
|
||||
et = _to_et_naive(t["exit_time"])
|
||||
if et is None:
|
||||
continue
|
||||
if (t["symbol"], et.isoformat()[:16]) in live_keys:
|
||||
skipped += 1
|
||||
continue
|
||||
pending.append((t, et))
|
||||
d = by_day.setdefault(et.date().isoformat(), {"n": 0, "pnl": 0.0, "w": 0})
|
||||
d["n"] += 1
|
||||
d["pnl"] += t["realized_pnl"]
|
||||
d["w"] += 1 if t["realized_pnl"] > 0 else 0
|
||||
|
||||
print("\nper market day:")
|
||||
for day in sorted(by_day):
|
||||
d = by_day[day]
|
||||
wr = (d["w"] / d["n"] * 100) if d["n"] else 0
|
||||
print(f" {day}: {d['n']:>3} trades win {wr:>3.0f}% pnl {d['pnl']:+.2f}")
|
||||
if skipped:
|
||||
print(f"(skipped {skipped} already journaled live)")
|
||||
|
||||
if args.dry_run:
|
||||
print(f"\n[dry-run] would write {len(pending)} events (no changes made)")
|
||||
return
|
||||
|
||||
if prior_backfill:
|
||||
store.con.execute(
|
||||
"DELETE FROM events WHERE event_type = 'position_closed' "
|
||||
"AND correlation_id = ?", [BACKFILL_TAG])
|
||||
print(f"cleared {len(prior_backfill)} prior backfill events")
|
||||
|
||||
for t, et in pending:
|
||||
store.emit(PositionClosedEvent(
|
||||
timestamp=et, mode=EventMode.PAPER, correlation_id=BACKFILL_TAG,
|
||||
message=(f"[backfill] {t['symbol']} closed @ {t['exit_price']} "
|
||||
f"({t['exit_reason']}) pnl={t['realized_pnl']:+.2f}"),
|
||||
position_id=f"backfill-{t['symbol']}-{et.isoformat()}",
|
||||
symbol=t["symbol"], exit_price=t["exit_price"], exit_reason=t["exit_reason"],
|
||||
realized_pnl=t["realized_pnl"], entry_price=t["entry_price"],
|
||||
stop_loss_price=t["stop_loss_price"], side=t["side"], quantity=t["qty"],
|
||||
))
|
||||
print(f"\nemitted {len(pending)} position_closed events across {len(by_day)} market days")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,326 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Multi-week comprehensive backtest with detailed statistical analysis."""
|
||||
|
||||
from datetime import datetime, timedelta, date
|
||||
import statistics
|
||||
|
||||
import pandas as pd
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from config import Config
|
||||
from research.connection import ResearchConnection
|
||||
from research.query import query_minute_bars
|
||||
from strategy.backtest.engine import BacktestEngine
|
||||
|
||||
console = Console()
|
||||
|
||||
def get_trading_days(start: date, end: date) -> list[date]:
|
||||
"""Get all weekdays between start and end (trading days only)."""
|
||||
days = []
|
||||
current = start
|
||||
while current <= end:
|
||||
# Only include weekdays (0=Monday, 6=Sunday)
|
||||
if current.weekday() < 5:
|
||||
days.append(current)
|
||||
current += timedelta(days=1)
|
||||
return days
|
||||
|
||||
|
||||
def run_backtest_multi_day(start_date: date, end_date: date, symbols: list[str] | None = None):
|
||||
"""Run backtest across multiple days and aggregate results."""
|
||||
|
||||
config = Config()
|
||||
con = ResearchConnection()
|
||||
|
||||
if symbols is None:
|
||||
symbols = ["SNDL", "MARA", "RIOT", "BBAI", "SOUN", "PLUG"]
|
||||
|
||||
console.print(f"\n[bold blue]Comprehensive Backtest: {start_date} to {end_date}[/bold blue]")
|
||||
console.print(f"[dim]Testing {len(symbols)} symbols across trading days[/dim]\n")
|
||||
|
||||
trading_days = get_trading_days(start_date, end_date)
|
||||
console.print(f"[cyan]{len(trading_days)} trading days in range[/cyan]\n")
|
||||
|
||||
# Aggregate results
|
||||
all_trades: list[dict] = []
|
||||
daily_results: dict[date, dict] = {}
|
||||
symbol_stats: dict[str, dict] = {}
|
||||
|
||||
engine = BacktestEngine(config)
|
||||
|
||||
# Progress bar
|
||||
processed = 0
|
||||
for trading_day in trading_days:
|
||||
# Query minute bars for all symbols that day
|
||||
day_symbols_with_data = []
|
||||
day_bars = {}
|
||||
|
||||
for symbol in symbols:
|
||||
try:
|
||||
bars = query_minute_bars(con, symbol, trading_day)
|
||||
if len(bars) > 0:
|
||||
day_symbols_with_data.append(symbol)
|
||||
day_bars[symbol] = bars
|
||||
except Exception as e:
|
||||
pass # No data for this day
|
||||
|
||||
if not day_symbols_with_data:
|
||||
processed += 1
|
||||
continue
|
||||
|
||||
# Initialize daily results if needed
|
||||
if trading_day not in daily_results:
|
||||
daily_results[trading_day] = {"trades": [], "pnl": 0.0}
|
||||
|
||||
# Run backtest for each symbol with data
|
||||
for symbol in day_symbols_with_data:
|
||||
bars = day_bars[symbol]
|
||||
|
||||
try:
|
||||
result = engine.run(bars, symbol)
|
||||
|
||||
# Aggregate
|
||||
daily_results[trading_day]["pnl"] += result.total_pnl
|
||||
|
||||
# Initialize symbol stats if needed
|
||||
if symbol not in symbol_stats:
|
||||
symbol_stats[symbol] = {"trades": [], "wins": 0, "losses": 0, "pnl": 0.0}
|
||||
|
||||
for trade in result.trades:
|
||||
trade_data = {
|
||||
"date": trading_day.isoformat(),
|
||||
"symbol": symbol,
|
||||
"entry_time": trade.entry_time,
|
||||
"entry_price": trade.entry_price,
|
||||
"stop_price": trade.stop_price,
|
||||
"target_price": trade.target_price,
|
||||
"quantity": trade.quantity,
|
||||
"exit_price": trade.exit_price,
|
||||
"exit_reason": trade.exit_reason,
|
||||
"pnl": trade.realized_pnl or 0.0,
|
||||
"r_multiple": trade.r_multiple or 0.0,
|
||||
}
|
||||
all_trades.append(trade_data)
|
||||
daily_results[trading_day]["trades"].append(trade_data)
|
||||
|
||||
# Symbol stats
|
||||
symbol_stats[symbol]["trades"].append(trade_data)
|
||||
symbol_stats[symbol]["pnl"] += trade.realized_pnl or 0.0
|
||||
if (trade.realized_pnl or 0) > 0:
|
||||
symbol_stats[symbol]["wins"] += 1
|
||||
else:
|
||||
symbol_stats[symbol]["losses"] += 1
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error backtesting {symbol} on {trading_day}: {e}[/red]")
|
||||
|
||||
processed += 1
|
||||
console.print(f"[dim]Processed {processed}/{len(trading_days)} days[/dim]", end="\r")
|
||||
|
||||
console.print()
|
||||
|
||||
# === SUMMARY STATISTICS ===
|
||||
if not all_trades:
|
||||
console.print("[yellow]No trades found in backtest range[/yellow]")
|
||||
return
|
||||
|
||||
# Daily summary
|
||||
console.print("\n[bold cyan]=== DAILY SUMMARY ===[/bold cyan]")
|
||||
daily_table = Table(title="P&L by Day")
|
||||
daily_table.add_column("Date", style="cyan")
|
||||
daily_table.add_column("Trades", justify="right")
|
||||
daily_table.add_column("P&L", justify="right")
|
||||
|
||||
daily_pnls = []
|
||||
for day in sorted(daily_results.keys()):
|
||||
daily_pnl = daily_results[day]["pnl"]
|
||||
daily_pnls.append(daily_pnl)
|
||||
color = "green" if daily_pnl > 0 else "red"
|
||||
daily_table.add_row(
|
||||
str(day),
|
||||
str(len(daily_results[day]["trades"])),
|
||||
f"[{color}]${daily_pnl:,.2f}[/{color}]"
|
||||
)
|
||||
console.print(daily_table)
|
||||
|
||||
# Symbol summary
|
||||
console.print("\n[bold cyan]=== SYMBOL SUMMARY ===[/bold cyan]")
|
||||
symbol_table = Table(title="Results by Symbol")
|
||||
symbol_table.add_column("Symbol", style="cyan")
|
||||
symbol_table.add_column("Trades", justify="right")
|
||||
symbol_table.add_column("Wins", justify="right")
|
||||
symbol_table.add_column("Win %", justify="right")
|
||||
symbol_table.add_column("P&L", justify="right")
|
||||
symbol_table.add_column("Avg R", justify="right")
|
||||
|
||||
for symbol in sorted(symbol_stats.keys()):
|
||||
stats = symbol_stats[symbol]
|
||||
trades = stats["trades"]
|
||||
if not trades:
|
||||
continue
|
||||
|
||||
win_pct = (stats["wins"] / len(trades)) * 100 if trades else 0
|
||||
avg_r = statistics.mean([t["r_multiple"] for t in trades]) if trades else 0
|
||||
|
||||
color = "green" if stats["pnl"] > 0 else "red"
|
||||
symbol_table.add_row(
|
||||
symbol,
|
||||
str(len(trades)),
|
||||
str(stats["wins"]),
|
||||
f"{win_pct:.1f}%",
|
||||
f"[{color}]${stats['pnl']:,.2f}[/{color}]",
|
||||
f"{avg_r:.2f}R"
|
||||
)
|
||||
console.print(symbol_table)
|
||||
|
||||
# Overall stats
|
||||
total_trades = len(all_trades)
|
||||
total_pnl = sum(t["pnl"] for t in all_trades)
|
||||
wins = len([t for t in all_trades if t["pnl"] > 0])
|
||||
losses = len([t for t in all_trades if t["pnl"] < 0])
|
||||
win_rate = (wins / total_trades) * 100 if total_trades else 0
|
||||
|
||||
r_multiples = [t["r_multiple"] for t in all_trades]
|
||||
avg_r = statistics.mean(r_multiples) if r_multiples else 0
|
||||
|
||||
# Separate wins and losses for analysis
|
||||
win_trades = [t for t in all_trades if t["pnl"] > 0]
|
||||
loss_trades = [t for t in all_trades if t["pnl"] < 0]
|
||||
|
||||
avg_win = statistics.mean([t["pnl"] for t in win_trades]) if win_trades else 0
|
||||
avg_loss = statistics.mean([t["pnl"] for t in loss_trades]) if loss_trades else 0
|
||||
max_loss = min([t["pnl"] for t in loss_trades]) if loss_trades else 0
|
||||
|
||||
profit_factor = (sum([t["pnl"] for t in win_trades]) / abs(sum([t["pnl"] for t in loss_trades]))) if loss_trades and sum([t["pnl"] for t in loss_trades]) < 0 else 0
|
||||
|
||||
console.print("\n[bold cyan]=== OVERALL STATISTICS ===[/bold cyan]")
|
||||
console.print(f"Total Trades: {total_trades}")
|
||||
console.print(f"Wins: {wins} | Losses: {losses}")
|
||||
console.print(f"Win Rate: {win_rate:.1f}%")
|
||||
console.print(f"Total P&L: [{'green' if total_pnl > 0 else 'red'}]${total_pnl:,.2f}[/{'green' if total_pnl > 0 else 'red'}]")
|
||||
console.print(f"Avg Win: ${avg_win:,.2f} | Avg Loss: ${avg_loss:,.2f}")
|
||||
console.print(f"Max Loss: ${max_loss:,.2f}")
|
||||
if profit_factor > 0:
|
||||
console.print(f"Profit Factor: {profit_factor:.2f}x")
|
||||
console.print(f"Avg R-Multiple: {avg_r:.2f}R")
|
||||
|
||||
# === LOSS ANALYSIS ===
|
||||
if loss_trades:
|
||||
console.print("\n[bold yellow]=== LOSS ANALYSIS ===[/bold yellow]")
|
||||
console.print(f"Total Losing Trades: {len(loss_trades)}")
|
||||
|
||||
loss_table = Table(title="Top 10 Losses (for pattern analysis)")
|
||||
loss_table.add_column("Date", style="cyan")
|
||||
loss_table.add_column("Symbol", style="cyan")
|
||||
loss_table.add_column("Entry", justify="right")
|
||||
loss_table.add_column("Stop", justify="right")
|
||||
loss_table.add_column("Exit", justify="right")
|
||||
loss_table.add_column("P&L", justify="right")
|
||||
loss_table.add_column("Reason")
|
||||
|
||||
sorted_losses = sorted(loss_trades, key=lambda t: t["pnl"])
|
||||
for trade in sorted_losses[:10]:
|
||||
loss_table.add_row(
|
||||
trade["date"],
|
||||
trade["symbol"],
|
||||
f"${trade['entry_price']:.2f}",
|
||||
f"${trade['stop_price']:.2f}",
|
||||
f"${trade['exit_price']:.2f}" if trade["exit_price"] else "-",
|
||||
f"[red]${trade['pnl']:,.2f}[/red]",
|
||||
trade["exit_reason"] or "?"
|
||||
)
|
||||
console.print(loss_table)
|
||||
|
||||
# Loss metrics
|
||||
stop_hits = len([t for t in loss_trades if "stop" in (t["exit_reason"] or "").lower()])
|
||||
console.print(f"\n[dim]Losses by reason:[/dim]")
|
||||
console.print(f" Hit stop: {stop_hits}/{len(loss_trades)}")
|
||||
|
||||
# Gap-through analysis
|
||||
for trade in sorted_losses[:5]:
|
||||
entry = trade["entry_price"]
|
||||
stop = trade["stop_price"]
|
||||
exit_price = trade["exit_price"]
|
||||
if exit_price and exit_price < stop:
|
||||
gap_pct = ((stop - exit_price) / stop) * 100
|
||||
console.print(f" [yellow]{trade['symbol']} {trade['date']}: gap-through {gap_pct:.2f}% ({stop:.2f} → {exit_price:.2f})[/yellow]")
|
||||
|
||||
# === WIN ANALYSIS ===
|
||||
if win_trades:
|
||||
console.print("\n[bold green]=== WIN ANALYSIS ===[/bold green]")
|
||||
console.print(f"Total Winning Trades: {len(win_trades)}")
|
||||
|
||||
win_table = Table(title="Recent Wins")
|
||||
win_table.add_column("Date", style="cyan")
|
||||
win_table.add_column("Symbol", style="cyan")
|
||||
win_table.add_column("Entry", justify="right")
|
||||
win_table.add_column("Target", justify="right")
|
||||
win_table.add_column("Exit", justify="right")
|
||||
win_table.add_column("P&L", justify="right")
|
||||
win_table.add_column("R")
|
||||
|
||||
sorted_wins = sorted(win_trades, key=lambda t: t["pnl"], reverse=True)
|
||||
for trade in sorted_wins[:5]:
|
||||
win_table.add_row(
|
||||
trade["date"],
|
||||
trade["symbol"],
|
||||
f"${trade['entry_price']:.2f}",
|
||||
f"${trade['target_price']:.2f}",
|
||||
f"${trade['exit_price']:.2f}" if trade["exit_price"] else "-",
|
||||
f"[green]${trade['pnl']:,.2f}[/green]",
|
||||
f"{trade['r_multiple']:.2f}R"
|
||||
)
|
||||
console.print(win_table)
|
||||
|
||||
# === RISK ASSESSMENT ===
|
||||
console.print("\n[bold yellow]=== RISK ASSESSMENT ===[/bold yellow]")
|
||||
|
||||
if avg_loss != 0:
|
||||
expectancy = (win_rate / 100 * avg_win) + ((1 - win_rate / 100) * avg_loss)
|
||||
console.print(f"Expected value per trade: ${expectancy:,.2f}")
|
||||
|
||||
if profit_factor > 0:
|
||||
if profit_factor < 1.5:
|
||||
console.print("[yellow]⚠️ Profit factor < 1.5 indicates marginal edge or slippage issues[/yellow]")
|
||||
else:
|
||||
console.print("[green]✓ Profit factor suggests adequate win/loss ratio[/green]")
|
||||
|
||||
# Check for consistent losses
|
||||
max_win = max([t["pnl"] for t in win_trades]) if win_trades else 0
|
||||
if max_loss < 0 and abs(max_loss) > max_win * 1.5:
|
||||
console.print(f"[red]⚠️ Max loss (${max_loss:,.2f}) >> max win (${max_win:,.2f}) — stop is too wide or slippage is severe[/red]")
|
||||
|
||||
# Sharpe-ish ratio (simplified)
|
||||
if len(daily_pnls) > 1:
|
||||
daily_std = statistics.stdev(daily_pnls) if len(daily_pnls) > 1 else 0
|
||||
if daily_std > 0:
|
||||
daily_mean = statistics.mean(daily_pnls)
|
||||
sharpe_approx = daily_mean / daily_std if daily_std > 0 else 0
|
||||
console.print(f"Daily Sharpe (approx): {sharpe_approx:.2f}")
|
||||
|
||||
console.print("\n[dim]Recommendations:[/dim]")
|
||||
if win_rate < 40:
|
||||
console.print("[red]• Win rate < 40% — strategy may lack edge even with profitable trades[/red]")
|
||||
if profit_factor < 1.3 and profit_factor > 0:
|
||||
console.print("[yellow]• Profit factor < 1.3 — consider tightening stops or improving entry[/yellow]")
|
||||
if len(loss_trades) > 0 and max_loss < avg_loss / 2:
|
||||
console.print("[yellow]• Losses are inconsistent — gap-throughs may be a factor[/yellow]")
|
||||
|
||||
console.print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Multi-week comprehensive backtest")
|
||||
parser.add_argument("--start", type=str, default="2026-06-05", help="Start date (YYYY-MM-DD)")
|
||||
parser.add_argument("--end", type=str, default="2026-06-12", help="End date (YYYY-MM-DD)")
|
||||
parser.add_argument("--symbols", type=str, default="SNDL,MARA,RIOT,BBAI,SOUN,PLUG", help="Comma-separated symbols")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
start = datetime.strptime(args.start, "%Y-%m-%d").date()
|
||||
end = datetime.strptime(args.end, "%Y-%m-%d").date()
|
||||
symbols = args.symbols.split(",")
|
||||
|
||||
run_backtest_multi_day(start, end, symbols)
|
||||
@@ -7,6 +7,31 @@ from typing import Any
|
||||
import yaml
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
# Ollama runs on the LAN host reached via EITHER the 127.0.0.1 netsh tunnel OR the direct
|
||||
# 192.168.1.5 — and which works FLIPS across WSL/host reboots (same landmine as the DB).
|
||||
# The two use DIFFERENT ports (tunnel 30069 -> direct 30068), so we probe known endpoints
|
||||
# and pick the first reachable, so enrichment self-heals on boot instead of silently failing.
|
||||
_OLLAMA_CANDIDATES = ("http://192.168.1.5:30068", "http://127.0.0.1:30069")
|
||||
|
||||
|
||||
def _ollama_ok(base: str) -> bool:
|
||||
import urllib.request
|
||||
try:
|
||||
with urllib.request.urlopen(base.rstrip("/") + "/api/tags", timeout=2) as r:
|
||||
return r.status == 200
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_ollama_host(host: str) -> str:
|
||||
"""Return the configured host if reachable, else the first reachable known endpoint."""
|
||||
if _ollama_ok(host):
|
||||
return host
|
||||
for cand in _OLLAMA_CANDIDATES:
|
||||
if cand != host and _ollama_ok(cand):
|
||||
return cand
|
||||
return host
|
||||
|
||||
|
||||
class SlippageConfig(BaseModel):
|
||||
base_spread_pct: float = 0.0015
|
||||
@@ -197,7 +222,7 @@ class OllamaConfig(BaseModel):
|
||||
|
||||
return cls(
|
||||
enabled=b("OLLAMA_ENABLED", False),
|
||||
host=v.get("OLLAMA_HOST", cls.model_fields["host"].default),
|
||||
host=_resolve_ollama_host(v.get("OLLAMA_HOST", cls.model_fields["host"].default)),
|
||||
model=v.get("OLLAMA_MODEL", cls.model_fields["model"].default),
|
||||
timeout_seconds=i("OLLAMA_TIMEOUT_SECONDS", 30),
|
||||
max_tokens=i("OLLAMA_MAX_TOKENS", 256),
|
||||
|
||||
@@ -1,259 +0,0 @@
|
||||
"""Module contracts and interfaces for the momentum trading system.
|
||||
|
||||
Every module must implement the ModuleContract protocol, defining:
|
||||
- input_schema: What the module expects
|
||||
- output_schema: What the module produces
|
||||
- run(): Execute the module
|
||||
- validate_input(): Ensure data meets contracts
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, date
|
||||
from typing import Any, Protocol, TypeVar
|
||||
|
||||
T_Input = TypeVar("T_Input")
|
||||
T_Output = TypeVar("T_Output")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModuleMetrics:
|
||||
"""Standard metrics every module should track."""
|
||||
|
||||
module_name: str
|
||||
timestamp: datetime
|
||||
duration_ms: float
|
||||
input_count: int = 0
|
||||
output_count: int = 0
|
||||
error_count: int = 0
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
custom_metrics: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"module_name": self.module_name,
|
||||
"timestamp": self.timestamp.isoformat(),
|
||||
"duration_ms": self.duration_ms,
|
||||
"input_count": self.input_count,
|
||||
"output_count": self.output_count,
|
||||
"error_count": self.error_count,
|
||||
"warnings": self.warnings,
|
||||
"custom_metrics": self.custom_metrics,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationFailure:
|
||||
"""Single validation failure with context."""
|
||||
|
||||
field: str
|
||||
value: Any
|
||||
reason: str
|
||||
severity: str = "error" # "error" or "warning"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModuleResult:
|
||||
"""Base class for all module results."""
|
||||
|
||||
success: bool
|
||||
timestamp: datetime
|
||||
metrics: ModuleMetrics
|
||||
validation_failures: list[ValidationFailure] = field(default_factory=list)
|
||||
|
||||
def has_errors(self) -> bool:
|
||||
return bool(self.validation_failures or not self.success)
|
||||
|
||||
def error_summary(self) -> str:
|
||||
"""Human-readable error summary."""
|
||||
if not self.validation_failures:
|
||||
return "No errors"
|
||||
errors = [f for f in self.validation_failures if f.severity == "error"]
|
||||
if not errors:
|
||||
return f"{len(self.validation_failures)} warnings only"
|
||||
return f"{len(errors)} error(s), {len([f for f in self.validation_failures if f.severity == 'warning'])} warning(s)"
|
||||
|
||||
|
||||
class ModuleContract(Protocol):
|
||||
"""Contract every module must fulfill."""
|
||||
|
||||
def run(self, input: Any) -> ModuleResult:
|
||||
"""Execute the module.
|
||||
|
||||
Args:
|
||||
input: Module-specific input conforming to input_schema
|
||||
|
||||
Returns:
|
||||
Module-specific result inheriting from ModuleResult
|
||||
"""
|
||||
...
|
||||
|
||||
def validate_input(self, input: Any) -> tuple[bool, list[ValidationFailure]]:
|
||||
"""Validate input before processing.
|
||||
|
||||
Args:
|
||||
input: Data to validate
|
||||
|
||||
Returns:
|
||||
(is_valid, failures)
|
||||
"""
|
||||
...
|
||||
|
||||
def get_metrics(self) -> ModuleMetrics:
|
||||
"""Get last execution metrics."""
|
||||
...
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# INGESTION MODULE CONTRACTS
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class BarIngestRequest:
|
||||
"""Input contract for ingestion module."""
|
||||
|
||||
symbols: list[str]
|
||||
session_date: date
|
||||
lookback_minutes: int = 120 # how many minutes of history to fetch
|
||||
source: str = "alpaca" # "alpaca", "test", "file"
|
||||
|
||||
|
||||
@dataclass
|
||||
class BarIngestResult(ModuleResult):
|
||||
"""Output contract for ingestion module."""
|
||||
|
||||
minute_rows_inserted: int = 0
|
||||
symbols_updated: list[str] = field(default_factory=list)
|
||||
source_health: dict[str, str] = field(default_factory=dict) # {source: "healthy"|"error"}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# RESEARCH MODULE CONTRACTS
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResearchRequest:
|
||||
"""Input contract for research module."""
|
||||
|
||||
symbols: list[str]
|
||||
session_date: date
|
||||
lookback_days: int = 30 # for daily history
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResearchResult(ModuleResult):
|
||||
"""Output contract for research module."""
|
||||
|
||||
symbols_researched: list[str] = field(default_factory=list)
|
||||
symbols_with_data: list[str] = field(default_factory=list)
|
||||
symbols_missing_data: list[str] = field(default_factory=list)
|
||||
total_bars_available: int = 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# EVALUATION MODULE CONTRACTS
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class CriterionDetail:
|
||||
"""Result of evaluating one criterion."""
|
||||
|
||||
name: str
|
||||
passed: bool
|
||||
value: float # actual metric value
|
||||
threshold: float | None = None # minimum to pass
|
||||
reason: str = ""
|
||||
confidence: float = 1.0 # 0-1, how confident in this result
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvaluationRequest:
|
||||
"""Input contract for evaluation module."""
|
||||
|
||||
symbol: str
|
||||
bars_df: Any # pd.DataFrame with {timestamp, open, high, low, close, volume}
|
||||
previous_close: float
|
||||
avg_daily_volume: float
|
||||
session_date: date
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvaluationResult(ModuleResult):
|
||||
"""Output contract for evaluation module."""
|
||||
|
||||
symbol: str = ""
|
||||
status: str = "late" # "ready", "blocked", "late", "error"
|
||||
score: float = 0.0 # 0-100
|
||||
criteria: list[CriterionDetail] = field(default_factory=list)
|
||||
entry_price: float | None = None
|
||||
stop_loss: float | None = None
|
||||
target_price: float | None = None
|
||||
quality_score: float = 0.0 # 0-1
|
||||
blocking_reason: str | None = None
|
||||
metrics_dict: dict[str, float] = field(default_factory=dict) # gap%, rvol, etc.
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# EXECUTION MODULE CONTRACTS
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecutionRequest:
|
||||
"""Input contract for execution module."""
|
||||
|
||||
symbol: str
|
||||
entry_price: float
|
||||
stop_price: float
|
||||
target_price: float
|
||||
score: float
|
||||
criteria_count: int
|
||||
account_equity: float
|
||||
current_positions: int
|
||||
today_realized_pnl: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecutionResult(ModuleResult):
|
||||
"""Output contract for execution module."""
|
||||
|
||||
symbol: str = ""
|
||||
status: str = "" # "approved", "rejected", "pending_approval", "error"
|
||||
reason: str = "" # why it was approved/rejected
|
||||
order_id: str | None = None
|
||||
shares: int | None = None
|
||||
risk_dollars: float | None = None
|
||||
risk_checks_applied: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# BROKER MODULE CONTRACTS
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrokerRequest:
|
||||
"""Input contract for broker module."""
|
||||
|
||||
symbol: str
|
||||
order_id: str
|
||||
entry_price: float
|
||||
stop_price: float
|
||||
target_price: float
|
||||
shares: int
|
||||
order_type: str = "limit" # "limit" or "market"
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrokerResult(ModuleResult):
|
||||
"""Output contract for broker module."""
|
||||
|
||||
symbol: str = ""
|
||||
status: str = "" # "submitted", "rejected", "error"
|
||||
broker_order_id: str | None = None
|
||||
submitted_at: datetime | None = None
|
||||
@@ -1,274 +0,0 @@
|
||||
"""Telemetry collection and event emission for module observability."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Callable
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from core.contracts import ModuleMetrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TelemetryCollector:
|
||||
"""Collect metrics during module execution."""
|
||||
|
||||
module_name: str
|
||||
start_time: float = field(default_factory=time.time)
|
||||
input_count: int = 0
|
||||
output_count: int = 0
|
||||
error_count: int = 0
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
custom_metrics: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def record_input(self, count: int = 1) -> None:
|
||||
"""Record input items processed."""
|
||||
self.input_count += count
|
||||
|
||||
def record_output(self, count: int = 1) -> None:
|
||||
"""Record output items produced."""
|
||||
self.output_count += count
|
||||
|
||||
def record_error(self) -> None:
|
||||
"""Record an error occurred."""
|
||||
self.error_count += 1
|
||||
|
||||
def record_warning(self, message: str) -> None:
|
||||
"""Record a warning."""
|
||||
self.warnings.append(message)
|
||||
|
||||
def record_metric(self, name: str, value: float | int | str) -> None:
|
||||
"""Record a custom metric."""
|
||||
self.custom_metrics[name] = value
|
||||
|
||||
def finalize(self) -> ModuleMetrics:
|
||||
"""Finalize and return metrics."""
|
||||
duration_ms = (time.time() - self.start_time) * 1000
|
||||
return ModuleMetrics(
|
||||
module_name=self.module_name,
|
||||
timestamp=datetime.now(),
|
||||
duration_ms=duration_ms,
|
||||
input_count=self.input_count,
|
||||
output_count=self.output_count,
|
||||
error_count=self.error_count,
|
||||
warnings=self.warnings,
|
||||
custom_metrics=self.custom_metrics,
|
||||
)
|
||||
|
||||
|
||||
class TelemetryEmitter:
|
||||
"""Emit telemetry events to event store and observability systems."""
|
||||
|
||||
def __init__(self, event_store=None):
|
||||
"""Initialize emitter.
|
||||
|
||||
Args:
|
||||
event_store: Optional EventStore for persisting telemetry events
|
||||
"""
|
||||
self.event_store = event_store
|
||||
self.callbacks: list[Callable[[dict], None]] = []
|
||||
|
||||
def add_callback(self, callback: Callable[[dict], None]) -> None:
|
||||
"""Add a callback to receive telemetry events."""
|
||||
self.callbacks.append(callback)
|
||||
|
||||
def emit_module_tick(
|
||||
self,
|
||||
module_name: str,
|
||||
stage: str,
|
||||
metrics: ModuleMetrics | None = None,
|
||||
errors: list[dict] | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Emit a module tick event.
|
||||
|
||||
Args:
|
||||
module_name: Name of module (e.g., "ingestion", "evaluation")
|
||||
stage: Stage name (e.g., "started", "processing", "completed")
|
||||
metrics: Optional ModuleMetrics object
|
||||
errors: Optional list of error details
|
||||
correlation_id: Correlation ID linking to signal_ready event
|
||||
"""
|
||||
event = {
|
||||
"type": "module_tick",
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"module": module_name,
|
||||
"stage": stage,
|
||||
"metrics": metrics.to_dict() if metrics else None,
|
||||
"errors": errors or [],
|
||||
"correlation_id": correlation_id,
|
||||
}
|
||||
self._emit(event)
|
||||
|
||||
def emit_data_validation(
|
||||
self,
|
||||
module_name: str,
|
||||
validation_type: str,
|
||||
valid: bool,
|
||||
details: dict,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Emit a data validation event.
|
||||
|
||||
Args:
|
||||
module_name: Name of module performing validation
|
||||
validation_type: Type of validation ("schema", "checksum", "business_rules")
|
||||
valid: Whether validation passed
|
||||
details: Validation details (errors, warnings, statistics)
|
||||
correlation_id: Optional correlation ID
|
||||
"""
|
||||
event = {
|
||||
"type": "data_validation",
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"module": module_name,
|
||||
"validation_type": validation_type,
|
||||
"valid": valid,
|
||||
"details": details,
|
||||
"correlation_id": correlation_id,
|
||||
}
|
||||
self._emit(event)
|
||||
|
||||
def emit_symbol_evaluation_detail(
|
||||
self,
|
||||
symbol: str,
|
||||
stage: str,
|
||||
metrics: dict[str, Any],
|
||||
passed: bool,
|
||||
blocking_reason: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Emit detailed evaluation event for a symbol.
|
||||
|
||||
Args:
|
||||
symbol: Symbol being evaluated
|
||||
stage: Evaluation stage (e.g., "bars_loaded", "gap_checked", "rvol_calculated")
|
||||
metrics: Metrics at this stage
|
||||
passed: Whether this stage passed
|
||||
blocking_reason: If not passed, why
|
||||
correlation_id: Optional correlation ID
|
||||
"""
|
||||
event = {
|
||||
"type": "symbol_evaluation_detail",
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"symbol": symbol,
|
||||
"stage": stage,
|
||||
"metrics": metrics,
|
||||
"passed": passed,
|
||||
"blocking_reason": blocking_reason,
|
||||
"correlation_id": correlation_id,
|
||||
}
|
||||
self._emit(event)
|
||||
|
||||
def _emit(self, event: dict) -> None:
|
||||
"""Emit event to all registered callbacks and event store."""
|
||||
logger.debug(f"Telemetry: {event['type']} - {event.get('module', event.get('symbol', 'n/a'))}")
|
||||
|
||||
# Call all registered callbacks
|
||||
for callback in self.callbacks:
|
||||
try:
|
||||
callback(event)
|
||||
except Exception as e:
|
||||
logger.error(f"Telemetry callback failed: {e}")
|
||||
|
||||
# Emit to event store if available
|
||||
if self.event_store:
|
||||
try:
|
||||
# Import here to avoid circular dependency
|
||||
from storage.event_schema import ModuleTickEvent, DataValidationEvent
|
||||
|
||||
if event["type"] == "module_tick":
|
||||
self.event_store.emit(
|
||||
ModuleTickEvent(
|
||||
timestamp=datetime.now(),
|
||||
module=event["module"],
|
||||
stage=event["stage"],
|
||||
duration_ms=event["metrics"]["duration_ms"]
|
||||
if event["metrics"]
|
||||
else 0,
|
||||
input_count=event["metrics"]["input_count"]
|
||||
if event["metrics"]
|
||||
else 0,
|
||||
output_count=event["metrics"]["output_count"]
|
||||
if event["metrics"]
|
||||
else 0,
|
||||
metrics=event["metrics"]["custom_metrics"]
|
||||
if event["metrics"]
|
||||
else {},
|
||||
errors=event["errors"],
|
||||
correlation_id=event.get("correlation_id"),
|
||||
)
|
||||
)
|
||||
elif event["type"] == "data_validation":
|
||||
self.event_store.emit(
|
||||
DataValidationEvent(
|
||||
timestamp=datetime.now(),
|
||||
module=event["module"],
|
||||
validation_type=event["validation_type"],
|
||||
valid=event["valid"],
|
||||
details=event["details"],
|
||||
correlation_id=event.get("correlation_id"),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to emit telemetry event to store: {e}")
|
||||
|
||||
|
||||
# Global instance (can be overridden in tests)
|
||||
_telemetry_emitter: TelemetryEmitter | None = None
|
||||
|
||||
|
||||
def set_telemetry_emitter(emitter: TelemetryEmitter) -> None:
|
||||
"""Set the global telemetry emitter."""
|
||||
global _telemetry_emitter
|
||||
_telemetry_emitter = emitter
|
||||
|
||||
|
||||
def get_telemetry_emitter() -> TelemetryEmitter:
|
||||
"""Get the global telemetry emitter."""
|
||||
global _telemetry_emitter
|
||||
if _telemetry_emitter is None:
|
||||
_telemetry_emitter = TelemetryEmitter()
|
||||
return _telemetry_emitter
|
||||
|
||||
|
||||
def emit_module_tick(
|
||||
module_name: str,
|
||||
stage: str,
|
||||
metrics: ModuleMetrics | None = None,
|
||||
errors: list[dict] | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Convenience function to emit module tick."""
|
||||
get_telemetry_emitter().emit_module_tick(module_name, stage, metrics, errors, correlation_id)
|
||||
|
||||
|
||||
def emit_data_validation(
|
||||
module_name: str,
|
||||
validation_type: str,
|
||||
valid: bool,
|
||||
details: dict,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Convenience function to emit data validation."""
|
||||
get_telemetry_emitter().emit_data_validation(
|
||||
module_name, validation_type, valid, details, correlation_id
|
||||
)
|
||||
|
||||
|
||||
def emit_symbol_evaluation_detail(
|
||||
symbol: str,
|
||||
stage: str,
|
||||
metrics: dict[str, Any],
|
||||
passed: bool,
|
||||
blocking_reason: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Convenience function to emit symbol evaluation detail."""
|
||||
get_telemetry_emitter().emit_symbol_evaluation_detail(
|
||||
symbol, stage, metrics, passed, blocking_reason, correlation_id
|
||||
)
|
||||
@@ -1,284 +0,0 @@
|
||||
"""Data validation utilities for module boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Callable
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from core.contracts import ValidationFailure
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaValidator:
|
||||
"""Validate data against a schema."""
|
||||
|
||||
required_fields: dict[str, type] # {field_name: expected_type}
|
||||
optional_fields: dict[str, type] | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.optional_fields is None:
|
||||
self.optional_fields = {}
|
||||
|
||||
def validate(self, data: dict) -> tuple[bool, list[dict]]:
|
||||
"""Validate data against schema.
|
||||
|
||||
Args:
|
||||
data: Dictionary to validate
|
||||
|
||||
Returns:
|
||||
(is_valid, list of failure dicts)
|
||||
"""
|
||||
failures = []
|
||||
|
||||
# Check required fields
|
||||
for field_name, expected_type in self.required_fields.items():
|
||||
if field_name not in data:
|
||||
failures.append(
|
||||
{
|
||||
"field": field_name,
|
||||
"value": None,
|
||||
"reason": "Required field missing",
|
||||
"severity": "error",
|
||||
}
|
||||
)
|
||||
elif not isinstance(data[field_name], expected_type):
|
||||
failures.append(
|
||||
{
|
||||
"field": field_name,
|
||||
"value": data[field_name],
|
||||
"reason": f"Expected {expected_type.__name__}, got {type(data[field_name]).__name__}",
|
||||
"severity": "error",
|
||||
}
|
||||
)
|
||||
|
||||
# Check optional fields if present
|
||||
if self.optional_fields:
|
||||
for field_name, expected_type in self.optional_fields.items():
|
||||
if field_name in data and not isinstance(data[field_name], expected_type):
|
||||
failures.append(
|
||||
{
|
||||
"field": field_name,
|
||||
"value": data[field_name],
|
||||
"reason": f"Expected {expected_type.__name__}, got {type(data[field_name]).__name__}",
|
||||
"severity": "warning",
|
||||
}
|
||||
)
|
||||
|
||||
return len([f for f in failures if f.severity == "error"]) == 0, failures
|
||||
|
||||
|
||||
class RangeValidator:
|
||||
"""Validate numeric values are within expected ranges."""
|
||||
|
||||
def __init__(self, min_val: float | None = None, max_val: float | None = None):
|
||||
self.min_val = min_val
|
||||
self.max_val = max_val
|
||||
|
||||
def validate(self, value: float, field_name: str = "value") -> tuple[bool, dict | None]:
|
||||
"""Validate value is in range.
|
||||
|
||||
Args:
|
||||
value: Numeric value to validate
|
||||
field_name: Name of field for error message
|
||||
|
||||
Returns:
|
||||
(is_valid, failure dict or None)
|
||||
"""
|
||||
if self.min_val is not None and value < self.min_val:
|
||||
return False, {
|
||||
"field": field_name,
|
||||
"value": value,
|
||||
"reason": f"Value {value} below minimum {self.min_val}",
|
||||
"severity": "error",
|
||||
}
|
||||
|
||||
if self.max_val is not None and value > self.max_val:
|
||||
return False, {
|
||||
"field": field_name,
|
||||
"value": value,
|
||||
"reason": f"Value {value} above maximum {self.max_val}",
|
||||
"severity": "error",
|
||||
}
|
||||
|
||||
return True, None
|
||||
|
||||
|
||||
class ChecksumValidator:
|
||||
"""Validate data integrity using checksums."""
|
||||
|
||||
@staticmethod
|
||||
def compute_checksum(data: dict | list) -> str:
|
||||
"""Compute simple checksum for data.
|
||||
|
||||
Args:
|
||||
data: Dictionary or list to checksum
|
||||
|
||||
Returns:
|
||||
Hex checksum string
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
data_str = json.dumps(data, sort_keys=True, default=str)
|
||||
return hashlib.md5(data_str.encode()).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def validate_checksum(
|
||||
data: dict | list, expected_checksum: str
|
||||
) -> tuple[bool, dict | None]:
|
||||
"""Validate data matches checksum.
|
||||
|
||||
Args:
|
||||
data: Data to validate
|
||||
expected_checksum: Expected checksum value
|
||||
|
||||
Returns:
|
||||
(is_valid, failure dict or None)
|
||||
"""
|
||||
actual_checksum = ChecksumValidator.compute_checksum(data)
|
||||
if actual_checksum != expected_checksum:
|
||||
return False, {
|
||||
"field": "checksum",
|
||||
"value": actual_checksum,
|
||||
"reason": f"Checksum mismatch: expected {expected_checksum}, got {actual_checksum}",
|
||||
"severity": "error",
|
||||
}
|
||||
return True, None
|
||||
|
||||
|
||||
class BusinessRuleValidator:
|
||||
"""Validate data against business logic rules."""
|
||||
|
||||
def __init__(self):
|
||||
self.rules: dict[str, Callable[[Any], tuple[bool, str]]] = {}
|
||||
|
||||
def add_rule(self, rule_name: str, rule_fn: Callable[[Any], tuple[bool, str]]) -> None:
|
||||
"""Add a business rule.
|
||||
|
||||
Args:
|
||||
rule_name: Name of the rule
|
||||
rule_fn: Function that takes data and returns (is_valid, reason)
|
||||
"""
|
||||
self.rules[rule_name] = rule_fn
|
||||
|
||||
def validate(self, data: Any) -> tuple[bool, list[dict]]:
|
||||
"""Apply all business rules.
|
||||
|
||||
Args:
|
||||
data: Data to validate
|
||||
|
||||
Returns:
|
||||
(is_valid, list of failure dicts)
|
||||
"""
|
||||
failures = []
|
||||
for rule_name, rule_fn in self.rules.items():
|
||||
try:
|
||||
is_valid, reason = rule_fn(data)
|
||||
if not is_valid:
|
||||
failures.append(
|
||||
{
|
||||
"field": "business_rule",
|
||||
"value": None,
|
||||
"reason": f"{rule_name}: {reason}",
|
||||
"severity": "error",
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
failures.append(
|
||||
{
|
||||
"field": "business_rule",
|
||||
"value": None,
|
||||
"reason": f"{rule_name} raised exception: {e}",
|
||||
"severity": "error",
|
||||
}
|
||||
)
|
||||
|
||||
return len(failures) == 0, failures
|
||||
|
||||
|
||||
# Concrete validators for common cases
|
||||
|
||||
class BarDataValidator:
|
||||
"""Validate OHLCV bar data."""
|
||||
|
||||
@staticmethod
|
||||
def validate_bar(bar: dict) -> tuple[bool, list[dict]]:
|
||||
"""Validate a single bar.
|
||||
|
||||
Args:
|
||||
bar: Bar dictionary with {timestamp, open, high, low, close, volume}
|
||||
|
||||
Returns:
|
||||
(is_valid, list of failure dicts)
|
||||
"""
|
||||
failures = []
|
||||
|
||||
# Check required fields
|
||||
required = {"timestamp", "open", "high", "low", "close", "volume"}
|
||||
for field in required:
|
||||
if field not in bar:
|
||||
failures.append(
|
||||
{
|
||||
"field": field,
|
||||
"value": None,
|
||||
"reason": "Missing required field",
|
||||
"severity": "error",
|
||||
}
|
||||
)
|
||||
|
||||
if failures:
|
||||
return False, failures
|
||||
|
||||
# Check value consistency
|
||||
open_val = bar["open"]
|
||||
high_val = bar["high"]
|
||||
low_val = bar["low"]
|
||||
close_val = bar["close"]
|
||||
volume_val = bar["volume"]
|
||||
|
||||
# High must be >= all others
|
||||
if high_val < open_val or high_val < close_val or high_val < low_val:
|
||||
failures.append(
|
||||
{
|
||||
"field": "high",
|
||||
"value": high_val,
|
||||
"reason": f"High ({high_val}) must be >= open ({open_val}), close ({close_val}), low ({low_val})",
|
||||
"severity": "error",
|
||||
}
|
||||
)
|
||||
|
||||
# Low must be <= all others
|
||||
if low_val > open_val or low_val > close_val or low_val > high_val:
|
||||
failures.append(
|
||||
{
|
||||
"field": "low",
|
||||
"value": low_val,
|
||||
"reason": f"Low ({low_val}) must be <= open ({open_val}), close ({close_val}), high ({high_val})",
|
||||
"severity": "error",
|
||||
}
|
||||
)
|
||||
|
||||
# Volume should be non-negative
|
||||
if volume_val < 0:
|
||||
failures.append(
|
||||
{
|
||||
"field": "volume",
|
||||
"value": volume_val,
|
||||
"reason": "Volume cannot be negative",
|
||||
"severity": "error",
|
||||
}
|
||||
)
|
||||
|
||||
# Warn on zero volume
|
||||
if volume_val == 0:
|
||||
failures.append(
|
||||
{
|
||||
"field": "volume",
|
||||
"value": volume_val,
|
||||
"reason": "Zero volume bar",
|
||||
"severity": "warning",
|
||||
}
|
||||
)
|
||||
|
||||
return len([f for f in failures if f.severity == "error"]) == 0, failures
|
||||
+17
-2
@@ -19,6 +19,8 @@ from dotenv import load_dotenv
|
||||
# 192.168.1.5:5432) — the event store reads it. Without this the standalone
|
||||
# dashboard fell back to the legacy local DuckDB path and showed nothing.
|
||||
load_dotenv()
|
||||
from runtime.env_overrides import load as _load_overrides
|
||||
_load_overrides() # admin-panel overrides beat compose env
|
||||
|
||||
from api.main import DashboardState, create_server # noqa: E402
|
||||
|
||||
@@ -30,8 +32,21 @@ def main() -> None:
|
||||
# WATCHER_EVENT_DB_PATH is a legacy fallback only (the path is ignored).
|
||||
db_path = os.environ.get("DATABASE_URL") or os.environ.get(
|
||||
"WATCHER_EVENT_DB_PATH", "momentum")
|
||||
mode = os.environ.get("TRADING_EXECUTION_MODE", "alpaca_paper")
|
||||
state = DashboardState(db_path, execution_service=None, execution_mode=mode)
|
||||
from trading_mode import TradingModeSettings
|
||||
mode = TradingModeSettings.from_env().execution_mode
|
||||
# Sparklines: the standalone dashboard needs the SAME research-DB reader the
|
||||
# embedded (orchestrator) mode passes — without it DashboardState.bars() short-
|
||||
# circuits to []. The missing wire was purely that this entrypoint never built
|
||||
# one (DATABASE_URL already reaches the market schema). Read-only use; degrade
|
||||
# to None (charts show "no research db") if the connection can't be opened.
|
||||
research_con = None
|
||||
try:
|
||||
from research.multi_schema import open_research_db
|
||||
research_con = open_research_db("market")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"[dashboard] research DB unavailable, sparklines disabled: {exc}")
|
||||
state = DashboardState(db_path, execution_service=None, execution_mode=mode,
|
||||
research_con=research_con)
|
||||
server = create_server(state, host, port)
|
||||
db_disp = db_path.split("@")[-1] if "@" in db_path else db_path
|
||||
print(f"[dashboard] serving http://{host}:{port} (event db: {db_disp}, mode: {mode})")
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# Deploying Momentum
|
||||
|
||||
Three supported shapes, one image/codebase. Pick by where it runs:
|
||||
|
||||
| Mode | Status | Use when | Docs |
|
||||
|---|---|---|---|
|
||||
| **Docker Compose** | ✅ **canonical** (production since 2026-07-09) | Any always-on Linux host; TrueNAS SCALE | [docker/README.md](docker/README.md), `docker-compose.yml` (root), [docker/truenas-compose.yml](docker/truenas-compose.yml) |
|
||||
| **Local (systemd)** | ✅ maintained | Development machine, WSL, bare metal without Docker | [systemd/README.md](systemd/README.md) + unit templates in `deploy/systemd/` |
|
||||
| **Helm / Kubernetes** | 🚧 planned refresh | Future k8s hosting | [helm/](helm/) — the chart predates the 2026-07 architecture; starting point, not deployable |
|
||||
|
||||
## Docker Compose (canonical)
|
||||
|
||||
The root `docker-compose.yml` runs the full stack self-contained: postgres 18,
|
||||
trading loop, dashboard (:8765), admin panel (:8767), telegram bot, transcripts
|
||||
pipeline, nightly scheduler, on-demand jobs, optional Grafana. The image is
|
||||
published to `docker.io/fez5587/momentum-app` and `ghcr.io/fez5587/momentum-app`
|
||||
by CI on every master push.
|
||||
|
||||
For TrueNAS SCALE use `docker/truenas-compose.yml` (pull-only, dataset host
|
||||
paths, inline env). **Read the gotchas in docker/README.md before editing the
|
||||
YAML** — anchor expansion, per-service env, and path case-sensitivity have each
|
||||
caused an outage.
|
||||
|
||||
## Local (systemd user services)
|
||||
|
||||
The original deployment: unit templates in `deploy/systemd/` running a git
|
||||
checkout + venv, config from `.env`. Still the best loop for development —
|
||||
tests, backtests, and research scripts run here. Keep the units **disabled**
|
||||
while the compose stack owns production: two loops against one Alpaca account
|
||||
create duplicate orders.
|
||||
|
||||
## Helm (future)
|
||||
|
||||
`deploy/helm/` holds the original chart (single app deployment + CronJobs). It
|
||||
predates the service split, the session gate, and the env-flag inventory; the
|
||||
refresh is tracked in GitHub issues. Until then, compose is the way.
|
||||
|
||||
## Config in all modes
|
||||
|
||||
- `.env.example` (committed, sanitized) is the full variable inventory and
|
||||
says which layer each belongs in.
|
||||
- Behavior flags that MUST be set explicitly in any deployment — silent
|
||||
defaults bite: `TRADING_AUTO_APPROVE=1` (else entries queue for manual
|
||||
approval), `ALPACA_DATA_FEED=sip`, `OLLAMA_ENABLED=true` (catalyst engine),
|
||||
`FMP_API_KEY` (float cache). `TRADING_SESSION_ONLY` defaults ON in
|
||||
deployed configs (bars after-hours entries).
|
||||
- Runtime overrides: the admin panel writes `data/overrides.env`, which every
|
||||
service loads over its baked env at boot (`runtime/env_overrides.py`).
|
||||
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,19 @@
|
||||
# Grafana datasource for the docker-compose stack — points at the in-network
|
||||
# `postgres` service (not the LAN IP used by grafana/provisioning/datasources).
|
||||
# Credentials are interpolated from the PG_* env vars set on the grafana service.
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Momentum
|
||||
uid: momentum_pg
|
||||
type: postgres
|
||||
access: proxy
|
||||
url: postgres:5432
|
||||
user: ${PG_USER}
|
||||
jsonData:
|
||||
database: ${PG_DATABASE}
|
||||
sslmode: disable
|
||||
postgresVersion: 1600
|
||||
secureJsonData:
|
||||
password: ${PG_PASSWORD}
|
||||
isDefault: true
|
||||
editable: true
|
||||
@@ -0,0 +1,117 @@
|
||||
# Running the Momentum stack in Docker Compose
|
||||
|
||||
One image, six services: `postgres`, `loop`, `dashboard` (:8765), `qabot`
|
||||
(Telegram), `transcripts`, `scheduler` (20:00 ET weekday housekeeping), plus
|
||||
on-demand `jobs` and an optional `monitoring` (Grafana) profile.
|
||||
|
||||
```bash
|
||||
docker compose build
|
||||
docker compose up -d
|
||||
docker compose ps # healthchecks: loop heartbeat, dashboard HTTP, pg_isready
|
||||
docker compose logs -f loop
|
||||
```
|
||||
|
||||
Secrets live in `.env` (gitignored, never in the image). The compose file
|
||||
overrides `DATABASE_URL` to the in-stack postgres so the stack is
|
||||
self-contained.
|
||||
|
||||
## Migrating the production DB into the stack
|
||||
|
||||
The live DB today is native Postgres 18 on 192.168.1.5. To move it into the
|
||||
compose volume:
|
||||
|
||||
```bash
|
||||
# 1. dump the live DB (~4 GB as of 2026-07; events retention keeps it bounded)
|
||||
pg_dump -h 192.168.1.5 -U admin -Fc momentum > momentum.dump
|
||||
|
||||
# 2. start ONLY postgres, restore into it
|
||||
docker compose up -d postgres
|
||||
docker compose exec -T postgres pg_restore -U momentum -d momentum --no-owner < momentum.dump
|
||||
|
||||
# 3. bring up the rest
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
Until then you can run the app services against the LAN DB (transition mode):
|
||||
comment out the `DATABASE_URL` line in the `x-app-env` anchor and `.env`'s
|
||||
value (192.168.1.5) flows through.
|
||||
|
||||
## Where to host it
|
||||
|
||||
Recommended: **the DB box (192.168.1.5) or any always-on Linux host** — NOT
|
||||
the WSL machine. The week of 2026-07-06 produced three trading-hours outages
|
||||
from WSL restarts and networking-mode changes (see memory:
|
||||
wsl-virtioproxy-networking). A dedicated Linux host removes that whole class.
|
||||
WSL + Docker works for development/validation but inherits WSL's fragility.
|
||||
|
||||
Cutover checklist (from the systemd stack):
|
||||
1. `docker compose build && docker compose up -d` on the target host
|
||||
2. Verify: dashboard on :8765, `docker compose ps` all healthy, loop log shows
|
||||
ingest/watch/execute cycles, Telegram bot answers
|
||||
3. Diff Alpaca open orders — a crash-restart on cutover can resubmit resting
|
||||
entries (memory: momentum-live-loop-architecture)
|
||||
4. Disable the WSL systemd units: `systemctl --user disable --now
|
||||
momentum-loop momentum-dashboard momentum-qabot momentum-transcripts
|
||||
momentum-premarket-eval.timer`
|
||||
5. Keep `momentum-testpg` on the dev machine (tests only, not production)
|
||||
|
||||
Notes:
|
||||
- TZ inside containers is America/New_York (market-clock logic is ET).
|
||||
- `data/` (logs, learned_params.json, transcripts) lives in the `app-data`
|
||||
named volume, shared by all app services.
|
||||
- Ollama stays external (192.168.1.9, `OLLAMA_*` in .env); enrichment
|
||||
fails safe when it's down.
|
||||
- Do the cutover OUTSIDE market hours.
|
||||
|
||||
## Config layering: what goes where
|
||||
|
||||
`.env.example` is the committed, sanitized inventory of every variable
|
||||
(regenerated 2026-07-09). Three layers:
|
||||
|
||||
1. **Secrets — keep in `.env` in a protected space** (root-owned `chmod 600`,
|
||||
or a Docker secret): `ALPACA_API_KEY`/`ALPACA_SECRET_KEY`,
|
||||
`TELEGRAM_BOT_TOKEN`, `YOUTUBE_API_KEY`, `POSTGRES_PASSWORD`/`DATABASE_URL`,
|
||||
Discord webhooks, Schwab keys. The compose file reads them via `env_file`;
|
||||
they are never baked into the image.
|
||||
2. **Strategy knobs — fine to promote into the compose `environment:` block**
|
||||
if you want them versioned with the deployment: `TRADING_*`, `TRIGGER_*`
|
||||
(gap band, squeeze exception, quality floor, caps), `WATCHER_*`, `LIVE_*`.
|
||||
`environment:` overrides `env_file`, so promoted values win.
|
||||
3. **Infra addresses**: `OLLAMA_HOST` (external box, see below),
|
||||
`ALPACA_DATA_FEED=sip`, dashboard host/port — either layer works.
|
||||
|
||||
## External Ollama (news enrichment)
|
||||
|
||||
Ollama is the ONE external dependency: `OLLAMA_HOST=http://192.168.1.9:11434`
|
||||
reaches the LAN box from inside containers via normal outbound NAT — no
|
||||
compose changes needed. If you ever run Ollama on the docker HOST itself, use
|
||||
`http://host.docker.internal:11434` and add
|
||||
`extra_hosts: ["host.docker.internal:host-gateway"]` to the app anchor.
|
||||
Enrichment is fully fail-safe: if Ollama is down, trading is unaffected
|
||||
(news ranking just loses the catalyst-type boost).
|
||||
|
||||
NOTE (2026-07-09): the live `.env` contains two commented-out PowerShell lines
|
||||
embedding the real Alpaca credentials — harmless there, but they nearly leaked
|
||||
into the committed `.env.example` (caught by the regeneration leak check).
|
||||
Consider deleting them from `.env`.
|
||||
|
||||
## TrueNAS gotchas (each one caused a real outage — read before editing)
|
||||
|
||||
1. **Anchors get expanded.** TrueNAS renders `x-app:`/`<<: *app` into inline
|
||||
per-service blocks. After the first save, the `x-app` block is decorative —
|
||||
env/volume edits there reach nothing. Edit each service's own block.
|
||||
2. **Verify env landed** with `docker exec <svc> sh -c 'echo $VAR'` — not by
|
||||
reading the YAML, and not via the admin panel (it reports its own container).
|
||||
3. **Paths are case-sensitive** and Docker auto-creates missing bind sources:
|
||||
`/mnt/Volume` vs `/mnt/volume` silently forked the database into two worlds.
|
||||
After any data move, verify by **size at destination** (`du -sh`), never `ls`.
|
||||
4. **Volume changes need container recreation** — Stop/Start or Update the app;
|
||||
a container restart keeps the old mounts (and a running postgres keeps
|
||||
writing to *deleted* files if you move its dir underneath it).
|
||||
5. **First deploy initializes postgres** — `POSTGRES_PASSWORD` only applies to
|
||||
an empty pgdata. A failed first attempt leaves a cluster with the old
|
||||
password; fix with `ALTER USER` via `docker exec ... psql -U momentum` or
|
||||
wipe before the real deploy.
|
||||
6. **uid 10001**: the app containers run non-root; `chown -R 10001:10001` the
|
||||
host data dir or every file-writer (scheduler TSVs, admin profiles,
|
||||
transcripts) hits `PermissionError`.
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
# Nightly logical backup of the momentum Postgres. Runs inside the db-backup
|
||||
# compose service — the postgres:18 image, so pg_dump always matches the
|
||||
# server major (the app image has NO pg_dump, and a Debian-stock client is an
|
||||
# older major that refuses to dump a newer server).
|
||||
#
|
||||
# pg_dump | gzip -> $BACKUP_DIR/momentum-YYYY-MM-DD.sql.gz
|
||||
# prune anything older than BACKUP_KEEP_DAYS (default 14)
|
||||
# write $BACKUP_DIR/last_backup_ok on success (freshness marker)
|
||||
#
|
||||
# Restore: gunzip -c momentum-<date>.sql.gz | psql "$DATABASE_URL"
|
||||
#
|
||||
# Run with bash (not sh): pipefail is what catches a mid-stream pg_dump death
|
||||
# that would otherwise leave a truncated-but-valid gzip behind.
|
||||
set -euo pipefail
|
||||
|
||||
BACKUP_DIR="${BACKUP_DIR:-/backups}"
|
||||
BACKUP_KEEP_DAYS="${BACKUP_KEEP_DAYS:-14}"
|
||||
: "${DATABASE_URL:?DATABASE_URL is required}"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
out="$BACKUP_DIR/momentum-$(date +%F).sql.gz"
|
||||
|
||||
echo "[backup] $(date) -> $out"
|
||||
pg_dump "$DATABASE_URL" | gzip > "$out.tmp"
|
||||
gunzip -t "$out.tmp"
|
||||
mv "$out.tmp" "$out"
|
||||
find "$BACKUP_DIR" -name 'momentum-*.sql.gz*' -mtime +"$BACKUP_KEEP_DAYS" -delete # *: also sweep stale .tmp from failed runs
|
||||
date > "$BACKUP_DIR/last_backup_ok"
|
||||
echo "[backup] OK $(du -h "$out" | cut -f1)"
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
# Seed the LIVE stack's fresh Postgres from the paper stack's nightly dump,
|
||||
# then wipe the money-record tables so live P&L starts at zero while every
|
||||
# research baseline (minute/daily bars, rvol + regime history, learned
|
||||
# setups) carries over. Run ON CUBE after the live stack's postgres is up.
|
||||
#
|
||||
# ./bootstrap_live_db.sh <backup.sql.gz> <live-postgres-container>
|
||||
# e.g. ./bootstrap_live_db.sh \
|
||||
# /mnt/Volume/Storage/Apps/momentum/data/backups/momentum-2026-07-28.sql.gz \
|
||||
# ix-momentum-live-postgres-1
|
||||
#
|
||||
# Idempotent-ish: restoring over a non-empty DB will error on existing objects;
|
||||
# for a re-run, recreate the live stack's pgdata volume first (it is DISPOSABLE
|
||||
# until the flip — that's the point of staging).
|
||||
set -euo pipefail
|
||||
|
||||
BACKUP="${1:?usage: bootstrap_live_db.sh <backup.sql.gz> <live-pg-container>}"
|
||||
CONTAINER="${2:?usage: bootstrap_live_db.sh <backup.sql.gz> <live-pg-container>}"
|
||||
PSQL=(docker exec -i "$CONTAINER" psql -U momentum -d momentum -v ON_ERROR_STOP=0 -q)
|
||||
|
||||
echo "[1/3] restoring $BACKUP into $CONTAINER ..."
|
||||
gunzip -c "$BACKUP" | "${PSQL[@]}" >/dev/null
|
||||
|
||||
echo "[2/3] zeroing money-record tables (all trades/P&L derive from events) ..."
|
||||
docker exec -i "$CONTAINER" psql -U momentum -d momentum -v ON_ERROR_STOP=1 <<'SQL'
|
||||
TRUNCATE events, telemetry_events;
|
||||
-- paper-only artifacts that must not leak into the live stack's views
|
||||
TRUNCATE simulated_orders, simulated_fills, paper_missed_trades, paper_sessions;
|
||||
SQL
|
||||
|
||||
echo "[3/3] sanity ..."
|
||||
docker exec -i "$CONTAINER" psql -U momentum -d momentum -t <<'SQL'
|
||||
SELECT 'minute_bars ' || count(*) || ' (want millions)' FROM minute_bars
|
||||
UNION ALL SELECT 'daily_bars ' || count(*) FROM daily_bars
|
||||
UNION ALL SELECT 'setup_events ' || count(*) FROM setup_events
|
||||
UNION ALL SELECT 'regime_scores ' || count(*) FROM regime_scores
|
||||
UNION ALL SELECT 'events (MUST be 0) ' || count(*) FROM events;
|
||||
SQL
|
||||
|
||||
echo "done. Copy the paper stack's data/learned_params.json into the live"
|
||||
echo "stack's data volume, then start the loop."
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
# Migrate the momentum Postgres from the existing instance to the TrueNAS
|
||||
# stack's new one, with verification. Safe to practice: the SOURCE is only
|
||||
# ever read (pg_dump), never written.
|
||||
#
|
||||
# ./deploy/docker/migrate_db.sh dump # 1. dump source -> momentum.dump
|
||||
# ./deploy/docker/migrate_db.sh restore # 2. restore dump -> target
|
||||
# ./deploy/docker/migrate_db.sh verify # 3. row-count diff source vs target
|
||||
#
|
||||
# Override via env: SRC_URL, DST_URL, DUMP_FILE, PG_BIN.
|
||||
# Requires postgres 18 client tools (server is 18.4) — defaults to the local
|
||||
# PG18 install at /home/philip/pg/bin when present.
|
||||
set -euo pipefail
|
||||
|
||||
PG_BIN="${PG_BIN:-$([ -x /home/philip/pg/bin/pg_dump ] && echo /home/philip/pg/bin || echo "")}"
|
||||
DUMP_FILE="${DUMP_FILE:-./momentum.dump}"
|
||||
SRC_URL="${SRC_URL:-postgresql://admin:CHANGE_ME@192.168.1.5:5432/momentum}"
|
||||
# target = the TrueNAS compose postgres, published on 5433 (see truenas-compose.yml)
|
||||
DST_URL="${DST_URL:-postgresql://momentum:CHANGE_ME@192.168.1.5:5433/momentum}"
|
||||
|
||||
pgdump="${PG_BIN:+$PG_BIN/}pg_dump"
|
||||
pgrestore="${PG_BIN:+$PG_BIN/}pg_restore"
|
||||
psql="${PG_BIN:+$PG_BIN/}psql"
|
||||
|
||||
# The big tables that matter: events (trading history), minute_bars/daily_bars
|
||||
# (the year of backtest truth), plus every small state table. Nothing excluded
|
||||
# by default; uncomment to skip pure observability bulk and shrink the dump:
|
||||
EXCLUDES=()
|
||||
# EXCLUDES+=(--exclude-table-data=scanner_snapshots --exclude-table-data=raw_fetch_attempts)
|
||||
|
||||
case "${1:-}" in
|
||||
dump)
|
||||
echo ">> dumping $SRC_URL -> $DUMP_FILE (custom format, compressed)"
|
||||
time "$pgdump" -Fc --no-owner --no-privileges "${EXCLUDES[@]}" \
|
||||
-d "$SRC_URL" -f "$DUMP_FILE"
|
||||
ls -lh "$DUMP_FILE"
|
||||
;;
|
||||
restore)
|
||||
echo ">> restoring $DUMP_FILE -> $DST_URL (4 parallel jobs)"
|
||||
time "$pgrestore" --no-owner --no-privileges --clean --if-exists -j 4 \
|
||||
-d "$DST_URL" "$DUMP_FILE"
|
||||
echo ">> ANALYZE for fresh planner stats"
|
||||
"$psql" -d "$DST_URL" -c "ANALYZE;"
|
||||
;;
|
||||
verify)
|
||||
echo ">> row counts, source vs target (should match):"
|
||||
q="SELECT relname, n_live_tup FROM pg_stat_user_tables
|
||||
WHERE schemaname='public' ORDER BY n_live_tup DESC LIMIT 12"
|
||||
paste <("$psql" -d "$SRC_URL" -At -F' ' -c "$q") \
|
||||
<("$psql" -d "$DST_URL" -At -F' ' -c "$q") \
|
||||
| awk '{printf " %-22s src=%-10s | %-22s dst=%s\n", $1, $2, $3, $4}'
|
||||
echo ">> newest event on each side:"
|
||||
for u in "$SRC_URL" "$DST_URL"; do
|
||||
"$psql" -d "$u" -At -c "SELECT max(timestamp) FROM events" || true
|
||||
done
|
||||
;;
|
||||
*)
|
||||
echo "usage: $0 dump|restore|verify"; exit 1 ;;
|
||||
esac
|
||||
@@ -0,0 +1,108 @@
|
||||
# Seed for /app/data/overrides.env on the LIVE stack — the admin panel
|
||||
# (port 8867.. n/a on live; edit via file or add admin service) manages this.
|
||||
# Every service loads it at boot; overrides BEAT compose env.
|
||||
|
||||
ALPACA_DATA_FEED=sip
|
||||
DISCOVER_INTERVAL_SECONDS=120
|
||||
DISCOVER_THREAD_ENABLED=1
|
||||
LIVE_BARS_INTERVAL_SECONDS=5
|
||||
LIVE_BARS_REFRESH_MINUTES=20
|
||||
NEWS_CATALYST_SCORE_ENABLED=1
|
||||
NEWS_DILUTION_VETO_ENABLED=1
|
||||
NEWS_ENRICH_BATCH_LIMIT=50
|
||||
NEWS_ENRICH_ENABLED=1
|
||||
NEWS_ENRICH_INTERVAL_SECONDS=120
|
||||
NEWS_ENRICH_LOOKBACK_HOURS=12
|
||||
NEWS_FEEDS=prnewswire=https://www.prnewswire.com/rss/news-releases-list.rss,businesswire=https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRWQ==,globenewswire=https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies
|
||||
NEWS_LOOKBACK_HOURS=8
|
||||
NEWS_RANK_BOOST=2.0
|
||||
# Ollama lives on 11434 (the standard port). The stack pointed at 30068/4000 with
|
||||
# a model that was never pulled, so catalyst enrichment produced nothing from
|
||||
# 2026-08-03 to 2026-09-13 — silently, because enrich_recent_news reports failure
|
||||
# in its return value rather than raising. Smoke-tested 2026-09-13: tools:latest
|
||||
# parses 8/8 headlines in ~2.8s each and classifies offering_dilution / fda_approval
|
||||
# correctly. Verify the model is pulled before changing either line.
|
||||
OLLAMA_ENABLED=true
|
||||
OLLAMA_HOST=http://192.168.1.5:11434
|
||||
OLLAMA_MODEL=tools:latest
|
||||
OLLAMA_TEMPERATURE=0.1
|
||||
OLLAMA_TIMEOUT_SECONDS=60
|
||||
TRADING_AUTO_APPROVE=1
|
||||
TRADING_BACKOUT_COOLDOWN_SECONDS=180
|
||||
TRADING_BLOCK_REENTRY_AFTER_EXIT=1
|
||||
TRADING_ENTRY_CONFIRM_BY_MINUTE=0
|
||||
TRADING_ENTRY_FILL_MODEL=marketable
|
||||
TRADING_ENTRY_INVALIDATE_PCT=0.015
|
||||
TRADING_ENTRY_ORDER_TYPE=limit
|
||||
TRADING_ENTRY_TIMEOUT_BARS=2
|
||||
TRADING_EOD_FLATTEN_ENABLED=1
|
||||
TRADING_EOD_FLATTEN_TIME=15:55
|
||||
TRADING_EXECUTION_ENABLED=1
|
||||
# EXIT MANAGEMENT — retuned 2026-09-13. The previous config stacked FOUR
|
||||
# independent ratchets (BE@+5% from the env default, step-trail every 0.25R,
|
||||
# prior-low trail, and profit-lock tiers). Together they capped every winner at
|
||||
# ~1R: over 62 live + 172 paper trades the bot kept only 9.3% of the move on
|
||||
# trades that ran >=20%, and held winners a median 10 min vs 19 min for losers.
|
||||
# Replaced with a single percentage giveback that starts only after +1R. The
|
||||
# 8/12/15/20% neighbourhood all beat the old config on BOTH stacks in BOTH
|
||||
# train and test splits (mean R 0.20-0.25 vs 0.125), so 12% is a plateau pick,
|
||||
# not a fitted one. Re-sweep with sweep_exits.py before changing.
|
||||
TRADING_EXIT_BREAKEVEN_R=1.0
|
||||
TRADING_EXIT_BREAKEVEN_PCT=0.0
|
||||
TRADING_EXIT_MANAGE_INTERVAL_SECONDS=4
|
||||
TRADING_EXIT_PROFIT_TIERS=
|
||||
TRADING_EXIT_THREAD_ENABLED=1
|
||||
TRADING_EXIT_THREAD_INTERVAL_SECONDS=1.0
|
||||
TRADING_EXIT_TRAIL_AFTER_R=1.0
|
||||
TRADING_EXIT_TRAIL_MODE=pct
|
||||
TRADING_EXIT_TRAIL_PCT=0.12
|
||||
TRADING_EXIT_TRAIL_R_STEP=0.0
|
||||
TRADING_LIQUIDITY_MAX_VOLUME_PCT=0.025
|
||||
TRADING_MAX_BACKOUTS_PER_SYMBOL=3
|
||||
TRADING_MAX_CONCURRENT_POSITIONS=10
|
||||
TRADING_MAX_DAILY_LOSS_PCT=0.05
|
||||
TRADING_MAX_FRESH_ENTRIES_PER_DAY=10
|
||||
TRADING_MAX_GROSS_NOTIONAL_PCT=0.60
|
||||
TRADING_MAX_ORDERS_PER_TICK=1
|
||||
TRADING_MAX_POSITION_PCT=0.16
|
||||
# Boot-time sizing fallback. calculate_position_size falls back to this ONLY when
|
||||
# equity was never readable; at the old 100000 a boot inside Tradier's ~01:06
|
||||
# maintenance window (equity reads 0.0 there — 16 times in 14 days) would have
|
||||
# sized 5% of 100k on a ~$250 account. Keep it BELOW real account equity so the
|
||||
# fallback can only ever under-size (live equity 241.75 on 2026-09-13).
|
||||
TRADING_DEFAULT_EQUITY=200
|
||||
TRADING_MAX_RISK_DOLLARS=50000
|
||||
TRADING_MIN_DAY_VOLUME=300000
|
||||
TRADING_MIN_ENTRY_GAP_PCT=0.10
|
||||
TRADING_MIN_QUALITY_SCORE=0.60
|
||||
TRADING_PRICE_MAX=20.0
|
||||
TRADING_PRICE_MIN=2.0
|
||||
TRADING_REENTRY_MIN_LOSS_PCT=0.01
|
||||
TRADING_REGIME_GATE_ENABLED=0
|
||||
TRADING_REGIME_GATE_P_MIN=0.5
|
||||
TRADING_REQUIRE_ABOVE_VWAP=1
|
||||
TRADING_REWARD_MULTIPLE=10.0
|
||||
TRADING_RISK_PER_TRADE_PCT=0.05
|
||||
TRADING_SESSION_ONLY=1
|
||||
TRADING_TRIGGER_SLIP_PCT=0.004
|
||||
TRADING_UNIFIED_ENTRY=1
|
||||
TRIGGER_ARM_INTERVAL_SECONDS=8
|
||||
TRIGGER_ENABLED=1
|
||||
TRIGGER_GAP_MAX=35
|
||||
TRIGGER_GAP_MIN=10.0
|
||||
TRIGGER_GAP_SQUEEZE_MIN=150
|
||||
TRIGGER_GAP_SQUEEZE_RVOL=5.0
|
||||
TRIGGER_INTERVAL_SECONDS=2
|
||||
TRIGGER_MAX_ARMED=10
|
||||
TRIGGER_MIN_DOLLAR_VOL=8000
|
||||
TRIGGER_PRICE_MAX_AGE=12
|
||||
TRIGGER_RVOL_MIN=2.0
|
||||
# Apply the shared tradeable-population gate before the watcher scores anything.
|
||||
# Without it the watcher wrote 191,458 criteria receipts across 57 symbols on
|
||||
# 2026-09-10, 80.6% of them for names outside the gap band. Set 0 to restore the
|
||||
# old evaluate-everything behaviour.
|
||||
WATCHER_POPULATION_GATE=1
|
||||
WATCHER_ENABLED=1
|
||||
WATCHER_INTERVAL_SECONDS=8
|
||||
WATCHER_PRICE_MAX=20
|
||||
WATCHER_PRICE_MIN=2
|
||||
@@ -0,0 +1,687 @@
|
||||
# TrueNAS SCALE custom-app compose - FLATTENED (no YAML anchors).
|
||||
# TrueNAS's app validator REJECTS anchors (x-app / <<: *app) with
|
||||
# 'Invalid YAML provided', so every service carries its own full env.
|
||||
# Anchored/DRY source of truth: deploy/docker/truenas-compose.yml.
|
||||
# Fill every <FILL-ME-*> before deploying (PG password repeats in all
|
||||
# DATABASE_URL blocks + postgres + db-backup). Regenerated? keep both in sync.
|
||||
#
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18
|
||||
environment:
|
||||
POSTGRES_USER: momentum
|
||||
POSTGRES_PASSWORD: <FILL-ME-PG-PASSWORD>
|
||||
POSTGRES_DB: momentum
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/pgdata:/var/lib/postgresql/data
|
||||
ports:
|
||||
- 5433:5432
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- pg_isready -U momentum -d momentum
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
loop:
|
||||
image: docker.io/fez5587/momentum-app:latest
|
||||
command:
|
||||
- python
|
||||
- -u
|
||||
- run_live_paper.py
|
||||
- --no-dashboard
|
||||
environment:
|
||||
ALPACA_API_KEY: <FILL-ME-ALPACA-KEY>
|
||||
ALPACA_BACKFILL_FEED: sip
|
||||
ALPACA_BASE_URL: https://paper-api.alpaca.markets/v2
|
||||
ALPACA_DATA_FEED: sip
|
||||
ALPACA_PAPER_SYNC_ENABLED: '1'
|
||||
ALPACA_SECRET_KEY: <FILL-ME-ALPACA-SECRET>
|
||||
DATABASE_URL: postgresql://momentum:<FILL-ME-PG-PASSWORD>@postgres:5432/momentum
|
||||
DEADMAN_DAILY_PING: '1'
|
||||
DEADMAN_STALE_MIN: '5'
|
||||
DISCOVER_INTERVAL_SECONDS: '120'
|
||||
FMP_API_KEY: <FILL-ME-FMP-KEY>
|
||||
INGEST_STALE_BLOCK: '0'
|
||||
INGEST_STALE_SECONDS: '180'
|
||||
LIVE_BARS_INTERVAL_SECONDS: '20'
|
||||
LIVE_BARS_REFRESH_MINUTES: '20'
|
||||
NEWS_CATALYST_SCORE_ENABLED: '1'
|
||||
NEWS_DILUTION_VETO_ENABLED: '1'
|
||||
NEWS_ENRICH_BATCH_LIMIT: '50'
|
||||
NEWS_ENRICH_ENABLED: '1'
|
||||
NEWS_ENRICH_INTERVAL_SECONDS: '120'
|
||||
NEWS_ENRICH_LOOKBACK_HOURS: '12'
|
||||
NEWS_FEEDS: prnewswire=https://www.prnewswire.com/rss/news-releases-list.rss,businesswire=https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRWQ==,globenewswire=https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies
|
||||
NEWS_LOOKBACK_HOURS: '8'
|
||||
NEWS_RANK_BOOST: '2.0'
|
||||
OLLAMA_ENABLED: '1'
|
||||
OLLAMA_HOST: http://192.168.1.5:30068
|
||||
OLLAMA_MODEL: qwen2.5:7b-instruct
|
||||
OLLAMA_TEMPERATURE: '0.1'
|
||||
OLLAMA_TIMEOUT_SECONDS: '60'
|
||||
ORB_BARS: '5'
|
||||
PG_SKIP_SCHEMA: '1'
|
||||
STRATEGY_SETUPS: opening_range_break
|
||||
TELEGRAM_BOT_TOKEN: <FILL-ME-TELEGRAM-TOKEN>
|
||||
TELEGRAM_CHAT_ID: <FILL-ME-TELEGRAM-CHAT-ID>
|
||||
TRADING_AUTO_APPROVE: '1'
|
||||
TRADING_BACKOUT_COOLDOWN_SECONDS: '180'
|
||||
TRADING_BLOCK_REENTRY_AFTER_EXIT: '1'
|
||||
TRADING_ENTRY_CONFIRM_BY_MINUTE: '0'
|
||||
TRADING_ENTRY_FILL_MODEL: marketable
|
||||
TRADING_ENTRY_INVALIDATE_PCT: '0.015'
|
||||
TRADING_ENTRY_ORDER_TYPE: limit
|
||||
TRADING_ENTRY_TIMEOUT_BARS: '2'
|
||||
TRADING_EOD_FLATTEN_ENABLED: '1'
|
||||
TRADING_EOD_FLATTEN_TIME: '15:55'
|
||||
TRADING_EXECUTION_ENABLED: '1'
|
||||
TRADING_EXIT_BREAKEVEN_R: '1.0'
|
||||
TRADING_EXIT_PROFIT_TIERS: 8:3,15:9,25:18,40:30
|
||||
TRADING_EXIT_TRAIL_AFTER_R: '1.0'
|
||||
TRADING_EXIT_TRAIL_MODE: prior_low
|
||||
TRADING_EXIT_TRAIL_R_STEP: '0.25'
|
||||
TRADING_LIQUIDITY_MAX_VOLUME_PCT: '0.025'
|
||||
TRADING_MAX_BACKOUTS_PER_SYMBOL: '3'
|
||||
TRADING_MAX_CONCURRENT_POSITIONS: '10'
|
||||
TRADING_MAX_DAILY_LOSS_PCT: '0.05'
|
||||
TRADING_MAX_FRESH_ENTRIES_PER_DAY: '10'
|
||||
TRADING_MAX_GROSS_NOTIONAL_PCT: '0.60'
|
||||
TRADING_MAX_ORDERS_PER_TICK: '1'
|
||||
TRADING_MAX_POSITION_PCT: '0.16'
|
||||
TRADING_MAX_RISK_DOLLARS: '500'
|
||||
TRADING_MIN_DAY_VOLUME: '300000'
|
||||
TRADING_MIN_ENTRY_GAP_PCT: '0.10'
|
||||
TRADING_MIN_QUALITY_SCORE: '0.60'
|
||||
TRADING_PRICE_MAX: '20.0'
|
||||
TRADING_PRICE_MIN: '2.0'
|
||||
TRADING_REENTRY_MIN_LOSS_PCT: '0.01'
|
||||
TRADING_REGIME_GATE_ENABLED: '1'
|
||||
TRADING_REGIME_GATE_P_MIN: '0.5'
|
||||
TRADING_REQUIRE_ABOVE_VWAP: '1'
|
||||
TRADING_REWARD_MULTIPLE: '10.0'
|
||||
TRADING_RISK_PER_TRADE_PCT: '0.01'
|
||||
TRADING_SESSION_ONLY: '1'
|
||||
TRADING_TRIGGER_SLIP_PCT: '0.004'
|
||||
TRADING_UNIFIED_ENTRY: '1'
|
||||
TRIGGER_ARM_INTERVAL_SECONDS: '20'
|
||||
TRIGGER_ENABLED: '1'
|
||||
TRIGGER_GAP_MAX: '35'
|
||||
TRIGGER_GAP_MIN: '10.0'
|
||||
TRIGGER_GAP_SQUEEZE_MIN: '150'
|
||||
TRIGGER_GAP_SQUEEZE_RVOL: '5.0'
|
||||
TRIGGER_INTERVAL_SECONDS: '4'
|
||||
TRIGGER_MAX_ARMED: '10'
|
||||
TRIGGER_MIN_DOLLAR_VOL: '8000'
|
||||
TRIGGER_PRICE_MAX_AGE: '12'
|
||||
TRIGGER_RVOL_MIN: '2.0'
|
||||
TZ: America/New_York
|
||||
WATCHER_ENABLED: '1'
|
||||
WATCHER_INTERVAL_SECONDS: '15'
|
||||
WATCHER_PRICE_MAX: '20'
|
||||
WATCHER_PRICE_MIN: '2'
|
||||
YOUTUBE_API_KEY: <FILL-ME-YOUTUBE-KEY>
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/data:/app/data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python
|
||||
- -c
|
||||
- import os; os.environ['PG_SKIP_SCHEMA']='1'; import datetime,sys; from research.multi_schema import open_research_db; r=open_research_db('market').execute('SELECT max(timestamp) FROM events').fetchall()[0][0]; sys.exit(0 if r and (datetime.datetime.now()-r).total_seconds()<300 else 1)
|
||||
interval: 60s
|
||||
timeout: 20s
|
||||
retries: 5
|
||||
start_period: 180s
|
||||
dashboard:
|
||||
image: docker.io/fez5587/momentum-app:latest
|
||||
command:
|
||||
- python
|
||||
- -u
|
||||
- dashboard_api.py
|
||||
environment:
|
||||
ALPACA_API_KEY: <FILL-ME-ALPACA-KEY>
|
||||
ALPACA_BACKFILL_FEED: sip
|
||||
ALPACA_BASE_URL: https://paper-api.alpaca.markets/v2
|
||||
ALPACA_DATA_FEED: sip
|
||||
ALPACA_PAPER_SYNC_ENABLED: '1'
|
||||
ALPACA_SECRET_KEY: <FILL-ME-ALPACA-SECRET>
|
||||
DATABASE_URL: postgresql://momentum:<FILL-ME-PG-PASSWORD>@postgres:5432/momentum
|
||||
DEADMAN_DAILY_PING: '1'
|
||||
DEADMAN_STALE_MIN: '5'
|
||||
DISCOVER_INTERVAL_SECONDS: '120'
|
||||
FMP_API_KEY: <FILL-ME-FMP-KEY>
|
||||
INGEST_STALE_BLOCK: '0'
|
||||
INGEST_STALE_SECONDS: '180'
|
||||
LIVE_BARS_INTERVAL_SECONDS: '20'
|
||||
LIVE_BARS_REFRESH_MINUTES: '20'
|
||||
NEWS_CATALYST_SCORE_ENABLED: '1'
|
||||
NEWS_DILUTION_VETO_ENABLED: '1'
|
||||
NEWS_ENRICH_BATCH_LIMIT: '50'
|
||||
NEWS_ENRICH_ENABLED: '1'
|
||||
NEWS_ENRICH_INTERVAL_SECONDS: '120'
|
||||
NEWS_ENRICH_LOOKBACK_HOURS: '12'
|
||||
NEWS_FEEDS: prnewswire=https://www.prnewswire.com/rss/news-releases-list.rss,businesswire=https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRWQ==,globenewswire=https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies
|
||||
NEWS_LOOKBACK_HOURS: '8'
|
||||
NEWS_RANK_BOOST: '2.0'
|
||||
OLLAMA_ENABLED: '1'
|
||||
OLLAMA_HOST: http://192.168.1.5:30068
|
||||
OLLAMA_MODEL: qwen2.5:7b-instruct
|
||||
OLLAMA_TEMPERATURE: '0.1'
|
||||
OLLAMA_TIMEOUT_SECONDS: '60'
|
||||
ORB_BARS: '5'
|
||||
PG_SKIP_SCHEMA: '1'
|
||||
STRATEGY_SETUPS: opening_range_break
|
||||
TELEGRAM_BOT_TOKEN: <FILL-ME-TELEGRAM-TOKEN>
|
||||
TELEGRAM_CHAT_ID: <FILL-ME-TELEGRAM-CHAT-ID>
|
||||
TRADING_AUTO_APPROVE: '1'
|
||||
TRADING_BACKOUT_COOLDOWN_SECONDS: '180'
|
||||
TRADING_BLOCK_REENTRY_AFTER_EXIT: '1'
|
||||
TRADING_ENTRY_CONFIRM_BY_MINUTE: '0'
|
||||
TRADING_ENTRY_FILL_MODEL: marketable
|
||||
TRADING_ENTRY_INVALIDATE_PCT: '0.015'
|
||||
TRADING_ENTRY_ORDER_TYPE: limit
|
||||
TRADING_ENTRY_TIMEOUT_BARS: '2'
|
||||
TRADING_EOD_FLATTEN_ENABLED: '1'
|
||||
TRADING_EOD_FLATTEN_TIME: '15:55'
|
||||
TRADING_EXECUTION_ENABLED: '1'
|
||||
TRADING_EXIT_BREAKEVEN_R: '1.0'
|
||||
TRADING_EXIT_PROFIT_TIERS: 8:3,15:9,25:18,40:30
|
||||
TRADING_EXIT_TRAIL_AFTER_R: '1.0'
|
||||
TRADING_EXIT_TRAIL_MODE: prior_low
|
||||
TRADING_EXIT_TRAIL_R_STEP: '0.25'
|
||||
TRADING_LIQUIDITY_MAX_VOLUME_PCT: '0.025'
|
||||
TRADING_MAX_BACKOUTS_PER_SYMBOL: '3'
|
||||
TRADING_MAX_CONCURRENT_POSITIONS: '10'
|
||||
TRADING_MAX_DAILY_LOSS_PCT: '0.05'
|
||||
TRADING_MAX_FRESH_ENTRIES_PER_DAY: '10'
|
||||
TRADING_MAX_GROSS_NOTIONAL_PCT: '0.60'
|
||||
TRADING_MAX_ORDERS_PER_TICK: '1'
|
||||
TRADING_MAX_POSITION_PCT: '0.16'
|
||||
TRADING_MAX_RISK_DOLLARS: '500'
|
||||
TRADING_MIN_DAY_VOLUME: '300000'
|
||||
TRADING_MIN_ENTRY_GAP_PCT: '0.10'
|
||||
TRADING_MIN_QUALITY_SCORE: '0.60'
|
||||
TRADING_PRICE_MAX: '20.0'
|
||||
TRADING_PRICE_MIN: '2.0'
|
||||
TRADING_REENTRY_MIN_LOSS_PCT: '0.01'
|
||||
TRADING_REGIME_GATE_ENABLED: '1'
|
||||
TRADING_REGIME_GATE_P_MIN: '0.5'
|
||||
TRADING_REQUIRE_ABOVE_VWAP: '1'
|
||||
TRADING_REWARD_MULTIPLE: '10.0'
|
||||
TRADING_RISK_PER_TRADE_PCT: '0.01'
|
||||
TRADING_SESSION_ONLY: '1'
|
||||
TRADING_TRIGGER_SLIP_PCT: '0.004'
|
||||
TRADING_UNIFIED_ENTRY: '1'
|
||||
TRIGGER_ARM_INTERVAL_SECONDS: '20'
|
||||
TRIGGER_ENABLED: '1'
|
||||
TRIGGER_GAP_MAX: '35'
|
||||
TRIGGER_GAP_MIN: '10.0'
|
||||
TRIGGER_GAP_SQUEEZE_MIN: '150'
|
||||
TRIGGER_GAP_SQUEEZE_RVOL: '5.0'
|
||||
TRIGGER_INTERVAL_SECONDS: '4'
|
||||
TRIGGER_MAX_ARMED: '10'
|
||||
TRIGGER_MIN_DOLLAR_VOL: '8000'
|
||||
TRIGGER_PRICE_MAX_AGE: '12'
|
||||
TRIGGER_RVOL_MIN: '2.0'
|
||||
TZ: America/New_York
|
||||
WATCHER_ENABLED: '1'
|
||||
WATCHER_INTERVAL_SECONDS: '15'
|
||||
WATCHER_PRICE_MAX: '20'
|
||||
WATCHER_PRICE_MIN: '2'
|
||||
YOUTUBE_API_KEY: <FILL-ME-YOUTUBE-KEY>
|
||||
DASHBOARD_HOST: 0.0.0.0
|
||||
DASHBOARD_PORT: '8765'
|
||||
DASHBOARD_TOKEN: <FILL-ME-DASHBOARD-TOKEN>
|
||||
COMMAND_BUS_ENABLED: '1'
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/data:/app/data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- 8765:8765
|
||||
qabot:
|
||||
image: docker.io/fez5587/momentum-app:latest
|
||||
command:
|
||||
- python
|
||||
- -u
|
||||
- telegram_bot.py
|
||||
environment:
|
||||
ALPACA_API_KEY: <FILL-ME-ALPACA-KEY>
|
||||
ALPACA_BACKFILL_FEED: sip
|
||||
ALPACA_BASE_URL: https://paper-api.alpaca.markets/v2
|
||||
ALPACA_DATA_FEED: sip
|
||||
ALPACA_PAPER_SYNC_ENABLED: '1'
|
||||
ALPACA_SECRET_KEY: <FILL-ME-ALPACA-SECRET>
|
||||
DATABASE_URL: postgresql://momentum:<FILL-ME-PG-PASSWORD>@postgres:5432/momentum
|
||||
DEADMAN_DAILY_PING: '1'
|
||||
DEADMAN_STALE_MIN: '5'
|
||||
DISCOVER_INTERVAL_SECONDS: '120'
|
||||
FMP_API_KEY: <FILL-ME-FMP-KEY>
|
||||
INGEST_STALE_BLOCK: '0'
|
||||
INGEST_STALE_SECONDS: '180'
|
||||
LIVE_BARS_INTERVAL_SECONDS: '20'
|
||||
LIVE_BARS_REFRESH_MINUTES: '20'
|
||||
NEWS_CATALYST_SCORE_ENABLED: '1'
|
||||
NEWS_DILUTION_VETO_ENABLED: '1'
|
||||
NEWS_ENRICH_BATCH_LIMIT: '50'
|
||||
NEWS_ENRICH_ENABLED: '1'
|
||||
NEWS_ENRICH_INTERVAL_SECONDS: '120'
|
||||
NEWS_ENRICH_LOOKBACK_HOURS: '12'
|
||||
NEWS_FEEDS: prnewswire=https://www.prnewswire.com/rss/news-releases-list.rss,businesswire=https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRWQ==,globenewswire=https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies
|
||||
NEWS_LOOKBACK_HOURS: '8'
|
||||
NEWS_RANK_BOOST: '2.0'
|
||||
OLLAMA_ENABLED: '1'
|
||||
OLLAMA_HOST: http://192.168.1.5:30068
|
||||
OLLAMA_MODEL: qwen2.5:7b-instruct
|
||||
OLLAMA_TEMPERATURE: '0.1'
|
||||
OLLAMA_TIMEOUT_SECONDS: '60'
|
||||
ORB_BARS: '5'
|
||||
PG_SKIP_SCHEMA: '1'
|
||||
STRATEGY_SETUPS: opening_range_break
|
||||
TELEGRAM_BOT_TOKEN: <FILL-ME-TELEGRAM-TOKEN>
|
||||
TELEGRAM_CHAT_ID: <FILL-ME-TELEGRAM-CHAT-ID>
|
||||
TRADING_AUTO_APPROVE: '1'
|
||||
TRADING_BACKOUT_COOLDOWN_SECONDS: '180'
|
||||
TRADING_BLOCK_REENTRY_AFTER_EXIT: '1'
|
||||
TRADING_ENTRY_CONFIRM_BY_MINUTE: '0'
|
||||
TRADING_ENTRY_FILL_MODEL: marketable
|
||||
TRADING_ENTRY_INVALIDATE_PCT: '0.015'
|
||||
TRADING_ENTRY_ORDER_TYPE: limit
|
||||
TRADING_ENTRY_TIMEOUT_BARS: '2'
|
||||
TRADING_EOD_FLATTEN_ENABLED: '1'
|
||||
TRADING_EOD_FLATTEN_TIME: '15:55'
|
||||
TRADING_EXECUTION_ENABLED: '1'
|
||||
TRADING_EXIT_BREAKEVEN_R: '1.0'
|
||||
TRADING_EXIT_PROFIT_TIERS: 8:3,15:9,25:18,40:30
|
||||
TRADING_EXIT_TRAIL_AFTER_R: '1.0'
|
||||
TRADING_EXIT_TRAIL_MODE: prior_low
|
||||
TRADING_EXIT_TRAIL_R_STEP: '0.25'
|
||||
TRADING_LIQUIDITY_MAX_VOLUME_PCT: '0.025'
|
||||
TRADING_MAX_BACKOUTS_PER_SYMBOL: '3'
|
||||
TRADING_MAX_CONCURRENT_POSITIONS: '10'
|
||||
TRADING_MAX_DAILY_LOSS_PCT: '0.05'
|
||||
TRADING_MAX_FRESH_ENTRIES_PER_DAY: '10'
|
||||
TRADING_MAX_GROSS_NOTIONAL_PCT: '0.60'
|
||||
TRADING_MAX_ORDERS_PER_TICK: '1'
|
||||
TRADING_MAX_POSITION_PCT: '0.16'
|
||||
TRADING_MAX_RISK_DOLLARS: '500'
|
||||
TRADING_MIN_DAY_VOLUME: '300000'
|
||||
TRADING_MIN_ENTRY_GAP_PCT: '0.10'
|
||||
TRADING_MIN_QUALITY_SCORE: '0.60'
|
||||
TRADING_PRICE_MAX: '20.0'
|
||||
TRADING_PRICE_MIN: '2.0'
|
||||
TRADING_REENTRY_MIN_LOSS_PCT: '0.01'
|
||||
TRADING_REGIME_GATE_ENABLED: '1'
|
||||
TRADING_REGIME_GATE_P_MIN: '0.5'
|
||||
TRADING_REQUIRE_ABOVE_VWAP: '1'
|
||||
TRADING_REWARD_MULTIPLE: '10.0'
|
||||
TRADING_RISK_PER_TRADE_PCT: '0.01'
|
||||
TRADING_SESSION_ONLY: '1'
|
||||
TRADING_TRIGGER_SLIP_PCT: '0.004'
|
||||
TRADING_UNIFIED_ENTRY: '1'
|
||||
TRIGGER_ARM_INTERVAL_SECONDS: '20'
|
||||
TRIGGER_ENABLED: '1'
|
||||
TRIGGER_GAP_MAX: '35'
|
||||
TRIGGER_GAP_MIN: '10.0'
|
||||
TRIGGER_GAP_SQUEEZE_MIN: '150'
|
||||
TRIGGER_GAP_SQUEEZE_RVOL: '5.0'
|
||||
TRIGGER_INTERVAL_SECONDS: '4'
|
||||
TRIGGER_MAX_ARMED: '10'
|
||||
TRIGGER_MIN_DOLLAR_VOL: '8000'
|
||||
TRIGGER_PRICE_MAX_AGE: '12'
|
||||
TRIGGER_RVOL_MIN: '2.0'
|
||||
TZ: America/New_York
|
||||
WATCHER_ENABLED: '1'
|
||||
WATCHER_INTERVAL_SECONDS: '15'
|
||||
WATCHER_PRICE_MAX: '20'
|
||||
WATCHER_PRICE_MIN: '2'
|
||||
YOUTUBE_API_KEY: <FILL-ME-YOUTUBE-KEY>
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/data:/app/data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
transcripts:
|
||||
image: docker.io/fez5587/momentum-app:latest
|
||||
command:
|
||||
- python
|
||||
- -u
|
||||
- -m
|
||||
- youtube_claims.run
|
||||
- run
|
||||
environment:
|
||||
ALPACA_API_KEY: <FILL-ME-ALPACA-KEY>
|
||||
ALPACA_BACKFILL_FEED: sip
|
||||
ALPACA_BASE_URL: https://paper-api.alpaca.markets/v2
|
||||
ALPACA_DATA_FEED: sip
|
||||
ALPACA_PAPER_SYNC_ENABLED: '1'
|
||||
ALPACA_SECRET_KEY: <FILL-ME-ALPACA-SECRET>
|
||||
DATABASE_URL: postgresql://momentum:<FILL-ME-PG-PASSWORD>@postgres:5432/momentum
|
||||
DEADMAN_DAILY_PING: '1'
|
||||
DEADMAN_STALE_MIN: '5'
|
||||
DISCOVER_INTERVAL_SECONDS: '120'
|
||||
FMP_API_KEY: <FILL-ME-FMP-KEY>
|
||||
INGEST_STALE_BLOCK: '0'
|
||||
INGEST_STALE_SECONDS: '180'
|
||||
LIVE_BARS_INTERVAL_SECONDS: '20'
|
||||
LIVE_BARS_REFRESH_MINUTES: '20'
|
||||
NEWS_CATALYST_SCORE_ENABLED: '1'
|
||||
NEWS_DILUTION_VETO_ENABLED: '1'
|
||||
NEWS_ENRICH_BATCH_LIMIT: '50'
|
||||
NEWS_ENRICH_ENABLED: '1'
|
||||
NEWS_ENRICH_INTERVAL_SECONDS: '120'
|
||||
NEWS_ENRICH_LOOKBACK_HOURS: '12'
|
||||
NEWS_FEEDS: prnewswire=https://www.prnewswire.com/rss/news-releases-list.rss,businesswire=https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRWQ==,globenewswire=https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies
|
||||
NEWS_LOOKBACK_HOURS: '8'
|
||||
NEWS_RANK_BOOST: '2.0'
|
||||
OLLAMA_ENABLED: '1'
|
||||
OLLAMA_HOST: http://192.168.1.5:30068
|
||||
OLLAMA_MODEL: qwen2.5:7b-instruct
|
||||
OLLAMA_TEMPERATURE: '0.1'
|
||||
OLLAMA_TIMEOUT_SECONDS: '60'
|
||||
ORB_BARS: '5'
|
||||
PG_SKIP_SCHEMA: '1'
|
||||
STRATEGY_SETUPS: opening_range_break
|
||||
TELEGRAM_BOT_TOKEN: <FILL-ME-TELEGRAM-TOKEN>
|
||||
TELEGRAM_CHAT_ID: <FILL-ME-TELEGRAM-CHAT-ID>
|
||||
TRADING_AUTO_APPROVE: '1'
|
||||
TRADING_BACKOUT_COOLDOWN_SECONDS: '180'
|
||||
TRADING_BLOCK_REENTRY_AFTER_EXIT: '1'
|
||||
TRADING_ENTRY_CONFIRM_BY_MINUTE: '0'
|
||||
TRADING_ENTRY_FILL_MODEL: marketable
|
||||
TRADING_ENTRY_INVALIDATE_PCT: '0.015'
|
||||
TRADING_ENTRY_ORDER_TYPE: limit
|
||||
TRADING_ENTRY_TIMEOUT_BARS: '2'
|
||||
TRADING_EOD_FLATTEN_ENABLED: '1'
|
||||
TRADING_EOD_FLATTEN_TIME: '15:55'
|
||||
TRADING_EXECUTION_ENABLED: '1'
|
||||
TRADING_EXIT_BREAKEVEN_R: '1.0'
|
||||
TRADING_EXIT_PROFIT_TIERS: 8:3,15:9,25:18,40:30
|
||||
TRADING_EXIT_TRAIL_AFTER_R: '1.0'
|
||||
TRADING_EXIT_TRAIL_MODE: prior_low
|
||||
TRADING_EXIT_TRAIL_R_STEP: '0.25'
|
||||
TRADING_LIQUIDITY_MAX_VOLUME_PCT: '0.025'
|
||||
TRADING_MAX_BACKOUTS_PER_SYMBOL: '3'
|
||||
TRADING_MAX_CONCURRENT_POSITIONS: '10'
|
||||
TRADING_MAX_DAILY_LOSS_PCT: '0.05'
|
||||
TRADING_MAX_FRESH_ENTRIES_PER_DAY: '10'
|
||||
TRADING_MAX_GROSS_NOTIONAL_PCT: '0.60'
|
||||
TRADING_MAX_ORDERS_PER_TICK: '1'
|
||||
TRADING_MAX_POSITION_PCT: '0.16'
|
||||
TRADING_MAX_RISK_DOLLARS: '500'
|
||||
TRADING_MIN_DAY_VOLUME: '300000'
|
||||
TRADING_MIN_ENTRY_GAP_PCT: '0.10'
|
||||
TRADING_MIN_QUALITY_SCORE: '0.60'
|
||||
TRADING_PRICE_MAX: '20.0'
|
||||
TRADING_PRICE_MIN: '2.0'
|
||||
TRADING_REENTRY_MIN_LOSS_PCT: '0.01'
|
||||
TRADING_REGIME_GATE_ENABLED: '1'
|
||||
TRADING_REGIME_GATE_P_MIN: '0.5'
|
||||
TRADING_REQUIRE_ABOVE_VWAP: '1'
|
||||
TRADING_REWARD_MULTIPLE: '10.0'
|
||||
TRADING_RISK_PER_TRADE_PCT: '0.01'
|
||||
TRADING_SESSION_ONLY: '1'
|
||||
TRADING_TRIGGER_SLIP_PCT: '0.004'
|
||||
TRADING_UNIFIED_ENTRY: '1'
|
||||
TRIGGER_ARM_INTERVAL_SECONDS: '20'
|
||||
TRIGGER_ENABLED: '1'
|
||||
TRIGGER_GAP_MAX: '35'
|
||||
TRIGGER_GAP_MIN: '10.0'
|
||||
TRIGGER_GAP_SQUEEZE_MIN: '150'
|
||||
TRIGGER_GAP_SQUEEZE_RVOL: '5.0'
|
||||
TRIGGER_INTERVAL_SECONDS: '4'
|
||||
TRIGGER_MAX_ARMED: '10'
|
||||
TRIGGER_MIN_DOLLAR_VOL: '8000'
|
||||
TRIGGER_PRICE_MAX_AGE: '12'
|
||||
TRIGGER_RVOL_MIN: '2.0'
|
||||
TZ: America/New_York
|
||||
WATCHER_ENABLED: '1'
|
||||
WATCHER_INTERVAL_SECONDS: '15'
|
||||
WATCHER_PRICE_MAX: '20'
|
||||
WATCHER_PRICE_MIN: '2'
|
||||
YOUTUBE_API_KEY: <FILL-ME-YOUTUBE-KEY>
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/data:/app/data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
admin:
|
||||
image: docker.io/fez5587/momentum-app:latest
|
||||
command:
|
||||
- python
|
||||
- -u
|
||||
- admin_api.py
|
||||
environment:
|
||||
ALPACA_API_KEY: <FILL-ME-ALPACA-KEY>
|
||||
ALPACA_BACKFILL_FEED: sip
|
||||
ALPACA_BASE_URL: https://paper-api.alpaca.markets/v2
|
||||
ALPACA_DATA_FEED: sip
|
||||
ALPACA_PAPER_SYNC_ENABLED: '1'
|
||||
ALPACA_SECRET_KEY: <FILL-ME-ALPACA-SECRET>
|
||||
DATABASE_URL: postgresql://momentum:<FILL-ME-PG-PASSWORD>@postgres:5432/momentum
|
||||
DEADMAN_DAILY_PING: '1'
|
||||
DEADMAN_STALE_MIN: '5'
|
||||
DISCOVER_INTERVAL_SECONDS: '120'
|
||||
FMP_API_KEY: <FILL-ME-FMP-KEY>
|
||||
INGEST_STALE_BLOCK: '0'
|
||||
INGEST_STALE_SECONDS: '180'
|
||||
LIVE_BARS_INTERVAL_SECONDS: '20'
|
||||
LIVE_BARS_REFRESH_MINUTES: '20'
|
||||
NEWS_CATALYST_SCORE_ENABLED: '1'
|
||||
NEWS_DILUTION_VETO_ENABLED: '1'
|
||||
NEWS_ENRICH_BATCH_LIMIT: '50'
|
||||
NEWS_ENRICH_ENABLED: '1'
|
||||
NEWS_ENRICH_INTERVAL_SECONDS: '120'
|
||||
NEWS_ENRICH_LOOKBACK_HOURS: '12'
|
||||
NEWS_FEEDS: prnewswire=https://www.prnewswire.com/rss/news-releases-list.rss,businesswire=https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRWQ==,globenewswire=https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies
|
||||
NEWS_LOOKBACK_HOURS: '8'
|
||||
NEWS_RANK_BOOST: '2.0'
|
||||
OLLAMA_ENABLED: '1'
|
||||
OLLAMA_HOST: http://192.168.1.5:30068
|
||||
OLLAMA_MODEL: qwen2.5:7b-instruct
|
||||
OLLAMA_TEMPERATURE: '0.1'
|
||||
OLLAMA_TIMEOUT_SECONDS: '60'
|
||||
ORB_BARS: '5'
|
||||
PG_SKIP_SCHEMA: '1'
|
||||
STRATEGY_SETUPS: opening_range_break
|
||||
TELEGRAM_BOT_TOKEN: <FILL-ME-TELEGRAM-TOKEN>
|
||||
TELEGRAM_CHAT_ID: <FILL-ME-TELEGRAM-CHAT-ID>
|
||||
TRADING_AUTO_APPROVE: '1'
|
||||
TRADING_BACKOUT_COOLDOWN_SECONDS: '180'
|
||||
TRADING_BLOCK_REENTRY_AFTER_EXIT: '1'
|
||||
TRADING_ENTRY_CONFIRM_BY_MINUTE: '0'
|
||||
TRADING_ENTRY_FILL_MODEL: marketable
|
||||
TRADING_ENTRY_INVALIDATE_PCT: '0.015'
|
||||
TRADING_ENTRY_ORDER_TYPE: limit
|
||||
TRADING_ENTRY_TIMEOUT_BARS: '2'
|
||||
TRADING_EOD_FLATTEN_ENABLED: '1'
|
||||
TRADING_EOD_FLATTEN_TIME: '15:55'
|
||||
TRADING_EXECUTION_ENABLED: '1'
|
||||
TRADING_EXIT_BREAKEVEN_R: '1.0'
|
||||
TRADING_EXIT_PROFIT_TIERS: 8:3,15:9,25:18,40:30
|
||||
TRADING_EXIT_TRAIL_AFTER_R: '1.0'
|
||||
TRADING_EXIT_TRAIL_MODE: prior_low
|
||||
TRADING_EXIT_TRAIL_R_STEP: '0.25'
|
||||
TRADING_LIQUIDITY_MAX_VOLUME_PCT: '0.025'
|
||||
TRADING_MAX_BACKOUTS_PER_SYMBOL: '3'
|
||||
TRADING_MAX_CONCURRENT_POSITIONS: '10'
|
||||
TRADING_MAX_DAILY_LOSS_PCT: '0.05'
|
||||
TRADING_MAX_FRESH_ENTRIES_PER_DAY: '10'
|
||||
TRADING_MAX_GROSS_NOTIONAL_PCT: '0.60'
|
||||
TRADING_MAX_ORDERS_PER_TICK: '1'
|
||||
TRADING_MAX_POSITION_PCT: '0.16'
|
||||
TRADING_MAX_RISK_DOLLARS: '500'
|
||||
TRADING_MIN_DAY_VOLUME: '300000'
|
||||
TRADING_MIN_ENTRY_GAP_PCT: '0.10'
|
||||
TRADING_MIN_QUALITY_SCORE: '0.60'
|
||||
TRADING_PRICE_MAX: '20.0'
|
||||
TRADING_PRICE_MIN: '2.0'
|
||||
TRADING_REENTRY_MIN_LOSS_PCT: '0.01'
|
||||
TRADING_REGIME_GATE_ENABLED: '1'
|
||||
TRADING_REGIME_GATE_P_MIN: '0.5'
|
||||
TRADING_REQUIRE_ABOVE_VWAP: '1'
|
||||
TRADING_REWARD_MULTIPLE: '10.0'
|
||||
TRADING_RISK_PER_TRADE_PCT: '0.01'
|
||||
TRADING_SESSION_ONLY: '1'
|
||||
TRADING_TRIGGER_SLIP_PCT: '0.004'
|
||||
TRADING_UNIFIED_ENTRY: '1'
|
||||
TRIGGER_ARM_INTERVAL_SECONDS: '20'
|
||||
TRIGGER_ENABLED: '1'
|
||||
TRIGGER_GAP_MAX: '35'
|
||||
TRIGGER_GAP_MIN: '10.0'
|
||||
TRIGGER_GAP_SQUEEZE_MIN: '150'
|
||||
TRIGGER_GAP_SQUEEZE_RVOL: '5.0'
|
||||
TRIGGER_INTERVAL_SECONDS: '4'
|
||||
TRIGGER_MAX_ARMED: '10'
|
||||
TRIGGER_MIN_DOLLAR_VOL: '8000'
|
||||
TRIGGER_PRICE_MAX_AGE: '12'
|
||||
TRIGGER_RVOL_MIN: '2.0'
|
||||
TZ: America/New_York
|
||||
WATCHER_ENABLED: '1'
|
||||
WATCHER_INTERVAL_SECONDS: '15'
|
||||
WATCHER_PRICE_MAX: '20'
|
||||
WATCHER_PRICE_MIN: '2'
|
||||
YOUTUBE_API_KEY: <FILL-ME-YOUTUBE-KEY>
|
||||
ADMIN_PORT: '8767'
|
||||
ADMIN_TOKEN: <FILL-ME-ADMIN-TOKEN>
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/data:/app/data
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- 8767:8767
|
||||
scheduler:
|
||||
image: docker.io/fez5587/momentum-app:latest
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- "echo \"[scheduler] up; fires 20:00 ET Mon-Fri\"\nwhile true; do\n if [ \"$$(date +%H:%M)\" = \"20:00\" ] && [ \"$$(date +%u)\" -le 5 ]; then\n python -m research.premarket_eval || echo \"[scheduler] eval FAILED\"\n python -m research.ingestion.market_context || echo \"[scheduler] market_context FAILED\"\n python -m research.eod_reconcile || echo \"[scheduler] reconcile FAILED\"\n python eod_replay.py || echo \"[scheduler] eod_replay FAILED\"\n python -m research.labeler build || echo \"[scheduler] labeler build FAILED\"\n python -m research.labeler build-tod1 || echo \"[scheduler] labeler build-tod1 FAILED\"\n python nightly_tune.py || echo \"[scheduler] nightly_tune FAILED\"\n python -m research.regime build-features || echo \"[scheduler] regime build FAILED\"\n python -m research.regime score --model threshold_v1 || echo \"[scheduler] regime threshold FAILED\"\n python -m research.regime score --model meta_v1 || echo \"[scheduler] regime meta FAILED\"\n python -m research.experiments run || echo \"[scheduler] experiments run FAILED\"\n python -m research.experiments adjudicate || echo \"[scheduler] experiments adjudicate FAILED\"\n python -m research.weekly_tallies || echo \"[scheduler] weekly_tallies FAILED\"\n sleep 120\n fi\n sleep 20\ndone\n"
|
||||
environment:
|
||||
ALPACA_API_KEY: <FILL-ME-ALPACA-KEY>
|
||||
ALPACA_BACKFILL_FEED: sip
|
||||
ALPACA_BASE_URL: https://paper-api.alpaca.markets/v2
|
||||
ALPACA_DATA_FEED: sip
|
||||
ALPACA_PAPER_SYNC_ENABLED: '1'
|
||||
ALPACA_SECRET_KEY: <FILL-ME-ALPACA-SECRET>
|
||||
DATABASE_URL: postgresql://momentum:<FILL-ME-PG-PASSWORD>@postgres:5432/momentum
|
||||
DEADMAN_DAILY_PING: '1'
|
||||
DEADMAN_STALE_MIN: '5'
|
||||
DISCOVER_INTERVAL_SECONDS: '120'
|
||||
FMP_API_KEY: <FILL-ME-FMP-KEY>
|
||||
INGEST_STALE_BLOCK: '0'
|
||||
INGEST_STALE_SECONDS: '180'
|
||||
LIVE_BARS_INTERVAL_SECONDS: '20'
|
||||
LIVE_BARS_REFRESH_MINUTES: '20'
|
||||
NEWS_CATALYST_SCORE_ENABLED: '1'
|
||||
NEWS_DILUTION_VETO_ENABLED: '1'
|
||||
NEWS_ENRICH_BATCH_LIMIT: '50'
|
||||
NEWS_ENRICH_ENABLED: '1'
|
||||
NEWS_ENRICH_INTERVAL_SECONDS: '120'
|
||||
NEWS_ENRICH_LOOKBACK_HOURS: '12'
|
||||
NEWS_FEEDS: prnewswire=https://www.prnewswire.com/rss/news-releases-list.rss,businesswire=https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRWQ==,globenewswire=https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies
|
||||
NEWS_LOOKBACK_HOURS: '8'
|
||||
NEWS_RANK_BOOST: '2.0'
|
||||
OLLAMA_ENABLED: '1'
|
||||
OLLAMA_HOST: http://192.168.1.5:30068
|
||||
OLLAMA_MODEL: qwen2.5:7b-instruct
|
||||
OLLAMA_TEMPERATURE: '0.1'
|
||||
OLLAMA_TIMEOUT_SECONDS: '60'
|
||||
ORB_BARS: '5'
|
||||
PG_SKIP_SCHEMA: '1'
|
||||
STRATEGY_SETUPS: opening_range_break
|
||||
TELEGRAM_BOT_TOKEN: <FILL-ME-TELEGRAM-TOKEN>
|
||||
TELEGRAM_CHAT_ID: <FILL-ME-TELEGRAM-CHAT-ID>
|
||||
TRADING_AUTO_APPROVE: '1'
|
||||
TRADING_BACKOUT_COOLDOWN_SECONDS: '180'
|
||||
TRADING_BLOCK_REENTRY_AFTER_EXIT: '1'
|
||||
TRADING_ENTRY_CONFIRM_BY_MINUTE: '0'
|
||||
TRADING_ENTRY_FILL_MODEL: marketable
|
||||
TRADING_ENTRY_INVALIDATE_PCT: '0.015'
|
||||
TRADING_ENTRY_ORDER_TYPE: limit
|
||||
TRADING_ENTRY_TIMEOUT_BARS: '2'
|
||||
TRADING_EOD_FLATTEN_ENABLED: '1'
|
||||
TRADING_EOD_FLATTEN_TIME: '15:55'
|
||||
TRADING_EXECUTION_ENABLED: '1'
|
||||
TRADING_EXIT_BREAKEVEN_R: '1.0'
|
||||
TRADING_EXIT_PROFIT_TIERS: 8:3,15:9,25:18,40:30
|
||||
TRADING_EXIT_TRAIL_AFTER_R: '1.0'
|
||||
TRADING_EXIT_TRAIL_MODE: prior_low
|
||||
TRADING_EXIT_TRAIL_R_STEP: '0.25'
|
||||
TRADING_LIQUIDITY_MAX_VOLUME_PCT: '0.025'
|
||||
TRADING_MAX_BACKOUTS_PER_SYMBOL: '3'
|
||||
TRADING_MAX_CONCURRENT_POSITIONS: '10'
|
||||
TRADING_MAX_DAILY_LOSS_PCT: '0.05'
|
||||
TRADING_MAX_FRESH_ENTRIES_PER_DAY: '10'
|
||||
TRADING_MAX_GROSS_NOTIONAL_PCT: '0.60'
|
||||
TRADING_MAX_ORDERS_PER_TICK: '1'
|
||||
TRADING_MAX_POSITION_PCT: '0.16'
|
||||
TRADING_MAX_RISK_DOLLARS: '500'
|
||||
TRADING_MIN_DAY_VOLUME: '300000'
|
||||
TRADING_MIN_ENTRY_GAP_PCT: '0.10'
|
||||
TRADING_MIN_QUALITY_SCORE: '0.60'
|
||||
TRADING_PRICE_MAX: '20.0'
|
||||
TRADING_PRICE_MIN: '2.0'
|
||||
TRADING_REENTRY_MIN_LOSS_PCT: '0.01'
|
||||
TRADING_REGIME_GATE_ENABLED: '1'
|
||||
TRADING_REGIME_GATE_P_MIN: '0.5'
|
||||
TRADING_REQUIRE_ABOVE_VWAP: '1'
|
||||
TRADING_REWARD_MULTIPLE: '10.0'
|
||||
TRADING_RISK_PER_TRADE_PCT: '0.01'
|
||||
TRADING_SESSION_ONLY: '1'
|
||||
TRADING_TRIGGER_SLIP_PCT: '0.004'
|
||||
TRADING_UNIFIED_ENTRY: '1'
|
||||
TRIGGER_ARM_INTERVAL_SECONDS: '20'
|
||||
TRIGGER_ENABLED: '1'
|
||||
TRIGGER_GAP_MAX: '35'
|
||||
TRIGGER_GAP_MIN: '10.0'
|
||||
TRIGGER_GAP_SQUEEZE_MIN: '150'
|
||||
TRIGGER_GAP_SQUEEZE_RVOL: '5.0'
|
||||
TRIGGER_INTERVAL_SECONDS: '4'
|
||||
TRIGGER_MAX_ARMED: '10'
|
||||
TRIGGER_MIN_DOLLAR_VOL: '8000'
|
||||
TRIGGER_PRICE_MAX_AGE: '12'
|
||||
TRIGGER_RVOL_MIN: '2.0'
|
||||
TZ: America/New_York
|
||||
WATCHER_ENABLED: '1'
|
||||
WATCHER_INTERVAL_SECONDS: '15'
|
||||
WATCHER_PRICE_MAX: '20'
|
||||
WATCHER_PRICE_MIN: '2'
|
||||
YOUTUBE_API_KEY: <FILL-ME-YOUTUBE-KEY>
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/data:/app/data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
db-backup:
|
||||
image: postgres:18
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
DATABASE_URL: postgresql://momentum:<FILL-ME-PG-PASSWORD>@postgres:5432/momentum
|
||||
BACKUP_DIR: /backups
|
||||
BACKUP_KEEP_DAYS: '14'
|
||||
TZ: America/New_York
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/data/backups:/backups
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- "set -o pipefail\necho \"[backup] up; fires 20:10 ET Mon-Fri\"\nwhile true; do\n if [ \"$$(date +%H:%M)\" = \"20:10\" ] && [ \"$$(date +%u)\" -le 5 ]; then\n out=\"$$BACKUP_DIR/momentum-$$(date +%F).sql.gz\"\n mkdir -p \"$$BACKUP_DIR\" && pg_dump \"$$DATABASE_URL\" | gzip > \"$$out.tmp\" && gunzip -t \"$$out.tmp\" && mv \"$$out.tmp\" \"$$out\" && echo \"[backup] OK\" || echo \"[backup] FAILED\"\n sleep 120\n fi\n sleep 20\ndone\n"
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
# TRADIER PARALLEL STACK — template (fill <FILL-ME-*>, save as
|
||||
# truenas-compose.tradier.local.yaml, never commit the filled copy).
|
||||
#
|
||||
# Runs the SAME image/strategy as the paper stack but executes through the
|
||||
# Tradier mirror (BROKER=tradier, sandbox by default). Fully self-contained:
|
||||
# own postgres (host 5434), own dashboard (host 8768). Market data stays on
|
||||
# Alpaca (keys still required). Telegram + deadman are OFF here so the main
|
||||
# stack's alerts stay unambiguous.
|
||||
#
|
||||
# Shares the main stack's data dir READ-ONLY: the loop inherits the nightly
|
||||
# learned_params.json and overrides.env without being able to clobber them.
|
||||
#
|
||||
# Fresh-DB bootstrap (same playbook as live staging), run on cube:
|
||||
# ./bootstrap_live_db.sh <nightly-backup.sql.gz> <this-stack-postgres-container>
|
||||
# or skip it and let the loop start from an empty DB (research baselines absent,
|
||||
# config knobs at code defaults — fine for a first shakeout).
|
||||
#
|
||||
# Requires an image built from >= 1cefb51 (tradier_broker in image).
|
||||
#
|
||||
# Sandbox caveat: Tradier sandbox fills are simulated and can be slow/absent —
|
||||
# an order parked in 'pending_new' is usually the sandbox, not the code.
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18
|
||||
environment:
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
POSTGRES_DB: momentum
|
||||
POSTGRES_PASSWORD: <FILL-ME-PG-PASSWORD>
|
||||
POSTGRES_USER: momentum
|
||||
TZ: America/New_York
|
||||
healthcheck:
|
||||
interval: 10s
|
||||
retries: 5
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- pg_isready -U momentum -d momentum
|
||||
timeout: 5s
|
||||
ports:
|
||||
- 5434:5432
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /mnt/Volume/Storage/Apps/momentum/pgdata-tradier:/var/lib/postgresql/data
|
||||
|
||||
loop:
|
||||
image: docker.io/fez5587/momentum-app:latest
|
||||
command:
|
||||
- python
|
||||
- -u
|
||||
- run_live_paper.py
|
||||
- --no-dashboard
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment: &tradier-env
|
||||
BROKER: tradier
|
||||
TRADIER_ACCESS_TOKEN: <FILL-ME-TRADIER-TOKEN>
|
||||
TRADIER_ACCOUNT_ID: <FILL-ME-TRADIER-ACCOUNT>
|
||||
TRADIER_ENV: sandbox
|
||||
ALPACA_API_KEY: <FILL-ME-ALPACA-KEY>
|
||||
ALPACA_SECRET_KEY: <FILL-ME-ALPACA-SECRET>
|
||||
ALPACA_BACKFILL_FEED: sip
|
||||
# fresh DB has no config_overrides, so compose must set the paid SIP
|
||||
# feed explicitly or this stack silently runs thin IEX volume
|
||||
ALPACA_DATA_FEED: sip
|
||||
ALPACA_BASE_URL: https://paper-api.alpaca.markets
|
||||
ALPACA_PAPER_SYNC_ENABLED: '1'
|
||||
DATABASE_URL: postgresql://momentum:<FILL-ME-PG-PASSWORD>@postgres:5432/momentum
|
||||
# alerts stay with the MAIN stack — no double Telegram / deadman pings
|
||||
TELEGRAM_BOT_TOKEN: ''
|
||||
TELEGRAM_CHAT_ID: ''
|
||||
DEADMAN_DAILY_PING: '0'
|
||||
DEADMAN_STALE_MIN: '5'
|
||||
INGEST_STALE_BLOCK: '0'
|
||||
INGEST_STALE_SECONDS: '180'
|
||||
ORB_BARS: '5'
|
||||
PG_SKIP_SCHEMA: '0'
|
||||
QUOTE_SNAPSHOT_INTERVAL_SECONDS: '8'
|
||||
STRATEGY_SETUPS: opening_range_break
|
||||
DEPTH_RECORD_ENABLED: '0'
|
||||
TZ: America/New_York
|
||||
healthcheck:
|
||||
interval: 60s
|
||||
retries: 5
|
||||
start_period: 180s
|
||||
test:
|
||||
- CMD
|
||||
- python
|
||||
- -c
|
||||
- import os; os.environ['PG_SKIP_SCHEMA']='1'; import datetime,sys; from research.multi_schema import open_research_db; r=open_research_db('market').execute('SELECT max(timestamp) FROM events').fetchall()[0][0]; sys.exit(0 if r and (datetime.datetime.now()-r).total_seconds()<300 else 1)
|
||||
timeout: 20s
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /mnt/Volume/Storage/Apps/momentum/data:/app/data:ro
|
||||
|
||||
dashboard:
|
||||
image: docker.io/fez5587/momentum-app:latest
|
||||
command:
|
||||
- python
|
||||
- -u
|
||||
- dashboard_api.py
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
<<: *tradier-env
|
||||
COMMAND_BUS_ENABLED: '1'
|
||||
DASHBOARD_HOST: 0.0.0.0
|
||||
DASHBOARD_PORT: '8765'
|
||||
DASHBOARD_TOKEN: <FILL-ME-DASHBOARD-TOKEN>
|
||||
ports:
|
||||
- 8768:8765
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /mnt/Volume/Storage/Apps/momentum/data:/app/data:ro
|
||||
@@ -0,0 +1,285 @@
|
||||
# Momentum stack for TrueNAS SCALE (Apps -> Discover Apps -> Custom App -> YAML).
|
||||
#
|
||||
# PULL-ONLY: TrueNAS custom apps cannot `build:` — the image comes from GHCR
|
||||
# (published by .github/workflows/docker.yml on every master push). Make the
|
||||
# ghcr.io/fez5587/momentum-app package PUBLIC once (GitHub -> Packages ->
|
||||
# momentum-app -> Package settings -> Change visibility), or add registry
|
||||
# credentials in TrueNAS (Apps -> Settings -> Manage Container Images).
|
||||
#
|
||||
# BEFORE DEPLOYING:
|
||||
# 1. Create a dataset for the stack, e.g. /mnt/<pool>/apps/momentum
|
||||
# with subdirs: data/ pgdata/
|
||||
# 2. Replace /mnt/tank/apps/momentum below with your dataset path.
|
||||
# 3. Fill every <FILL-ME> in the environment block (these are the secrets
|
||||
# you wanted to manage from a protected space — TrueNAS stores the YAML
|
||||
# server-side, or mount an env file from a restricted dataset instead).
|
||||
#
|
||||
# TZ is America/New_York: the loop's market clock thinks in ET.
|
||||
|
||||
x-app: &app
|
||||
image: docker.io/fez5587/momentum-app:latest # also published to ghcr.io/fez5587/momentum-app
|
||||
environment: &app-env
|
||||
ALPACA_API_KEY: '<FILL-ME>'
|
||||
ALPACA_BACKFILL_FEED: 'sip'
|
||||
ALPACA_BASE_URL: 'https://paper-api.alpaca.markets/v2'
|
||||
ALPACA_DATA_FEED: 'sip'
|
||||
ALPACA_PAPER_SYNC_ENABLED: '1'
|
||||
ALPACA_SECRET_KEY: '<FILL-ME>'
|
||||
DATABASE_URL: 'postgresql://momentum:<FILL-ME-PG-PASSWORD>@postgres:5432/momentum'
|
||||
DEADMAN_DAILY_PING: '1'
|
||||
DEADMAN_STALE_MIN: '5'
|
||||
DISCOVER_INTERVAL_SECONDS: '120'
|
||||
FMP_API_KEY: '<FILL-ME>'
|
||||
INGEST_STALE_BLOCK: '0'
|
||||
INGEST_STALE_SECONDS: '180'
|
||||
LIVE_BARS_INTERVAL_SECONDS: '5'
|
||||
LIVE_BARS_REFRESH_MINUTES: '20'
|
||||
NEWS_CATALYST_SCORE_ENABLED: '1'
|
||||
NEWS_DILUTION_VETO_ENABLED: '1'
|
||||
NEWS_ENRICH_BATCH_LIMIT: '50'
|
||||
NEWS_ENRICH_ENABLED: '1'
|
||||
NEWS_ENRICH_INTERVAL_SECONDS: '120'
|
||||
NEWS_ENRICH_LOOKBACK_HOURS: '12'
|
||||
NEWS_FEEDS: 'prnewswire=https://www.prnewswire.com/rss/news-releases-list.rss,businesswire=https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRWQ==,globenewswire=https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies'
|
||||
NEWS_LOOKBACK_HOURS: '8'
|
||||
NEWS_RANK_BOOST: '2.0'
|
||||
OLLAMA_ENABLED: '1'
|
||||
OLLAMA_HOST: 'http://192.168.1.5:30068'
|
||||
OLLAMA_MODEL: 'qwen2.5:7b-instruct'
|
||||
OLLAMA_TEMPERATURE: '0.1'
|
||||
OLLAMA_TIMEOUT_SECONDS: '60'
|
||||
ORB_BARS: '5'
|
||||
PG_SKIP_SCHEMA: '0'
|
||||
STRATEGY_SETUPS: 'opening_range_break'
|
||||
TELEGRAM_BOT_TOKEN: '<FILL-ME>'
|
||||
TELEGRAM_CHAT_ID: "'2082795871'"
|
||||
TRADING_AUTO_APPROVE: '1'
|
||||
TRADING_BACKOUT_COOLDOWN_SECONDS: '180'
|
||||
TRADING_BLOCK_REENTRY_AFTER_EXIT: '1'
|
||||
TRADING_ENTRY_CONFIRM_BY_MINUTE: '0'
|
||||
TRADING_ENTRY_FILL_MODEL: 'resting'
|
||||
TRADING_ENTRY_INVALIDATE_PCT: '0.015'
|
||||
TRADING_ENTRY_ORDER_TYPE: 'limit'
|
||||
TRADING_ENTRY_TIMEOUT_BARS: '2'
|
||||
TRADING_EOD_FLATTEN_ENABLED: '1'
|
||||
TRADING_EOD_FLATTEN_TIME: '15:55'
|
||||
TRADING_EXECUTION_ENABLED: '1'
|
||||
TRADING_EXIT_BREAKEVEN_R: '1.0'
|
||||
TRADING_EXIT_PROFIT_TIERS: '8:3,15:9,25:18,40:30'
|
||||
# Regime gate (plan 011) LIVE: block fresh entries on sessions the walk-forward
|
||||
# metalabel scores unfavorable (sit out bad regimes). Only the LOOP acts on it;
|
||||
# fail-open (unfit model / error never blocks). Needs regime_features populated
|
||||
# (the scheduler builds them nightly). Set 0 to disable.
|
||||
TRADING_REGIME_GATE_ENABLED: '1'
|
||||
# Favorable threshold: block a session when P(profit) < this. 0.5 = code
|
||||
# default (matches the backtest). Lower (e.g. 0.4) trades MORE days; raise to
|
||||
# trade fewer, higher-confidence days. Optional — omit to use the 0.5 default.
|
||||
TRADING_REGIME_GATE_P_MIN: '0.5'
|
||||
TRADING_EXIT_TRAIL_AFTER_R: '1.0'
|
||||
TRADING_EXIT_TRAIL_MODE: 'prior_low'
|
||||
TRADING_EXIT_MANAGE_INTERVAL_SECONDS: '4' # 12->4s: react to fast small-cap round-trips (was giving back gains)
|
||||
TRADING_EXIT_THREAD_ENABLED: '1'
|
||||
DISCOVER_THREAD_ENABLED: '1'
|
||||
TRADING_EXIT_THREAD_INTERVAL_SECONDS: '1.0'
|
||||
TRADING_EXIT_TRAIL_R_STEP: '0.25'
|
||||
TRADING_LIQUIDITY_MAX_VOLUME_PCT: '0.025'
|
||||
TRADING_MAX_BACKOUTS_PER_SYMBOL: '3'
|
||||
TRADING_MAX_CONCURRENT_POSITIONS: '10'
|
||||
TRADING_MAX_DAILY_LOSS_PCT: '0.05'
|
||||
TRADING_MAX_FRESH_ENTRIES_PER_DAY: '10'
|
||||
TRADING_MAX_GROSS_NOTIONAL_PCT: '0.60'
|
||||
TRADING_MAX_ORDERS_PER_TICK: '1'
|
||||
TRADING_MAX_POSITION_PCT: '0.16'
|
||||
TRADING_MAX_RISK_DOLLARS: '500'
|
||||
TRADING_MIN_DAY_VOLUME: '300000'
|
||||
TRADING_MIN_ENTRY_GAP_PCT: '0.10'
|
||||
TRADING_MIN_QUALITY_SCORE: '0.60'
|
||||
TRADING_PRICE_MAX: '20.0'
|
||||
TRADING_PRICE_MIN: '2.0'
|
||||
TRADING_REENTRY_MIN_LOSS_PCT: '0.01'
|
||||
TRADING_REQUIRE_ABOVE_VWAP: '1'
|
||||
TRADING_REWARD_MULTIPLE: '10.0'
|
||||
TRADING_RISK_PER_TRADE_PCT: '0.01'
|
||||
TRADING_SESSION_ONLY: '1'
|
||||
TRADING_TRIGGER_SLIP_PCT: '0.004'
|
||||
TRADING_UNIFIED_ENTRY: '1'
|
||||
TRIGGER_ARM_INTERVAL_SECONDS: '8'
|
||||
TRIGGER_ENABLED: '1'
|
||||
TRIGGER_GAP_MAX: '35'
|
||||
TRIGGER_GAP_MIN: '10.0'
|
||||
TRIGGER_GAP_SQUEEZE_MIN: '150'
|
||||
TRIGGER_GAP_SQUEEZE_RVOL: '5.0'
|
||||
TRIGGER_INTERVAL_SECONDS: '2'
|
||||
TRIGGER_MAX_ARMED: '10'
|
||||
TRIGGER_MIN_DOLLAR_VOL: '8000'
|
||||
TRIGGER_PRICE_MAX_AGE: '12'
|
||||
TRIGGER_RVOL_MIN: '2.0'
|
||||
TZ: 'America/New_York'
|
||||
WATCHER_ENABLED: '1'
|
||||
WATCHER_INTERVAL_SECONDS: '8'
|
||||
WATCHER_PRICE_MAX: '20'
|
||||
WATCHER_PRICE_MIN: '2'
|
||||
YOUTUBE_API_KEY: '<FILL-ME>'
|
||||
# comma-separated playlist URLs (or bare ids); synced into the playlists
|
||||
# table at transcripts-service boot — edit here to add/track more playlists
|
||||
YT_PLAYLIST_URLS: ''
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/data:/app/data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18
|
||||
environment:
|
||||
POSTGRES_USER: momentum
|
||||
POSTGRES_PASSWORD: <FILL-ME-PG-PASSWORD>
|
||||
POSTGRES_DB: momentum
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/pgdata:/var/lib/postgresql/data
|
||||
ports:
|
||||
# published on 5433 so it can coexist with the EXISTING postgres (5432)
|
||||
# during migration — pgAdmin/pg_restore reach the new instance on 5433.
|
||||
# In-stack services still use postgres:5432 (the container port).
|
||||
- "5433:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U momentum -d momentum"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
loop:
|
||||
<<: *app
|
||||
command: ["python", "-u", "run_live_paper.py", "--no-dashboard"]
|
||||
healthcheck:
|
||||
# PG_SKIP_SCHEMA=1: a healthcheck must NOT re-apply schema DDL — the 60s
|
||||
# CREATE-IF-NOT-EXISTS lock burst cancelled minute_bars autovacuum forever
|
||||
# (2026-07-09 post-migration: stats rotted, dashboard queries crawled)
|
||||
test: ["CMD", "python", "-c", "import os; os.environ['PG_SKIP_SCHEMA']='1'; import datetime,sys; from research.multi_schema import open_research_db; r=open_research_db('market').execute('SELECT max(timestamp) FROM events').fetchall()[0][0]; sys.exit(0 if r and (datetime.datetime.now()-r).total_seconds()<300 else 1)"]
|
||||
interval: 60s
|
||||
timeout: 20s
|
||||
retries: 5
|
||||
start_period: 180s
|
||||
|
||||
dashboard:
|
||||
<<: *app
|
||||
command: ["python", "-u", "dashboard_api.py"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
DASHBOARD_HOST: "0.0.0.0"
|
||||
DASHBOARD_PORT: "8765"
|
||||
# read-only viewer: never re-apply the 473-line schema (the SSE change
|
||||
# detector holds one long-lived connection) — mirrors the loop healthcheck.
|
||||
PG_SKIP_SCHEMA: '1'
|
||||
# Shared secret required on every POST (approve/reject/exit + the command
|
||||
# bus). Empty = commands disabled + POSTs unauthenticated; fill to arm the
|
||||
# kill switch. Reference by name only — never inline the secret value.
|
||||
DASHBOARD_TOKEN: <FILL-ME-DASHBOARD-TOKEN>
|
||||
# Gate the operator command bus on (also needs DASHBOARD_TOKEN set).
|
||||
COMMAND_BUS_ENABLED: '1'
|
||||
|
||||
qabot:
|
||||
<<: *app
|
||||
command: ["python", "-u", "telegram_bot.py"]
|
||||
|
||||
transcripts:
|
||||
<<: *app
|
||||
command: ["python", "-u", "-m", "youtube_claims.run", "run"]
|
||||
|
||||
admin:
|
||||
<<: *app
|
||||
command: ["python", "-u", "admin_api.py"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
ADMIN_PORT: "8767"
|
||||
ADMIN_TOKEN: <FILL-ME-ADMIN-TOKEN>
|
||||
ports:
|
||||
- "8767:8767"
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/data:/app/data
|
||||
- /var/run/docker.sock:/var/run/docker.sock # service status + restarts
|
||||
|
||||
scheduler:
|
||||
<<: *app
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
echo "[scheduler] up; fires 20:00 ET Mon-Fri"
|
||||
while true; do
|
||||
if [ "$$(date +%H:%M)" = "20:00" ] && [ "$$(date +%u)" -le 5 ]; then
|
||||
python -m research.premarket_eval || echo "[scheduler] eval FAILED"
|
||||
# Premarket-structure labeler (Phase 1, observe-only): distill each
|
||||
# gapper's premarket tape for entry-conversion analysis (H1-H3
|
||||
# pre-registered in the module docstring). Gates nothing.
|
||||
python -m research.premarket_structure build || echo "[scheduler] premarket_structure FAILED"
|
||||
# SPY/QQQ ingest -> forward source for the labeler's market-context features (plan 009)
|
||||
python -m research.ingestion.market_context || echo "[scheduler] market_context ingest FAILED"
|
||||
python -m research.eod_reconcile || echo "[scheduler] reconcile FAILED"
|
||||
# Flywheel: grow the whole-market dataset first, then fit the tuner on
|
||||
# it. 20:00 ET is after close, so replay of "today" is complete data —
|
||||
# NOT look-ahead (replay is a labeling job, not a live gate).
|
||||
python eod_replay.py || echo "[scheduler] eod_replay FAILED"
|
||||
python nightly_tune.py || echo "[scheduler] nightly_tune FAILED"
|
||||
# Regime gate (plan 011, ALL-SHADOW): a session-level metalabel
|
||||
# P(our ORB profits today). build-features labels each session from its
|
||||
# OWN realized expR; score writes the WALK-FORWARD favorable flag the
|
||||
# experiments' regime_gate intervention reads. Must run BEFORE
|
||||
# experiments run. Does NOT gate live trading (that is owner-gated).
|
||||
python -m research.regime build-features || echo "[scheduler] regime build-features FAILED"
|
||||
python -m research.regime score --model threshold_v1 || echo "[scheduler] regime score threshold FAILED"
|
||||
python -m research.regime score --model meta_v1 || echo "[scheduler] regime score meta FAILED"
|
||||
# Experiment OS (plan 006, ALL-SHADOW): replay running experiments over
|
||||
# newly-logged fires, then adjudicate against pre-registered bars.
|
||||
python -m research.experiments run || echo "[scheduler] experiments run FAILED"
|
||||
python -m research.experiments adjudicate || echo "[scheduler] experiments adjudicate FAILED"
|
||||
# Weekly execution tallies (read-only): midday-entry net + market_exit
|
||||
# post-run. Script self-gates to Friday; other nights it no-ops.
|
||||
python -m research.weekly_tallies || echo "[scheduler] weekly_tallies FAILED"
|
||||
sleep 120
|
||||
fi
|
||||
sleep 20
|
||||
done
|
||||
|
||||
db-backup:
|
||||
# Nightly pg_dump at 20:10 ET Mon-Fri, on the postgres:18 image so the
|
||||
# client always matches the server major (the app image has NO pg_dump).
|
||||
# This host is pull-only (no repo checkout), so the backup logic is INLINE
|
||||
# — keep it in sync with deploy/docker/backup_db.sh. Restore:
|
||||
# gunzip -c momentum-<date>.sql.gz | psql "$DATABASE_URL"
|
||||
image: postgres:18
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
DATABASE_URL: 'postgresql://momentum:<FILL-ME-PG-PASSWORD>@postgres:5432/momentum'
|
||||
BACKUP_DIR: '/backups'
|
||||
BACKUP_KEEP_DAYS: '14'
|
||||
TZ: 'America/New_York' # postgres image defaults to UTC; the 20:10 gate is ET
|
||||
volumes:
|
||||
- /mnt/volume/Apps/momentum/data/backups:/backups
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -o pipefail
|
||||
echo "[backup] up; fires 20:10 ET Mon-Fri"
|
||||
while true; do
|
||||
if [ "$$(date +%H:%M)" = "20:10" ] && [ "$$(date +%u)" -le 5 ]; then
|
||||
out="$$BACKUP_DIR/momentum-$$(date +%F).sql.gz"
|
||||
mkdir -p "$$BACKUP_DIR" \
|
||||
&& pg_dump "$$DATABASE_URL" | gzip > "$$out.tmp" \
|
||||
&& gunzip -t "$$out.tmp" && mv "$$out.tmp" "$$out" \
|
||||
&& find "$$BACKUP_DIR" -name 'momentum-*.sql.gz*' -mtime +"$$BACKUP_KEEP_DAYS" -delete \
|
||||
&& date > "$$BACKUP_DIR/last_backup_ok" \
|
||||
&& echo "[backup] OK $$out" \
|
||||
|| echo "[backup] FAILED"
|
||||
sleep 120
|
||||
fi
|
||||
sleep 20
|
||||
done
|
||||
@@ -0,0 +1,182 @@
|
||||
# Kubernetes deployment
|
||||
|
||||
A container image and Helm chart that run the whole Momentum System — including
|
||||
its **PostgreSQL datastore** — inside a single Kubernetes namespace.
|
||||
|
||||
## What gets deployed
|
||||
|
||||
| Workload | Kind | Notes |
|
||||
| --- | --- | --- |
|
||||
| PostgreSQL | StatefulSet + PVC | Single source of truth. Schema auto-creates on the app's first connection from `storage/pg_schema.sql`. |
|
||||
| App loop + dashboard | Deployment (`replicas: 1`, `Recreate`) | `run_live_paper.py` with the **embedded** dashboard on :8010. Stateful singleton — never scale up. |
|
||||
| journal / eod-replay / nightly-tune | CronJobs | US/Eastern (`spec.timeZone`), weekdays. Share the app's `data/` PVC. |
|
||||
| Grafana | Deployment + Service | Optional (`grafana.enabled`). Datasource auto-points at the in-cluster Postgres. |
|
||||
|
||||
The app loop and the CronJobs run the **same image** (built from the repo
|
||||
`Dockerfile`), just with different commands.
|
||||
|
||||
## Local development with Docker Compose
|
||||
|
||||
For a single-machine stack (same components, no Kubernetes) use the root
|
||||
`docker-compose.yml`. It builds the app from the `Dockerfile`, brings up
|
||||
PostgreSQL with a persistent volume, and runs the loop with its embedded
|
||||
dashboard. Config comes from a local `.env` (optional — without `ALPACA_*` keys
|
||||
the loop runs in dry mode; the dashboard and DB still work). `DATABASE_URL` is
|
||||
overridden to the in-network `postgres` service, so the stack is self-contained.
|
||||
|
||||
```bash
|
||||
cp .env.example .env # optional: add ALPACA_* keys for live data
|
||||
|
||||
docker compose up -d --build # postgres + app loop + dashboard (:8010)
|
||||
docker compose logs -f app # watch the loop
|
||||
open http://localhost:8010 # dashboard
|
||||
|
||||
docker compose --profile monitoring up -d # also start Grafana (:3000)
|
||||
```
|
||||
|
||||
The scheduled jobs sit under the `jobs` profile so they don't auto-start; run
|
||||
them on demand (or from host cron / a scheduler on the ET times in `.env`):
|
||||
|
||||
```bash
|
||||
docker compose run --rm nightly-tune
|
||||
docker compose run --rm eod-replay
|
||||
docker compose run --rm journal
|
||||
```
|
||||
|
||||
Postgres credentials default to `momentum`/`momentum`/`momentum`; override with
|
||||
`POSTGRES_USER` / `POSTGRES_PASSWORD` / `POSTGRES_DB` in `.env`. Named volumes
|
||||
`pg-data`, `app-data`, and `grafana-data` persist across restarts (the jobs share
|
||||
`app-data` with the app so `nightly_tune.py`'s `learned_params.json` reaches the
|
||||
loop). The Compose Grafana uses its own datasource override at
|
||||
`deploy/compose/grafana/datasources/momentum.yaml` (pointed at the `postgres`
|
||||
service); the legacy `grafana/docker-compose.yml` is unrelated and left for
|
||||
Grafana-only use against an external DB.
|
||||
|
||||
## Build & publish the image
|
||||
|
||||
Pushes to the working branch / `main` / `v*` tags trigger
|
||||
`.github/workflows/docker-publish.yml`, which builds and pushes to
|
||||
`ghcr.io/fez5587/momentum-system`. Point the chart at a pushed tag via
|
||||
`image.tag` (a commit SHA, or `latest` on the default branch).
|
||||
|
||||
Local build:
|
||||
|
||||
```bash
|
||||
docker build -t momentum:dev .
|
||||
```
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
helm install momentum ./deploy/helm/momentum \
|
||||
-n momentum --create-namespace \
|
||||
--set image.tag=latest \
|
||||
--set secrets.alpacaApiKey=YOUR_KEY \
|
||||
--set secrets.alpacaSecretKey=YOUR_SECRET
|
||||
```
|
||||
|
||||
Without Alpaca keys the loop runs in **dry mode** (no live data/orders); the
|
||||
dashboard and database still work.
|
||||
|
||||
### Secrets
|
||||
|
||||
The chart builds `DATABASE_URL` from the in-cluster Postgres service and a
|
||||
password that is generated on first install (or set `postgres.password` to pin
|
||||
it). Broker keys come from `secrets.*`.
|
||||
|
||||
For real keys, prefer an existing Secret over `values.yaml`:
|
||||
|
||||
```bash
|
||||
kubectl -n momentum create secret generic momentum-broker \
|
||||
--from-literal=DATABASE_URL=... \
|
||||
--from-literal=ALPACA_API_KEY=... --from-literal=ALPACA_SECRET_KEY=... \
|
||||
--from-literal=APCA_API_KEY_ID=... --from-literal=APCA_API_SECRET_KEY=...
|
||||
helm install momentum ./deploy/helm/momentum -n momentum \
|
||||
--set secrets.existingSecret=momentum-broker
|
||||
```
|
||||
|
||||
When `secrets.existingSecret` is set the chart renders no Secret of its own and
|
||||
expects that one to carry `DATABASE_URL`, the `ALPACA_*`/`APCA_*` keys, and
|
||||
`POSTGRES_PASSWORD`.
|
||||
|
||||
## Reach the dashboard
|
||||
|
||||
```bash
|
||||
kubectl -n momentum port-forward svc/momentum-app 8010:8010
|
||||
open http://localhost:8010
|
||||
```
|
||||
|
||||
Set `ingress.enabled=true` (with `ingress.host`/`className`) to expose it
|
||||
instead.
|
||||
|
||||
## Verify
|
||||
|
||||
```bash
|
||||
helm lint deploy/helm/momentum
|
||||
helm template momentum deploy/helm/momentum | less
|
||||
|
||||
# After install:
|
||||
kubectl -n momentum get pods
|
||||
kubectl -n momentum exec sts/momentum-postgres -- psql -U momentum -d momentum -c '\dt'
|
||||
kubectl -n momentum create job --from=cronjob/momentum-nightly-tune tune-test
|
||||
```
|
||||
|
||||
## Notes & constraints
|
||||
|
||||
- **Singleton loop.** The trading loop holds in-memory state and one broker
|
||||
session; `replicas` must stay `1` and the strategy is `Recreate`.
|
||||
- **Shared `data/` PVC is ReadWriteOnce.** The CronJob pods mount the same PVC
|
||||
as the app (so `nightly_tune.py` can write `learned_params.json` for the loop
|
||||
to read), which schedules them onto the app pod's node. For multi-node
|
||||
clusters, use an RWX StorageClass or split the volumes.
|
||||
- **CronJob `spec.timeZone`** requires Kubernetes ≥ 1.27.
|
||||
- The legacy `grafana/docker-compose.yml` is left in place for local use; the
|
||||
chart provisions its own Grafana against the in-cluster Postgres.
|
||||
|
||||
## Future hardening (roadmap)
|
||||
|
||||
The package deploys and runs, but the items below separate "it comes up" from
|
||||
"runs reliably for a trading system." They are **not implemented yet** — captured
|
||||
here so they can be picked up later. Each notes what, why, and the rough approach.
|
||||
|
||||
### Reliability
|
||||
|
||||
- **Loop-heartbeat liveness** *(needs an app-code change).* Today `/api/health`
|
||||
(`api/main.py:343`) returns a static `{"ok": true}`, which only proves the
|
||||
dashboard *thread* is alive. The trading loop runs in the main thread
|
||||
(`run_live_paper.py:328`), so a **crashed** loop exits the process and Kubernetes
|
||||
restarts it (fine), but a **hung** loop (stuck network call, deadlock) keeps
|
||||
answering health checks `200` and runs undetected. Approach: have the loop stamp a
|
||||
last-tick timestamp into `DashboardState` each iteration, make `/api/health` (or a
|
||||
new `/healthz`) return `503` when the last tick is older than a threshold, and
|
||||
point the Deployment's `livenessProbe` at it. Highest-value item.
|
||||
- **Postgres backup CronJob.** The event store is the source of truth and nothing
|
||||
backs it up. Approach: a chart CronJob running `pg_dump` (postgres image, creds
|
||||
from the Secret) to the app `data` PVC with N-day retention; optionally offload to
|
||||
object storage.
|
||||
- **Schema bootstrap hook.** The schema auto-applies on the app's first connection,
|
||||
so a first-run race between the app and a CronJob is possible. Approach: a
|
||||
`pre-install`/`pre-upgrade` Helm-hook Job that applies `storage/pg_schema.sql`
|
||||
deterministically, with `PG_SKIP_SCHEMA=1` set on the app and CronJobs.
|
||||
|
||||
### Security
|
||||
|
||||
- **Pod hardening.** Add a `securityContext` across workloads: `runAsNonRoot`, drop
|
||||
ALL capabilities, `seccompProfile: RuntimeDefault`, and `readOnlyRootFilesystem`
|
||||
where feasible (the app needs `/app/data` writable — already a mounted volume).
|
||||
- **NetworkPolicy.** Default-deny in the namespace; allow only the app, CronJobs, and
|
||||
Grafana to reach Postgres:5432, and restrict who can reach the dashboard.
|
||||
- **Grafana auth.** The chart inherits anonymous-admin from the original compose
|
||||
(`templates/grafana.yaml` env). Make it off by default in-cluster, with a real
|
||||
admin password sourced from the Secret (`GF_SECURITY_ADMIN_PASSWORD`).
|
||||
|
||||
### CI / operational
|
||||
|
||||
- **Chart CI validation.** Add a CI job running `helm lint` + `helm template` +
|
||||
`kubeconform` on every push — catches template/schema errors automatically (the
|
||||
chart could not be rendered in the sandbox where it was authored, since image-blob
|
||||
CDNs were egress-blocked).
|
||||
- **Image digest pinning.** For production installs, pin `image` by SHA digest rather
|
||||
than the mutable `latest` tag.
|
||||
- **PodDisruptionBudget / singleton guard.** A PDB for the singleton loop plus a
|
||||
guard so it is never scaled above `1` or evicted to zero unexpectedly.
|
||||
@@ -0,0 +1,5 @@
|
||||
.DS_Store
|
||||
.git
|
||||
*.tmp
|
||||
*.bak
|
||||
*.orig
|
||||
@@ -0,0 +1,14 @@
|
||||
apiVersion: v2
|
||||
name: momentum
|
||||
description: Small-cap momentum research & paper-trading system, with an in-cluster PostgreSQL datastore.
|
||||
type: application
|
||||
# Chart version is independent of the app; bump on chart changes.
|
||||
version: 0.1.0
|
||||
# Tracks the application release (pyproject.toml).
|
||||
appVersion: "0.4.0"
|
||||
kubeVersion: ">=1.27.0-0"
|
||||
keywords:
|
||||
- trading
|
||||
- postgres
|
||||
- momentum
|
||||
home: https://github.com/fez5587/momentum-system
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"uid": "momentum-live",
|
||||
"title": "Momentum — Live",
|
||||
"tags": ["momentum"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "5s",
|
||||
"time": { "from": "now/d", "to": "now" },
|
||||
"templating": { "list": [] },
|
||||
"annotations": { "list": [] },
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Account equity",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "postgres", "uid": "momentum_pg" },
|
||||
"fieldConfig": { "defaults": { "unit": "currencyUSD" }, "overrides": [] },
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none" },
|
||||
"targets": [{
|
||||
"refId": "A",
|
||||
"format": "table",
|
||||
"rawQuery": true,
|
||||
"rawSql": "SELECT (payload_json::jsonb->>'total_equity')::float AS equity FROM events WHERE event_type='account_summary_updated' ORDER BY timestamp DESC LIMIT 1"
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Ready signals (today)",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
||||
"datasource": { "type": "postgres", "uid": "momentum_pg" },
|
||||
"fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "blue", "value": 1 } ] } }, "overrides": [] },
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
|
||||
"targets": [{
|
||||
"refId": "A",
|
||||
"format": "table",
|
||||
"rawQuery": true,
|
||||
"rawSql": "SELECT count(DISTINCT payload_json::jsonb->>'symbol')::int AS ready FROM events WHERE event_type='signal_ready'"
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "stat",
|
||||
"title": "Symbols evaluated",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "postgres", "uid": "momentum_pg" },
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none" },
|
||||
"targets": [{
|
||||
"refId": "A",
|
||||
"format": "table",
|
||||
"rawQuery": true,
|
||||
"rawSql": "SELECT count(DISTINCT payload_json::jsonb->>'symbol')::int AS evaluated FROM events WHERE event_type='criteria_evaluated'"
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "table",
|
||||
"title": "Evaluation board — what it's looking at + the determination",
|
||||
"gridPos": { "h": 14, "w": 14, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "postgres", "uid": "momentum_pg" },
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byName", "options": "status" },
|
||||
"properties": [ { "id": "custom.cellOptions", "value": { "type": "color-background" } },
|
||||
{ "id": "mappings", "value": [
|
||||
{ "type": "value", "options": { "ready": { "color": "green", "index": 0 }, "late": { "color": "yellow", "index": 1 }, "blocked": { "color": "red", "index": 2 } } } ] } ] },
|
||||
{ "matcher": { "id": "byName", "options": "gap" }, "properties": [ { "id": "unit", "value": "percentunit" } ] },
|
||||
{ "matcher": { "id": "byName", "options": "score" }, "properties": [ { "id": "unit", "value": "percent" } ] }
|
||||
]
|
||||
},
|
||||
"options": { "sortBy": [ { "displayName": "score", "desc": true } ] },
|
||||
"targets": [{
|
||||
"refId": "A",
|
||||
"format": "table",
|
||||
"rawQuery": true,
|
||||
"rawSql": "SELECT * FROM ( SELECT DISTINCT ON (sym) sym AS symbol, status, score, gap, rvol, reason FROM ( SELECT payload_json::jsonb->>'symbol' sym, payload_json::jsonb->'criteria_results'->>'status' status, (payload_json::jsonb->>'success_score_pct')::float score, (payload_json::jsonb->'criteria_results'->>'gap_pct')::float gap, (payload_json::jsonb->'criteria_results'->>'relative_volume')::float rvol, payload_json::jsonb->'criteria_results'->>'reason' reason, timestamp ts FROM events WHERE event_type='criteria_evaluated' ) t ORDER BY sym, ts DESC ) x ORDER BY score DESC"
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "table",
|
||||
"title": "Ready signals",
|
||||
"gridPos": { "h": 7, "w": 10, "x": 14, "y": 4 },
|
||||
"datasource": { "type": "postgres", "uid": "momentum_pg" },
|
||||
"targets": [{
|
||||
"refId": "A",
|
||||
"format": "table",
|
||||
"rawQuery": true,
|
||||
"rawSql": "SELECT timestamp AS time, payload_json::jsonb->>'symbol' AS symbol, message FROM events WHERE event_type='signal_ready' ORDER BY timestamp DESC LIMIT 25"
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "table",
|
||||
"title": "Risk events (breaker / backouts)",
|
||||
"gridPos": { "h": 7, "w": 10, "x": 14, "y": 11 },
|
||||
"datasource": { "type": "postgres", "uid": "momentum_pg" },
|
||||
"targets": [{
|
||||
"refId": "A",
|
||||
"format": "table",
|
||||
"rawQuery": true,
|
||||
"rawSql": "SELECT timestamp AS time, payload_json::jsonb->>'rule_type' AS rule, message FROM events WHERE event_type='risk_rule_triggered' ORDER BY timestamp DESC LIMIT 25"
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Minute bars ingested",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 18 },
|
||||
"datasource": { "type": "postgres", "uid": "momentum_pg" },
|
||||
"fieldConfig": { "defaults": { "custom": { "drawStyle": "bars", "fillOpacity": 40 } }, "overrides": [] },
|
||||
"targets": [{
|
||||
"refId": "A",
|
||||
"format": "time_series",
|
||||
"rawQuery": true,
|
||||
"rawSql": "SELECT timestamp AS time, (payload_json::jsonb->>'output_count')::float AS minute_rows FROM events WHERE event_type='module_tick' AND payload_json::jsonb->>'module'='ingestion' AND $__timeFilter(timestamp) ORDER BY 1"
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Account equity",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 18 },
|
||||
"datasource": { "type": "postgres", "uid": "momentum_pg" },
|
||||
"fieldConfig": { "defaults": { "unit": "currencyUSD" }, "overrides": [] },
|
||||
"targets": [{
|
||||
"refId": "A",
|
||||
"format": "time_series",
|
||||
"rawQuery": true,
|
||||
"rawSql": "SELECT timestamp AS time, (payload_json::jsonb->>'total_equity')::float AS equity FROM events WHERE event_type='account_summary_updated' AND $__timeFilter(timestamp) ORDER BY 1"
|
||||
}]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
Momentum System deployed as release "{{ .Release.Name }}" in namespace "{{ .Release.Namespace }}".
|
||||
|
||||
Components:
|
||||
- PostgreSQL (StatefulSet, single source of truth) : svc/{{ include "momentum.postgresName" . }}:{{ .Values.postgres.port }}
|
||||
- App loop + embedded dashboard (replicas: 1) : svc/{{ include "momentum.appName" . }}:{{ .Values.service.port }}
|
||||
{{- if .Values.cronjobs.enabled }}
|
||||
- CronJobs (journal, eod-replay, nightly-tune) : timeZone {{ .Values.cronjobs.timeZone }}
|
||||
{{- end }}
|
||||
{{- if .Values.grafana.enabled }}
|
||||
- Grafana : svc/{{ include "momentum.grafanaName" . }}:{{ .Values.grafana.service.port }}
|
||||
{{- end }}
|
||||
|
||||
Reach the dashboard:
|
||||
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "momentum.appName" . }} {{ .Values.service.port }}:{{ .Values.service.port }}
|
||||
open http://localhost:{{ .Values.service.port }}
|
||||
|
||||
{{- if not .Values.secrets.existingSecret }}
|
||||
{{- if or (not .Values.secrets.alpacaApiKey) (not .Values.secrets.alpacaSecretKey) }}
|
||||
|
||||
WARNING: Alpaca keys are not set — the loop runs in dry mode (no live data/orders).
|
||||
Supply them: helm upgrade {{ .Release.Name }} ... \
|
||||
--set secrets.alpacaApiKey=YOUR_KEY --set secrets.alpacaSecretKey=YOUR_SECRET
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
The DB schema auto-creates on the app's first connection. Verify:
|
||||
kubectl -n {{ .Release.Namespace }} exec sts/{{ include "momentum.postgresName" . }} -- \
|
||||
psql -U {{ .Values.postgres.username }} -d {{ .Values.postgres.database }} -c '\dt'
|
||||
@@ -0,0 +1,69 @@
|
||||
{{/* Base name, overridable. */}}
|
||||
{{- define "momentum.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Fully qualified release name. */}}
|
||||
{{- define "momentum.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if contains $name .Release.Name -}}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Common labels. */}}
|
||||
{{- define "momentum.labels" -}}
|
||||
app.kubernetes.io/name: {{ include "momentum.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Component-scoped selector labels. Call with (dict "ctx" . "component" "app"). */}}
|
||||
{{- define "momentum.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "momentum.name" .ctx }}
|
||||
app.kubernetes.io/instance: {{ .ctx.Release.Name }}
|
||||
app.kubernetes.io/component: {{ .component }}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Names of shared resources. */}}
|
||||
{{- define "momentum.postgresName" -}}{{ printf "%s-postgres" (include "momentum.fullname" .) }}{{- end -}}
|
||||
{{- define "momentum.appName" -}}{{ printf "%s-app" (include "momentum.fullname" .) }}{{- end -}}
|
||||
{{- define "momentum.grafanaName" -}}{{ printf "%s-grafana" (include "momentum.fullname" .) }}{{- end -}}
|
||||
{{- define "momentum.configMapName" -}}{{ printf "%s-config" (include "momentum.fullname" .) }}{{- end -}}
|
||||
{{- define "momentum.appDataPvcName" -}}{{ printf "%s-app-data" (include "momentum.fullname" .) }}{{- end -}}
|
||||
|
||||
{{/* Name of the Secret holding DB URL + broker keys (existing or chart-managed). */}}
|
||||
{{- define "momentum.secretName" -}}
|
||||
{{- if .Values.secrets.existingSecret -}}
|
||||
{{- .Values.secrets.existingSecret -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-secrets" (include "momentum.fullname" .) -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Resolve the Postgres password once: explicit value, else reuse the one already
|
||||
stored in the chart-managed Secret (so upgrades don't rotate it), else generate.
|
||||
*/}}
|
||||
{{- define "momentum.postgresPassword" -}}
|
||||
{{- if .Values.postgres.password -}}
|
||||
{{- .Values.postgres.password -}}
|
||||
{{- else -}}
|
||||
{{- $existing := lookup "v1" "Secret" .Release.Namespace (printf "%s-secrets" (include "momentum.fullname" .)) -}}
|
||||
{{- if and $existing $existing.data (index $existing.data "POSTGRES_PASSWORD") -}}
|
||||
{{- index $existing.data "POSTGRES_PASSWORD" | b64dec -}}
|
||||
{{- else -}}
|
||||
{{- randAlphaNum 24 -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@@ -0,0 +1,16 @@
|
||||
{{- if .Values.app.persistence.enabled }}
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "momentum.appDataPvcName" . }}
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes: ["{{ .Values.app.persistence.accessMode }}"]
|
||||
{{- if .Values.app.persistence.storageClass }}
|
||||
storageClassName: {{ .Values.app.persistence.storageClass | quote }}
|
||||
{{- end }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.app.persistence.size | quote }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,80 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "momentum.appName" . }}
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: app
|
||||
spec:
|
||||
# Stateful singleton: the loop holds in-memory trading state and a 1:1 broker
|
||||
# session. Never scale > 1. Recreate avoids two loops overlapping on rollout.
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "momentum.selectorLabels" (dict "ctx" . "component" "app") | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "momentum.selectorLabels" (dict "ctx" . "component" "app") | nindent 8 }}
|
||||
annotations:
|
||||
# Roll the pod when config/secret content changes.
|
||||
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
spec:
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
# Block until Postgres accepts connections so the first app connection
|
||||
# (which auto-applies storage/pg_schema.sql) succeeds.
|
||||
- name: wait-for-postgres
|
||||
image: {{ .Values.postgres.image | quote }}
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until pg_isready -h {{ include "momentum.postgresName" . }} -p {{ .Values.postgres.port }} -U {{ .Values.postgres.username }}; do
|
||||
echo "waiting for postgres..."; sleep 2;
|
||||
done
|
||||
containers:
|
||||
- name: app
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command: {{ toYaml .Values.app.command | nindent 12 }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8010
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "momentum.configMapName" . }}
|
||||
- secretRef:
|
||||
name: {{ include "momentum.secretName" . }}
|
||||
volumeMounts:
|
||||
- name: app-data
|
||||
mountPath: /app/data
|
||||
{{- if .Values.app.probes.enabled }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
initialDelaySeconds: {{ .Values.app.probes.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.app.probes.periodSeconds }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
initialDelaySeconds: {{ .Values.app.probes.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.app.probes.periodSeconds }}
|
||||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.app.resources | nindent 12 }}
|
||||
volumes:
|
||||
- name: app-data
|
||||
{{- if .Values.app.persistence.enabled }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "momentum.appDataPvcName" . }}
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,14 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "momentum.appName" . }}
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
spec:
|
||||
type: {{ .Values.service.type }}
|
||||
selector:
|
||||
{{- include "momentum.selectorLabels" (dict "ctx" . "component" "app") | nindent 4 }}
|
||||
ports:
|
||||
- name: http
|
||||
port: {{ .Values.service.port }}
|
||||
targetPort: http
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "momentum.configMapName" . }}
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
data:
|
||||
{{- range $key, $val := .Values.config }}
|
||||
{{ $key }}: {{ $val | quote }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,59 @@
|
||||
{{- if .Values.cronjobs.enabled }}
|
||||
{{- $root := . }}
|
||||
{{- $jobs := dict "journal" .Values.cronjobs.journal "eod-replay" .Values.cronjobs.eodReplay "nightly-tune" .Values.cronjobs.nightlyTune }}
|
||||
{{- range $key, $job := $jobs }}
|
||||
{{- if $job.enabled }}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: {{ include "momentum.fullname" $root }}-{{ $key }}
|
||||
labels:
|
||||
{{- include "momentum.labels" $root | nindent 4 }}
|
||||
app.kubernetes.io/component: cron-{{ $key }}
|
||||
spec:
|
||||
schedule: {{ $job.schedule | quote }}
|
||||
# US/Eastern so the market-clock schedules in .env's *.sh comments hold.
|
||||
timeZone: {{ $root.Values.cronjobs.timeZone | quote }}
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "momentum.selectorLabels" (dict "ctx" $root "component" (printf "cron-%s" $key)) | nindent 12 }}
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
{{- with $root.Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: {{ $key }}
|
||||
image: "{{ $root.Values.image.repository }}:{{ $root.Values.image.tag }}"
|
||||
imagePullPolicy: {{ $root.Values.image.pullPolicy }}
|
||||
command: {{ toYaml $job.command | nindent 16 }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "momentum.configMapName" $root }}
|
||||
- secretRef:
|
||||
name: {{ include "momentum.secretName" $root }}
|
||||
volumeMounts:
|
||||
- name: app-data
|
||||
mountPath: /app/data
|
||||
volumes:
|
||||
- name: app-data
|
||||
{{- if $root.Values.app.persistence.enabled }}
|
||||
# Shared with the app pod (nightly-tune writes learned_params.json
|
||||
# the loop reads on boot). RWO ⇒ cron pods schedule onto the app's node.
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "momentum.appDataPvcName" $root }}
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,158 @@
|
||||
{{- if .Values.grafana.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "momentum.grafanaName" . }}-provisioning
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
data:
|
||||
# Datasource points at the in-cluster Postgres service (not the old LAN IP).
|
||||
datasource.yaml: |
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Momentum
|
||||
uid: momentum_pg
|
||||
type: postgres
|
||||
access: proxy
|
||||
url: {{ include "momentum.postgresName" . }}:{{ .Values.postgres.port }}
|
||||
user: {{ .Values.postgres.username }}
|
||||
jsonData:
|
||||
database: {{ .Values.postgres.database }}
|
||||
sslmode: disable
|
||||
postgresVersion: 1600
|
||||
secureJsonData:
|
||||
# Resolved at runtime from the env var below so it always matches the
|
||||
# password stored in the Secret (avoids re-deriving it here).
|
||||
password: $POSTGRES_PASSWORD
|
||||
isDefault: true
|
||||
editable: true
|
||||
provider.yaml: |
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: momentum
|
||||
orgId: 1
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 10
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
path: /etc/grafana/provisioning/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "momentum.grafanaName" . }}-dashboards
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
data:
|
||||
momentum.json: |-
|
||||
{{- .Files.Get "files/dashboard.json" | nindent 4 }}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "momentum.grafanaName" . }}
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: grafana
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "momentum.selectorLabels" (dict "ctx" . "component" "grafana") | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "momentum.selectorLabels" (dict "ctx" . "component" "grafana") | nindent 8 }}
|
||||
spec:
|
||||
containers:
|
||||
- name: grafana
|
||||
image: {{ .Values.grafana.image | quote }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3000
|
||||
env:
|
||||
- name: GF_AUTH_ANONYMOUS_ENABLED
|
||||
value: "true"
|
||||
- name: GF_AUTH_ANONYMOUS_ORG_ROLE
|
||||
value: "Admin"
|
||||
- name: GF_AUTH_DISABLE_LOGIN_FORM
|
||||
value: "true"
|
||||
- name: GF_DASHBOARDS_MIN_REFRESH_INTERVAL
|
||||
value: "2s"
|
||||
# Interpolated into the provisioned datasource (secureJsonData.password).
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "momentum.secretName" . }}
|
||||
key: POSTGRES_PASSWORD
|
||||
volumeMounts:
|
||||
- name: datasources
|
||||
mountPath: /etc/grafana/provisioning/datasources
|
||||
- name: dashboard-provider
|
||||
mountPath: /etc/grafana/provisioning/dashboards/provider.yaml
|
||||
subPath: provider.yaml
|
||||
- name: dashboards
|
||||
mountPath: /etc/grafana/provisioning/dashboards/momentum.json
|
||||
subPath: momentum.json
|
||||
- name: storage
|
||||
mountPath: /var/lib/grafana
|
||||
resources:
|
||||
{{- toYaml .Values.grafana.resources | nindent 12 }}
|
||||
volumes:
|
||||
- name: datasources
|
||||
configMap:
|
||||
name: {{ include "momentum.grafanaName" . }}-provisioning
|
||||
items:
|
||||
- key: datasource.yaml
|
||||
path: datasource.yaml
|
||||
- name: dashboard-provider
|
||||
configMap:
|
||||
name: {{ include "momentum.grafanaName" . }}-provisioning
|
||||
items:
|
||||
- key: provider.yaml
|
||||
path: provider.yaml
|
||||
- name: dashboards
|
||||
configMap:
|
||||
name: {{ include "momentum.grafanaName" . }}-dashboards
|
||||
- name: storage
|
||||
{{- if .Values.grafana.persistence.enabled }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "momentum.grafanaName" . }}
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "momentum.grafanaName" . }}
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
spec:
|
||||
type: {{ .Values.grafana.service.type }}
|
||||
selector:
|
||||
{{- include "momentum.selectorLabels" (dict "ctx" . "component" "grafana") | nindent 4 }}
|
||||
ports:
|
||||
- name: http
|
||||
port: {{ .Values.grafana.service.port }}
|
||||
targetPort: http
|
||||
{{- if .Values.grafana.persistence.enabled }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "momentum.grafanaName" . }}
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
{{- if .Values.grafana.persistence.storageClass }}
|
||||
storageClassName: {{ .Values.grafana.persistence.storageClass | quote }}
|
||||
{{- end }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.grafana.persistence.size | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,31 @@
|
||||
{{- if .Values.ingress.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "momentum.appName" . }}
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.ingress.className }}
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
{{- end }}
|
||||
{{- with .Values.ingress.tls }}
|
||||
tls:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
- host: {{ .Values.ingress.host | quote }}
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "momentum.appName" . }}
|
||||
port:
|
||||
number: {{ .Values.service.port }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "momentum.postgresName" . }}
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
spec:
|
||||
# Headless: stable DNS for the single StatefulSet pod.
|
||||
clusterIP: None
|
||||
selector:
|
||||
{{- include "momentum.selectorLabels" (dict "ctx" . "component" "postgres") | nindent 4 }}
|
||||
ports:
|
||||
- name: postgres
|
||||
port: {{ .Values.postgres.port }}
|
||||
targetPort: postgres
|
||||
@@ -0,0 +1,67 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "momentum.postgresName" . }}
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: postgres
|
||||
spec:
|
||||
serviceName: {{ include "momentum.postgresName" . }}
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "momentum.selectorLabels" (dict "ctx" . "component" "postgres") | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "momentum.selectorLabels" (dict "ctx" . "component" "postgres") | nindent 8 }}
|
||||
spec:
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: postgres
|
||||
image: {{ .Values.postgres.image | quote }}
|
||||
ports:
|
||||
- name: postgres
|
||||
containerPort: {{ .Values.postgres.port }}
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: {{ .Values.postgres.username | quote }}
|
||||
- name: POSTGRES_DB
|
||||
value: {{ .Values.postgres.database | quote }}
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "momentum.secretName" . }}
|
||||
key: POSTGRES_PASSWORD
|
||||
# Keep PGDATA in a subdir so the volume root (lost+found) doesn't clash.
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data/pgdata
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/postgresql/data
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["pg_isready", "-U", {{ .Values.postgres.username | quote }}, "-d", {{ .Values.postgres.database | quote }}]
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["pg_isready", "-U", {{ .Values.postgres.username | quote }}, "-d", {{ .Values.postgres.database | quote }}]
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
resources:
|
||||
{{- toYaml .Values.postgres.resources | nindent 12 }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes: ["{{ .Values.postgres.persistence.accessMode }}"]
|
||||
{{- if .Values.postgres.persistence.storageClass }}
|
||||
storageClassName: {{ .Values.postgres.persistence.storageClass | quote }}
|
||||
{{- end }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.postgres.persistence.size | quote }}
|
||||
@@ -0,0 +1,26 @@
|
||||
{{- if not .Values.secrets.existingSecret }}
|
||||
{{- $pgPassword := include "momentum.postgresPassword" . }}
|
||||
{{- $pgHost := include "momentum.postgresName" . }}
|
||||
{{- $databaseUrl := printf "postgresql://%s:%s@%s:%v/%s" .Values.postgres.username $pgPassword $pgHost .Values.postgres.port .Values.postgres.database }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "momentum.fullname" . }}-secrets
|
||||
labels:
|
||||
{{- include "momentum.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
stringData:
|
||||
# Consumed by the Postgres StatefulSet.
|
||||
POSTGRES_PASSWORD: {{ $pgPassword | quote }}
|
||||
# Consumed by the app + CronJobs (storage/db_pg.py prefers DATABASE_URL).
|
||||
DATABASE_URL: {{ $databaseUrl | quote }}
|
||||
# Alpaca paper broker (the client reads both the ALPACA_* and APCA_* names).
|
||||
ALPACA_API_KEY: {{ .Values.secrets.alpacaApiKey | quote }}
|
||||
ALPACA_SECRET_KEY: {{ .Values.secrets.alpacaSecretKey | quote }}
|
||||
APCA_API_KEY_ID: {{ .Values.secrets.alpacaApiKey | quote }}
|
||||
APCA_API_SECRET_KEY: {{ .Values.secrets.alpacaSecretKey | quote }}
|
||||
APCA_API_BASE_URL: {{ .Values.secrets.alpacaBaseUrl | quote }}
|
||||
{{- range $k, $v := .Values.secrets.schwab }}
|
||||
{{ $k }}: {{ $v | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,162 @@
|
||||
# Default values for the momentum chart.
|
||||
# Override per-environment with `-f my-values.yaml` or `--set key=value`.
|
||||
|
||||
# -- Container image for every workload (app loop, dashboard, CronJobs).
|
||||
image:
|
||||
repository: ghcr.io/fez5587/momentum-system
|
||||
# Pin to a pushed tag (commit SHA or `latest`) from the docker-publish workflow.
|
||||
tag: latest
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# Optional image pull secrets for a private GHCR package.
|
||||
imagePullSecrets: []
|
||||
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# App: the live trading loop WITH its embedded dashboard (port 8010).
|
||||
# It is a stateful singleton — do not raise replicas above 1.
|
||||
# ----------------------------------------------------------------------------
|
||||
app:
|
||||
# Runs the loop with the dashboard attached (the dashboard's approve/reject/
|
||||
# exit endpoints require the in-process TradingExecutionService).
|
||||
command: ["python", "run_live_paper.py"]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
# Shared writable dir (DuckDB research file, logs, learned_params.json, tokens).
|
||||
# The nightly-tune CronJob writes learned_params.json here for the loop to read.
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 5Gi
|
||||
storageClass: ""
|
||||
accessMode: ReadWriteOnce
|
||||
# HTTP probes hit the dashboard root.
|
||||
probes:
|
||||
enabled: true
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
|
||||
# Dashboard service (ClusterIP by default; port-forward to reach it).
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8010
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
className: ""
|
||||
annotations: {}
|
||||
host: momentum.local
|
||||
tls: []
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# In-cluster PostgreSQL — the single source of truth (StatefulSet + PVC).
|
||||
# Schema auto-creates on first app connection from storage/pg_schema.sql.
|
||||
# ----------------------------------------------------------------------------
|
||||
postgres:
|
||||
image: postgres:16
|
||||
database: momentum
|
||||
username: momentum
|
||||
# If empty, a random password is generated on first install and stored in the
|
||||
# chart-managed Secret. Set explicitly to keep it stable across upgrades.
|
||||
password: ""
|
||||
port: 5432
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
persistence:
|
||||
size: 10Gi
|
||||
storageClass: ""
|
||||
accessMode: ReadWriteOnce
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Secrets — DB URL is built from the in-cluster Postgres service automatically.
|
||||
# Provide broker keys via --set or, better, an existingSecret (sealed-secrets,
|
||||
# external-secrets, etc). Never commit real keys into values.yaml.
|
||||
# ----------------------------------------------------------------------------
|
||||
secrets:
|
||||
# If set, the chart will NOT render its own Secret and instead expects this
|
||||
# one to already contain DATABASE_URL, ALPACA_*/APCA_*, and any SCHWAB_* keys.
|
||||
existingSecret: ""
|
||||
alpacaApiKey: ""
|
||||
alpacaSecretKey: ""
|
||||
alpacaBaseUrl: "https://paper-api.alpaca.markets"
|
||||
schwab: {} # e.g. SCHWAB_MARKET_DATA_APP_KEY: "..."
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Non-secret tuning env (ConfigMap). Mirrors .env.example; override freely.
|
||||
# ----------------------------------------------------------------------------
|
||||
config:
|
||||
TRADING_EXECUTION_MODE: "alpaca_paper"
|
||||
TRADING_EXECUTION_ENABLED: "1"
|
||||
TRADING_AUTO_APPROVE: "0"
|
||||
TRADING_MAX_CONCURRENT_POSITIONS: "6"
|
||||
TRADING_RISK_PER_TRADE_PCT: "0.01"
|
||||
TRADING_MAX_DAILY_LOSS_PCT: "0.03"
|
||||
TRADING_EOD_FLATTEN_ENABLED: "1"
|
||||
WATCHER_ENABLED: "1"
|
||||
WATCHER_INTERVAL_SECONDS: "60"
|
||||
WATCHER_SYMBOLS: "AAPL,TSLA,NVDA,AMD,SOFI"
|
||||
WATCHER_PRICE_MIN: "1"
|
||||
WATCHER_PRICE_MAX: "20"
|
||||
STRATEGY_SETUPS: "opening_range_break"
|
||||
LIVE_BARS_ENABLED: "1"
|
||||
LIVE_BARS_INTERVAL_SECONDS: "60"
|
||||
ALPACA_DATA_FEED: "iex"
|
||||
ALPACA_BACKFILL_FEED: "sip"
|
||||
TRIGGER_ENABLED: "1"
|
||||
TRIGGER_GAP_MIN: "3.0"
|
||||
TRIGGER_GAP_MAX: "35"
|
||||
TRIGGER_RVOL_MIN: "2.0"
|
||||
ORB_BARS: "5"
|
||||
DASHBOARD_HOST: "0.0.0.0"
|
||||
DASHBOARD_PORT: "8010"
|
||||
# Leave schema auto-creation on (storage/db_pg.py applies pg_schema.sql).
|
||||
PG_SKIP_SCHEMA: "0"
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Scheduled jobs (CronJobs). Times are US/Eastern via spec.timeZone (K8s >=1.27),
|
||||
# weekdays only — matching the existing *.sh crontab comments.
|
||||
# ----------------------------------------------------------------------------
|
||||
cronjobs:
|
||||
enabled: true
|
||||
timeZone: America/New_York
|
||||
journal:
|
||||
enabled: true
|
||||
schedule: "2 16 * * 1-5"
|
||||
command: ["python", "momentum_cli.py", "journal"]
|
||||
eodReplay:
|
||||
enabled: true
|
||||
schedule: "5 16 * * 1-5"
|
||||
command: ["python", "eod_replay.py"]
|
||||
nightlyTune:
|
||||
enabled: true
|
||||
schedule: "30 17 * * 1-5"
|
||||
command: ["python", "nightly_tune.py"]
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Grafana (optional) — provisioned against the in-cluster Postgres service.
|
||||
# ----------------------------------------------------------------------------
|
||||
grafana:
|
||||
enabled: true
|
||||
image: grafana/grafana:11.2.0
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 3000
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 1Gi
|
||||
storageClass: ""
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
@@ -0,0 +1,30 @@
|
||||
# Local deployment (systemd user services)
|
||||
|
||||
Unit templates for running the stack from a git checkout + venv (the original
|
||||
production shape; now the development shape). Paths inside the units assume
|
||||
`/mnt/c/code/momentum` + `/home/philip/.venvs/momentum` — adjust to your box.
|
||||
|
||||
```bash
|
||||
cp deploy/systemd/momentum-*.service ~/.config/systemd/user/
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable --now momentum-loop momentum-dashboard momentum-qabot momentum-transcripts
|
||||
systemctl --user enable --now momentum-premarket-eval.timer # 20:00 ET weekdays
|
||||
loginctl enable-linger $USER # start on boot without login
|
||||
```
|
||||
|
||||
| Unit | What |
|
||||
|---|---|
|
||||
| momentum-loop | trading loop (`run_live_paper.py --no-dashboard`) |
|
||||
| momentum-dashboard | web dashboard :8765 (`dashboard_api.py`) |
|
||||
| momentum-qabot | Telegram Q&A bot (read-only) |
|
||||
| momentum-transcripts | YouTube → transcript → claims pipeline |
|
||||
| momentum-premarket-eval(.timer) | nightly SIP refetch + housekeeping + shadow-track re-validation |
|
||||
| momentum-testpg | local Postgres for test isolation (`/home/philip/pgrun` socket) — keep even when production is elsewhere |
|
||||
| momentum-controlroom | NiceGUI control room :8766 — **disabled by owner preference**; kept for reference |
|
||||
|
||||
Config comes from `.env` in the repo root (see `.env.example`). The same
|
||||
behavior-flag warnings as compose apply (`TRADING_AUTO_APPROVE=1` etc. — see
|
||||
`deploy/README.md`).
|
||||
|
||||
**Never run these enabled while the compose stack owns production** — two loops
|
||||
on one Alpaca account duplicate orders.
|
||||
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
Description=Momentum dashboard (read-only web UI on :8765)
|
||||
# Self-heal forever: a DB blip or crash must not leave the website down. Decoupled
|
||||
# from the loop on purpose -- a loop restart never takes the dashboard down and vice versa.
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/mnt/c/code/momentum
|
||||
# dashboard_api.py loads .env (DATABASE_URL -> the LAN Postgres, with the host fallback)
|
||||
# and binds DASHBOARD_HOST (default 0.0.0.0) : DASHBOARD_PORT (.env=8765) so it is reachable
|
||||
# from the Windows browser via the netsh portproxy.
|
||||
ExecStart=/home/philip/.venvs/momentum/bin/python -u dashboard_api.py
|
||||
Restart=always
|
||||
RestartSec=15
|
||||
StandardOutput=append:/mnt/c/code/momentum/data/dashboard.log
|
||||
StandardError=append:/mnt/c/code/momentum/data/dashboard.log
|
||||
SyslogIdentifier=momentum-dashboard
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -0,0 +1,24 @@
|
||||
[Unit]
|
||||
Description=Momentum live paper-trading loop
|
||||
# Self-heal: if the loop crashes or the box reboots, systemd brings it back.
|
||||
# The DB host (192.168.1.5) can go fully unreachable (a network/host outage) and
|
||||
# the loop crashes on connect — so retry FOREVER (no give-up limit) and it
|
||||
# auto-reconnects the moment the host returns. StartLimitIntervalSec=0 disables
|
||||
# the "give up after N starts" cap entirely.
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/mnt/c/code/momentum
|
||||
# the loop calls load_dotenv() -> finds .env in WorkingDirectory -> DATABASE_URL
|
||||
# and all secrets load from there; no secrets in this unit file.
|
||||
ExecStart=/home/philip/.venvs/momentum/bin/python -u run_live_paper.py --no-dashboard
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
# keep the same log file the operator + the review crons already tail
|
||||
StandardOutput=append:/mnt/c/code/momentum/data/live_run.log
|
||||
StandardError=append:/mnt/c/code/momentum/data/live_run.log
|
||||
SyslogIdentifier=momentum-loop
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Momentum premarket (pm1) shadow track rebuild + re-validate
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/mnt/c/code/momentum
|
||||
# DATABASE_URL loads from the repo's gitignored .env via research.premarket_eval (python-dotenv)
|
||||
ExecStart=/home/philip/.venvs/momentum/bin/python -m research.premarket_eval
|
||||
StandardOutput=append:/mnt/c/code/momentum/data/premarket_eval.log
|
||||
StandardError=append:/mnt/c/code/momentum/data/premarket_eval.log
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Weekday-evening rebuild + re-validate of the premarket (pm1) shadow track
|
||||
|
||||
[Timer]
|
||||
# after each trading day's premarket data is ingested; Persistent catches missed runs (reboots)
|
||||
OnCalendar=Mon-Fri *-*-* 20:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,17 @@
|
||||
[Unit]
|
||||
Description=Momentum Telegram Q&A bot
|
||||
# retry forever through a DB-host outage (see momentum-loop.service)
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/mnt/c/code/momentum
|
||||
ExecStart=/home/philip/.venvs/momentum/bin/python -u telegram_bot.py
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
StandardOutput=append:/mnt/c/code/momentum/data/telegram_bot.log
|
||||
StandardError=append:/mnt/c/code/momentum/data/telegram_bot.log
|
||||
SyslogIdentifier=momentum-qabot
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Local Postgres for momentum test isolation (socket /home/philip/pgrun)
|
||||
|
||||
[Service]
|
||||
Type=forking
|
||||
ExecStart=/home/philip/pg/bin/pg_ctl -D /home/philip/pgdata -o "-k /home/philip/pgrun" -l /home/philip/pgdata/logfile start
|
||||
ExecStop=/home/philip/pg/bin/pg_ctl -D /home/philip/pgdata stop -m fast
|
||||
Restart=on-failure
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=Momentum YouTube transcript -> claim-extraction pipeline
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/mnt/c/code/momentum
|
||||
# secrets (DATABASE_URL, YOUTUBE_API_KEY, YT_WATCHLIST) load from the repo's gitignored .env
|
||||
# via youtube_claims.config (python-dotenv). Whisper is option A (CPU) by default; flip to the
|
||||
# LAN GPU (option B) with YT_WHISPER_MODEL=large-v3 YT_WHISPER_DEVICE=cuda YT_WHISPER_COMPUTE_TYPE=float16.
|
||||
ExecStart=/home/philip/.venvs/momentum/bin/python -u -m youtube_claims.run run
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
StandardOutput=append:/mnt/c/code/momentum/data/transcripts_pipeline.log
|
||||
StandardError=append:/mnt/c/code/momentum/data/transcripts_pipeline.log
|
||||
SyslogIdentifier=momentum-transcripts
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -0,0 +1,282 @@
|
||||
# Full self-contained stack for the Momentum System.
|
||||
#
|
||||
# docker compose up -d # postgres + loop + dashboard + qabot + transcripts + scheduler
|
||||
# docker compose --profile monitoring up -d # also start Grafana
|
||||
# docker compose run --rm nightly-tune # run a job on demand
|
||||
#
|
||||
# Secrets come from .env (never baked into the image). DATABASE_URL is
|
||||
# overridden below to the in-stack postgres so the stack is self-contained —
|
||||
# migrating from the LAN Postgres (192.168.1.5) is a pg_dump/restore into the
|
||||
# pg-data volume; see deploy/docker/README.md. To run the app services against
|
||||
# the LAN Postgres instead (transition mode), comment out the DATABASE_URL
|
||||
# override and let .env's value through.
|
||||
#
|
||||
# TZ is America/New_York in the image: the loop's market clock thinks in ET.
|
||||
|
||||
x-app: &app
|
||||
build: .
|
||||
image: momentum-app:latest
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
volumes:
|
||||
- app-data:/app/data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options: { max-size: "20m", max-file: "3" }
|
||||
|
||||
x-app-env: &app-env
|
||||
# Self-contained default: the in-stack postgres. Overrides .env.
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-momentum}:${POSTGRES_PASSWORD:-momentum}@postgres:5432/${POSTGRES_DB:-momentum}
|
||||
PG_SKIP_SCHEMA: "0"
|
||||
LIVE_BARS_INTERVAL_SECONDS: "5"
|
||||
TRIGGER_ARM_INTERVAL_SECONDS: "8"
|
||||
WATCHER_INTERVAL_SECONDS: "8"
|
||||
TRIGGER_INTERVAL_SECONDS: "2"
|
||||
# catalyst enrichment ON as a labeler FEATURE source (plan 009) — loop + scheduler
|
||||
# (labeler builds) both inherit this anchor. Needs a reachable Ollama (see .env);
|
||||
# enabled-but-down never impedes trading (fully gated).
|
||||
NEWS_ENRICH_ENABLED: "1"
|
||||
# Managed-exit config — matches deploy/docker/truenas-compose.yml so both
|
||||
# templates agree. Without these the loop runs code defaults (target 2R,
|
||||
# BE@+5%, NO trail) which scratches slow v1 winners at breakeven. prior_low
|
||||
# trailing after +1R lifts the win rate (counterfactual: v1 17%->24%). Only the
|
||||
# LOOP acts on these (ExitConfig.from_env, run_live_paper.py); other services
|
||||
# inherit them harmlessly.
|
||||
TRADING_EXIT_TRAIL_MODE: "prior_low"
|
||||
TRADING_EXIT_TRAIL_AFTER_R: "1.0"
|
||||
TRADING_EXIT_MANAGE_INTERVAL_SECONDS: '4' # 12->4s: react to fast small-cap round-trips (was giving back gains)
|
||||
TRADING_EXIT_THREAD_ENABLED: '1'
|
||||
DISCOVER_THREAD_ENABLED: '1'
|
||||
TRADING_EXIT_THREAD_INTERVAL_SECONDS: '1.0'
|
||||
TRADING_EXIT_TRAIL_R_STEP: "0.25"
|
||||
TRADING_EXIT_BREAKEVEN_R: "1.0"
|
||||
TRADING_EXIT_PROFIT_TIERS: "8:3,15:9,25:18,40:30"
|
||||
# Regime gate (plan 011) LIVE: block fresh entries on sessions the walk-forward
|
||||
# metalabel scores unfavorable (sit out bad regimes). Only the LOOP acts on it;
|
||||
# fail-open (unfit model / error never blocks). Needs regime_features populated
|
||||
# (the scheduler builds them nightly). Set 0 to disable.
|
||||
TRADING_REGIME_GATE_ENABLED: "1"
|
||||
# Favorable threshold: block a session when P(profit) < this. 0.5 = code default
|
||||
# (matches the backtest). Lower (e.g. 0.4) trades MORE days; raise for fewer,
|
||||
# higher-confidence days. Optional — omit to use the 0.5 default.
|
||||
TRADING_REGIME_GATE_P_MIN: "0.5"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-momentum}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-momentum}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-momentum}
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
volumes:
|
||||
- pg-data:/var/lib/postgresql/data
|
||||
ports:
|
||||
- "5432:5432" # psql/Grafana-on-host debugging; remove if not needed
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-momentum} -d ${POSTGRES_DB:-momentum}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
loop:
|
||||
<<: *app
|
||||
container_name: momentum-loop
|
||||
command: ["python", "-u", "run_live_paper.py", "--no-dashboard"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
healthcheck:
|
||||
# heartbeat: the loop must have written SOME event in the last 5 minutes
|
||||
# (module ticks flow around the clock, not just market hours)
|
||||
# PG_SKIP_SCHEMA=1: a healthcheck must NOT re-apply schema DDL — the 60s
|
||||
# CREATE-IF-NOT-EXISTS lock burst cancelled minute_bars autovacuum forever
|
||||
# (2026-07-09 post-migration: stats rotted, dashboard queries crawled)
|
||||
test: ["CMD", "python", "-c", "import os; os.environ['PG_SKIP_SCHEMA']='1'; import datetime,sys; from research.multi_schema import open_research_db; r=open_research_db('market').execute('SELECT max(timestamp) FROM events').fetchall()[0][0]; sys.exit(0 if r and (datetime.datetime.now()-r).total_seconds()<300 else 1)"]
|
||||
interval: 60s
|
||||
timeout: 20s
|
||||
retries: 5
|
||||
start_period: 180s
|
||||
|
||||
dashboard:
|
||||
<<: *app
|
||||
container_name: momentum-dashboard
|
||||
command: ["python", "-u", "dashboard_api.py"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
DASHBOARD_HOST: "0.0.0.0"
|
||||
DASHBOARD_PORT: "8765"
|
||||
# read-only viewer: never re-apply the 473-line schema (the SSE change
|
||||
# detector holds one long-lived connection) — mirrors the loop healthcheck.
|
||||
PG_SKIP_SCHEMA: "1"
|
||||
# Shared secret required on every POST (approve/reject/exit + the command
|
||||
# bus). Empty = commands disabled and POSTs unauthenticated; set a value to
|
||||
# arm the kill switch. Reference by name only — never inline the secret.
|
||||
DASHBOARD_TOKEN: "${DASHBOARD_TOKEN:-}"
|
||||
# Gate the operator command bus on (also needs DASHBOARD_TOKEN set).
|
||||
COMMAND_BUS_ENABLED: "${COMMAND_BUS_ENABLED:-0}"
|
||||
ports:
|
||||
- "8765:8765"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "http://localhost:8765/"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
|
||||
qabot:
|
||||
<<: *app
|
||||
container_name: momentum-qabot
|
||||
command: ["python", "-u", "telegram_bot.py"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
|
||||
transcripts:
|
||||
<<: *app
|
||||
container_name: momentum-transcripts
|
||||
command: ["python", "-u", "-m", "youtube_claims.run", "run"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
|
||||
admin:
|
||||
<<: *app
|
||||
container_name: momentum-admin
|
||||
command: ["python", "-u", "admin_api.py"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
ADMIN_PORT: "8767"
|
||||
# set ADMIN_TOKEN in .env — without it the panel is open on the LAN
|
||||
ports:
|
||||
- "8767:8767"
|
||||
volumes:
|
||||
- app-data:/app/data
|
||||
- /var/run/docker.sock:/var/run/docker.sock # service status + restarts
|
||||
|
||||
scheduler:
|
||||
# Weekday-evening premarket eval + DB/log housekeeping at 20:00 ET —
|
||||
# replaces the momentum-premarket-eval systemd timer.
|
||||
<<: *app
|
||||
container_name: momentum-scheduler
|
||||
environment:
|
||||
<<: *app-env
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
echo "[scheduler] up; will fire 20:00 ET Mon-Fri"
|
||||
while true; do
|
||||
if [ "$$(date +%H:%M)" = "20:00" ] && [ "$$(date +%u)" -le 5 ]; then
|
||||
echo "[scheduler] premarket_eval $$(date)"
|
||||
python -m research.premarket_eval || echo "[scheduler] eval FAILED"
|
||||
# SPY/QQQ ingest -> forward source for the labeler's market-context features (plan 009)
|
||||
python -m research.ingestion.market_context || echo "[scheduler] market_context ingest FAILED"
|
||||
python -m research.eod_reconcile || echo "[scheduler] reconcile FAILED"
|
||||
# Flywheel: grow the whole-market dataset first, then fit the tuner on
|
||||
# it. 20:00 ET is after close, so replay of "today" is complete data —
|
||||
# NOT look-ahead (replay is a labeling job, not a live gate).
|
||||
python eod_replay.py || echo "[scheduler] eod_replay FAILED"
|
||||
python nightly_tune.py || echo "[scheduler] nightly_tune FAILED"
|
||||
# Regime gate (plan 011, ALL-SHADOW): a session-level metalabel
|
||||
# P(our ORB profits today). build-features labels each session from its
|
||||
# OWN realized expR; score writes the WALK-FORWARD favorable flag the
|
||||
# experiments' regime_gate intervention reads. Must run BEFORE
|
||||
# experiments run. Does NOT gate live trading (that is owner-gated).
|
||||
python -m research.regime build-features || echo "[scheduler] regime build-features FAILED"
|
||||
python -m research.regime score --model threshold_v1 || echo "[scheduler] regime score threshold FAILED"
|
||||
python -m research.regime score --model meta_v1 || echo "[scheduler] regime score meta FAILED"
|
||||
# Experiment OS (plan 006, ALL-SHADOW): replay running experiments over
|
||||
# newly-logged fires, then adjudicate against pre-registered bars.
|
||||
python -m research.experiments run || echo "[scheduler] experiments run FAILED"
|
||||
python -m research.experiments adjudicate || echo "[scheduler] experiments adjudicate FAILED"
|
||||
sleep 120
|
||||
fi
|
||||
sleep 20
|
||||
done
|
||||
|
||||
db-backup:
|
||||
# Nightly pg_dump at 20:10 ET Mon-Fri (after the 20:00 scheduler pass).
|
||||
# Runs on the postgres:18 image — a version-matched pg_dump for free; the
|
||||
# app image has none. Backups land in the app-data volume under backups/.
|
||||
image: postgres:18
|
||||
container_name: momentum-db-backup
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# in-stack postgres, same value the app services use (NOT .env's
|
||||
# DATABASE_URL, which may point at the old LAN instance)
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-momentum}:${POSTGRES_PASSWORD:-momentum}@postgres:5432/${POSTGRES_DB:-momentum}
|
||||
BACKUP_DIR: /data/backups
|
||||
BACKUP_KEEP_DAYS: "14"
|
||||
TZ: America/New_York # the postgres image defaults to UTC; the 20:10 gate is ET
|
||||
volumes:
|
||||
- ./deploy/docker/backup_db.sh:/backup_db.sh:ro
|
||||
- app-data:/data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
echo "[backup] up; fires 20:10 ET Mon-Fri"
|
||||
while true; do
|
||||
if [ "$$(date +%H:%M)" = "20:10" ] && [ "$$(date +%u)" -le 5 ]; then
|
||||
bash /backup_db.sh || echo "[backup] FAILED"
|
||||
sleep 120
|
||||
fi
|
||||
sleep 20
|
||||
done
|
||||
|
||||
# ---- On-demand jobs: docker compose run --rm <name>
|
||||
journal:
|
||||
<<: *app
|
||||
profiles: ["jobs"]
|
||||
command: ["python", "momentum_cli.py", "journal"]
|
||||
environment: *app-env
|
||||
restart: "no"
|
||||
|
||||
eod-replay:
|
||||
<<: *app
|
||||
profiles: ["jobs"]
|
||||
command: ["python", "eod_replay.py"]
|
||||
environment: *app-env
|
||||
restart: "no"
|
||||
|
||||
nightly-tune:
|
||||
<<: *app
|
||||
profiles: ["jobs"]
|
||||
command: ["python", "nightly_tune.py"]
|
||||
environment: *app-env
|
||||
restart: "no"
|
||||
|
||||
# ---- Grafana (optional): docker compose --profile monitoring up -d
|
||||
grafana:
|
||||
image: grafana/grafana:11.2.0
|
||||
profiles: ["monitoring"]
|
||||
environment:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin"
|
||||
GF_AUTH_DISABLE_LOGIN_FORM: "true"
|
||||
GF_DASHBOARDS_MIN_REFRESH_INTERVAL: "2s"
|
||||
PG_USER: ${POSTGRES_USER:-momentum}
|
||||
PG_PASSWORD: ${POSTGRES_PASSWORD:-momentum}
|
||||
PG_DATABASE: ${POSTGRES_DB:-momentum}
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- ./deploy/compose/grafana/datasources:/etc/grafana/provisioning/datasources:ro
|
||||
- ./deploy/compose/grafana/dashboards:/etc/grafana/provisioning/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
pg-data:
|
||||
app-data:
|
||||
grafana-data:
|
||||
@@ -0,0 +1,110 @@
|
||||
# Live-Money Cutover Runbook
|
||||
|
||||
Staged 2026-07-22. Everything below is pre-built; flip day is configuration,
|
||||
not construction. Target: minimal downtime, zero blending of paper and live
|
||||
records.
|
||||
|
||||
## Architecture decision
|
||||
|
||||
Real money runs as a **second, fully isolated compose stack** ("momentum-live"):
|
||||
own Postgres, own volumes, own dashboard. Rationale: the ~30 dashboard/P&L
|
||||
projections do not filter by event mode — one missed filter would silently
|
||||
blend money records. A separate database makes blending impossible by
|
||||
construction. The paper stack stays untouched as the research substrate
|
||||
(nightly labeler, experiments, backtests, regime scoring).
|
||||
|
||||
## What is already staged
|
||||
|
||||
| Piece | Where | State |
|
||||
|---|---|---|
|
||||
| Live compose (paper keys, current settings) | `deploy/docker/truenas-compose.live.local.yaml` (LOCAL, secrets, never commit) | ready; flip checklist in its header |
|
||||
| DB seed script | `deploy/docker/bootstrap_live_db.sh` | ready |
|
||||
| Equity-floor kill switch | `TRADING_EQUITY_FLOOR` (commit 5287c8e) | in image, 0=off |
|
||||
| Boot order reconciliation | `TRADING_BOOT_RECONCILE` (default on, 5287c8e) | in image |
|
||||
| Stale-session dashboard alarm | commit 2e39b75/5f33799 | in image |
|
||||
|
||||
## Known dependencies (discovered during staging)
|
||||
|
||||
1. **One paper account cannot serve two stacks.** Both composes currently
|
||||
carry the same paper keys. Running both would double-trade one account AND
|
||||
each stack's boot-reconcile would cancel the other's resting orders as
|
||||
orphans. Before first boot: create a second Alpaca paper key pair for the
|
||||
live stack's shakedown (or pause the paper stack's loop during it).
|
||||
2. **Fresh DB must be seeded, not born empty.** Empty DB = rvol baselines
|
||||
missing (~1 day to self-heal) and, worse, the regime gate fail-opens for
|
||||
weeks (no premarket history) — the one validated protection would be inert
|
||||
exactly at go-live. `bootstrap_live_db.sh` seeds from the paper stack's
|
||||
nightly dump and truncates the money-record tables (`events`,
|
||||
`telemetry_events`, simulated/paper tables), so research history carries
|
||||
over while live P&L starts at zero. The dump also carries every index.
|
||||
3. **`data/learned_params.json`** — copy from the paper stack's data dir into
|
||||
`momentum-live/data/` (nightly tune runs only on the paper stack).
|
||||
4. **Telegram** — both stacks share one chat; create a second bot token for
|
||||
the live stack (`TELEGRAM_BOT_TOKEN` in its compose) or alerts interleave
|
||||
indistinguishably.
|
||||
5. Host prerequisites: dirs `/mnt/Volume/Storage/Apps/momentum-live/data/backups`
|
||||
created; host ports **8865** (dashboard) and **5434** (postgres) free.
|
||||
|
||||
## Config model (added 2026-07-22)
|
||||
|
||||
Strategy knobs live in the **`config_overrides` database table** (commit
|
||||
495a3fa), edited via the admin panel (paper :8767, live :8867) with a
|
||||
whitelist (`TRADING_*`, `TRIGGER_*`, `WATCHER_*`, `NEWS_*`, `OLLAMA_*`,
|
||||
`DISCOVER_*`; secrets refused). Boot precedence: compose env <
|
||||
`data/overrides.env` file < database — every service applies the table last;
|
||||
DB down falls through silently so boot never blocks on config. Compose and
|
||||
the overrides file retain only infra + secrets (keys, URLs, DB address — the
|
||||
DB address cannot live inside the DB). One-shot migration inside any stack
|
||||
container:
|
||||
|
||||
python -m runtime.env_overrides seed-from-env
|
||||
|
||||
copies that container's effective knobs into the table, after which the
|
||||
compose knob lines are dead weight (strip at any redeploy). For the live
|
||||
stack: run the seed after `bootstrap_live_db.sh` (the seeded table travels
|
||||
with the DB restore otherwise — which is fine too, since the paper dump's
|
||||
table carries the same knobs; just re-check flip-day values). Flip-day knob
|
||||
changes (risk, regime gate, equity floor) happen in the admin UI; only keys +
|
||||
ALPACA_BASE_URL remain compose edits (secret-marked, deliberately not
|
||||
UI-editable).
|
||||
|
||||
## Cutover procedure (~15 min, paper stack keeps running throughout)
|
||||
|
||||
1. `mkdir -p /mnt/Volume/Storage/Apps/momentum-live/data/backups`
|
||||
2. Add the TrueNAS app from `truenas-compose.live.local.yaml`; start ONLY the
|
||||
postgres service (or start all and let the loop crash-loop until step 3 —
|
||||
harmless, `restart: unless-stopped`).
|
||||
3. Seed: `./bootstrap_live_db.sh <latest paper backup> <live-pg-container>`
|
||||
(paper backups land nightly 20:10 ET in
|
||||
`/mnt/Volume/Storage/Apps/momentum/data/backups/`).
|
||||
4. `cp .../momentum/data/learned_params.json .../momentum-live/data/` and
|
||||
`cp deploy/docker/overrides.live.seed.env .../momentum-live/data/overrides.env`
|
||||
5. Shakedown on the SECOND paper key pair for 1-2 sessions: watch the 8865
|
||||
dashboard — armed triggers populate, entries fire, no stale-session banner,
|
||||
boot-reconcile quiet.
|
||||
6. **The flip** (edit live compose env, both loop + dashboard, then redeploy):
|
||||
- `ALPACA_BASE_URL: https://api.alpaca.markets`
|
||||
- `ALPACA_API_KEY` / `ALPACA_SECRET_KEY`: live keys
|
||||
- `TRADING_RISK_PER_TRADE_PCT: '0.0025'` (start tiny)
|
||||
- `TRADING_MAX_RISK_DOLLARS: '250'` (hard ceiling back on)
|
||||
- `TRADING_REGIME_GATE_ENABLED: '1'` (the one validated lever)
|
||||
- `TRADING_EQUITY_FLOOR:` 0.90 x starting equity (absolute dollars)
|
||||
7. Verify within the first hour: dashboard LIVE badge, one order's audit trail
|
||||
end-to-end, `python -m research.slippage --since <today>` after the first
|
||||
close, Telegram alert received.
|
||||
|
||||
## Rollback
|
||||
|
||||
Point the live compose back at paper keys/URL and redeploy — the stack itself
|
||||
is the same code either way. The paper stack was never touched. If the live DB
|
||||
is ever suspect, it is disposable pre-flip: destroy pgdata, re-run the seed.
|
||||
|
||||
## Notes
|
||||
|
||||
- FINRA PDT rule retired June 4 2026 (Alpaca Intraday Margin Framework) — no
|
||||
$25k floor; standard margin minimums apply.
|
||||
- Measured fill reality on gate-passing names: 0.3-0.5% median spread, ~12%
|
||||
profit drag vs paper; p90 tail 2.4-3.6% and book-walks are the real risks
|
||||
(`research/slippage.py` is the standing report).
|
||||
- Gate for the flip: 2-3 consecutive green regime-gated paper weeks on the
|
||||
fixed stack (week of 07-20: +$1,732, 8-for-8, regime OFF/watch-mode).
|
||||
@@ -0,0 +1,80 @@
|
||||
# Tradier broker fallback
|
||||
|
||||
Redundancy path if the Alpaca live account stays blocked. `tradier_broker/`
|
||||
mirrors every trading call the live loop makes on `AlpacaPaperClient` — same
|
||||
method names, signatures, and Alpaca-shaped return dicts — so
|
||||
`trading_execution`, `runtime/exit_manager`, `runtime/flatten`,
|
||||
`alpaca_paper.execution` and `alpaca_paper.sync` run unchanged.
|
||||
|
||||
## Switch (config only)
|
||||
|
||||
```
|
||||
BROKER=tradier
|
||||
TRADIER_ACCESS_TOKEN=... # sandbox or live token
|
||||
TRADIER_ACCOUNT_ID=... # e.g. VA71549758
|
||||
TRADIER_ENV=sandbox # 'live' for real money — explicit opt-in
|
||||
```
|
||||
|
||||
`BROKER` unset/`alpaca` = today's behavior, byte-for-byte. Alpaca keys stay
|
||||
required either way: **market data (bars, screeners, quotes, news) always
|
||||
stays on Alpaca** — the Tradier client only replaces order execution and
|
||||
account/position state, delegating all data calls to the Alpaca client.
|
||||
|
||||
## Verify before trusting
|
||||
|
||||
```
|
||||
python3 -m tradier_broker.smoke # balances/clock/positions/orders
|
||||
python3 -m tradier_broker.smoke --preview AAPL # broker-validates bracket form, never submits
|
||||
python3 -m tradier_broker.smoke --place-cancel AAPL # full submit -> tag lookup -> cancel
|
||||
```
|
||||
|
||||
Smoke refuses order exercises against the live base URL.
|
||||
|
||||
## Mapping notes
|
||||
|
||||
- Alpaca bracket -> Tradier OTOCO (entry + OCO take-profit limit / stop);
|
||||
oto -> OTO; standalone gtc stop sells map directly.
|
||||
- `client_order_id` -> Tradier `tag` (lost-submit reconcile works via tag lookup).
|
||||
- Tradier positions carry no mark; `get_positions` prices them off
|
||||
`/markets/quotes` (sandbox quotes are 15-min delayed — fine for redundancy
|
||||
checks, and live tokens get real-time).
|
||||
- `replace_order` maps to PUT (moves stop/limit without cancelling — OCO stays
|
||||
intact); Tradier cannot change qty on a working order (raises, cancel+resubmit).
|
||||
- `get_orders` is session-scoped at Tradier (today's orders) and status/symbol
|
||||
filters are client-side; the live loop only queries today + held symbols, so
|
||||
behavior matches.
|
||||
|
||||
## Parallel shakeout stack (TrueNAS)
|
||||
|
||||
`deploy/docker/truenas-compose.tradier.yaml` is the committed template; the
|
||||
filled copy lives (git-excluded) at `truenas-compose.tradier.local.yaml`.
|
||||
Runs beside the paper stack, fully isolated:
|
||||
|
||||
- own postgres (host **5434**, fresh `pgdata-tradier` volume), own dashboard
|
||||
(host **8768**), loop with `BROKER=tradier` against the sandbox account
|
||||
- market data still Alpaca; Telegram + deadman OFF (main stack keeps alerts)
|
||||
- main data dir mounted **read-only** — inherits nightly
|
||||
`learned_params.json` / `overrides.env`, cannot clobber them; if a service
|
||||
ever crashes on a read-only-filesystem error, switch that mount to a
|
||||
writable copy
|
||||
- fresh DB starts empty (knobs at code defaults). For research-baseline +
|
||||
knob parity with the paper stack, reuse the live-staging playbook:
|
||||
`./bootstrap_live_db.sh <nightly-backup.sql.gz> <tradier-postgres-container>`
|
||||
- fresh DB also lacks the research tables the dashboard polls
|
||||
(`experiment_verdicts` etc. — postgres logs `relation ... does not exist`
|
||||
on every poll). Harmless, but either bootstrap as above or create the
|
||||
missing tables schema-only; done by hand on the first deploy 2026-07-27
|
||||
(18 tables), plus a copy of the paper stack's `config_overrides` rows
|
||||
(75 knobs — without them the stack runs factory defaults and entries
|
||||
pile up in the manual approval queue)
|
||||
- needs an image built from >= 1cefb51 (`tradier_broker/` in image)
|
||||
- sandbox fills are simulated and can be slow or absent — an order parked in
|
||||
`pending_new` is usually the sandbox, not the code
|
||||
|
||||
## Before flipping live money to Tradier
|
||||
|
||||
1. Rotate the sandbox token (it transited chat) and mint a live token.
|
||||
2. Run a full paper week on sandbox with `BROKER=tradier` in the paper stack.
|
||||
3. Re-run the naked-stop drill from docs/LIVE-CUTOVER.md — leg-stripping
|
||||
semantics differ per broker; verify a filled OTOCO's stop leg survives an
|
||||
entry-order cancel attempt.
|
||||
@@ -0,0 +1,61 @@
|
||||
# Tradier Real-Money Flip — delta to LIVE-CUTOVER.md
|
||||
|
||||
LIVE-CUTOVER.md was written for Alpaca live (rejected 2026-07-27). Plan of
|
||||
record is now Tradier real money on the EXISTING tradier parallel stack
|
||||
(postgres 5434, dashboard 8768). This doc is only the delta.
|
||||
|
||||
## The flip (3 env lines in truenas-compose.tradier.local.yaml, loop + dashboard)
|
||||
|
||||
TRADIER_ENV: live # switches base URL to api.tradier.com
|
||||
TRADIER_ACCESS_TOKEN: <live token> # from the REAL account, never the sandbox token
|
||||
TRADIER_ACCOUNT_ID: <live account id>
|
||||
|
||||
Market data stays on Alpaca (unchanged). Rollback = set `TRADIER_ENV: sandbox`
|
||||
+ sandbox creds, redeploy.
|
||||
|
||||
## Before the flip — owner tasks (nobody else can do these)
|
||||
|
||||
1. Open a Tradier **cash** account (margin needs $2k; $300 = cash), fund $300.
|
||||
2. Create an API token on the live account. Treat as secret: compose file
|
||||
only, never the repo, never the config_overrides table.
|
||||
3. Confirm account approvals: API access enabled on the live account
|
||||
(Tradier sometimes gates API on request).
|
||||
|
||||
## Before the flip — config (admin UI :8768 or config_overrides on 5434)
|
||||
|
||||
| Knob | Value | Why |
|
||||
|---|---|---|
|
||||
| `TRADING_MAX_FRESH_ENTRIES_PER_DAY` | `1` | **Cash account T+1: >1 round trip/day risks good-faith violations; 3 GFVs = 90-day freeze.** Raise only with equity growth or margin. |
|
||||
| `TRADING_EQUITY_FLOOR` | `270` | Kill switch at 0.90 x $300. |
|
||||
| `TRADING_MAX_RISK_DOLLARS` | `1000` (keep) | Inert at this equity (16% notional cap = ~$48 positions binds first); harmless. |
|
||||
| `TRADING_RISK_PER_TRADE_PCT` | `0.05` (keep) | $15 budget; the $48 position cap means actual risk/trade lands ~$2-8. |
|
||||
| `TRADING_REGIME_GATE_ENABLED` | `1` | The one lever that survived OOS validation (plan 011); flip-day requirement per LIVE-CUTOVER. |
|
||||
|
||||
## Before the flip — DB hygiene
|
||||
|
||||
The tradier DB currently holds SANDBOX trade records. Live must start clean
|
||||
(cutover principle: no blending). Before first live boot:
|
||||
|
||||
TRUNCATE events, telemetry_events; -- on 5434; keep research tables
|
||||
|
||||
(Same pattern as bootstrap_live_db.sh. minute_bars/daily_bars/volume_ratios
|
||||
stay — that history is real market data and feeds rvol baselines.)
|
||||
|
||||
## Day-1 verification (first hour)
|
||||
|
||||
1. Dashboard badge shows tradier-LIVE (factory label), not sandbox.
|
||||
2. First fill: check the bracket legs exist at Tradier (stop visible via
|
||||
by-symbol order lookup) — the 2026-07-28 incident class.
|
||||
3. **Naked-stop drill (memory: required pre-flip, still undone):** with a
|
||||
1-share position open, manually cancel its stop leg in the Tradier UI;
|
||||
the guard must force-exit within 2 passes. This is the drill on real
|
||||
plumbing with pocket-change size.
|
||||
4. Telegram alert received; `python -m research.slippage --since <today>`
|
||||
after close — real-fill slippage vs the 0.3-0.5% paper-measured spread.
|
||||
|
||||
## Expectations at $300
|
||||
|
||||
Positions ~$48 (16% cap), 1-20 shares on $2-20 names. P&L per trade: single
|
||||
dollars. This proves execution (fills, brackets, stops, GFV discipline), not
|
||||
income. Sizing scales automatically as equity grows — the knobs above are the
|
||||
only things to revisit.
|
||||
@@ -0,0 +1,106 @@
|
||||
# UI Review — Momentum Dashboard (`api/static/dashboard.html`)
|
||||
|
||||
**Audited**: 2026-07-20 · retroactive 6-pillar visual audit (no UI-SPEC exists →
|
||||
audited against abstract standards). Adversarial stance: assume failure until
|
||||
the code proves otherwise. Scores are 1–4, not averaged upward.
|
||||
|
||||
**Scope**: the primary operator dashboard (`dashboard.html`, ~1600 lines,
|
||||
single-file inline CSS/JS) plus its siblings `backtest.html` / `admin.html`.
|
||||
Method: static analysis of color/spacing/typography tokens, structure, and
|
||||
interaction states (the auditor's grep-based method; a headless render pass is
|
||||
the standing follow-up).
|
||||
|
||||
## Scorecard
|
||||
|
||||
| Pillar | Score | One-line verdict |
|
||||
|---|---|---|
|
||||
| 1. Copywriting | **4** | Domain-precise, explains *why* — not generic |
|
||||
| 2. Visuals / hierarchy | **3** | Clear strip→panel structure; dense but legible |
|
||||
| 3. Color | **3** | Real token system, undermined by duplicate semantic colors + 90 hardcoded values |
|
||||
| 4. Typography | **3** | Sensible mono scale; 9px floor is too small |
|
||||
| 5. Spacing | **2** | No systematic scale — 20 distinct px values incl. arbitrary 7/9/11/13 |
|
||||
| 6. Experience design | **4** | Standout — confirm-to-destruct, full empty/loading/error/connection states |
|
||||
|
||||
**Overall**: a genuinely competent, purpose-built operator UI. The failures are
|
||||
consistency-and-accessibility drift, not structural — this is a good dashboard
|
||||
with three fixable rough edges, not a rebuild.
|
||||
|
||||
## Findings
|
||||
|
||||
### Pillar 1 — Copywriting · 4
|
||||
Labels state intent, not just nouns: "closest to fire on top", "why each entry
|
||||
was taken or refused · value → threshold per gate", "held long with no / short
|
||||
protective stop". Empty states teach ("No armed gappers — waiting for the open").
|
||||
No lorem, no vague "Loading…" without context. **No fix needed.**
|
||||
|
||||
### Pillar 2 — Visuals / hierarchy · 3
|
||||
Strong top-down hierarchy: risk-strip → P&L-strip → collapsible `<details>`
|
||||
panels, colour-coded side pills, sparklines, R-bars. Weakness: extreme density
|
||||
(10–11px body across 51 elements) with little whitespace — correct for a trading
|
||||
terminal, but the eye has few rest points. **WARNING** — acceptable for the
|
||||
audience; revisit if non-expert users are ever a target.
|
||||
|
||||
### Pillar 3 — Color · 3
|
||||
Real foundation: 17 CSS custom properties, 149 `var(--…)` uses, coherent dark
|
||||
theme, defensible 60/30/10 (bg dominant / panels secondary / `#5ba3ff` accent on
|
||||
interactive). **But** two concrete defects:
|
||||
- **Duplicate semantic colors** — THREE greens (`#2fbf71`, `#2ecc71`, `#34d17e`)
|
||||
and FOUR reds (`#e5484d`, `#ff4d4f`, `#ff5a60`, `#ff8589`) all meaning
|
||||
"up/good" and "down/bad". These should be one `--green` / `--red` (+ one
|
||||
intentional bright variant), not near-identical one-offs.
|
||||
- **90 hardcoded hex values** bypass the token system that already exists.
|
||||
**WARNING** — the system is right; application drifted.
|
||||
|
||||
### Pillar 4 — Typography · 3
|
||||
One monospace family (correct for aligned numeric data), a real size ladder
|
||||
(9→27px), weight used for hierarchy. Defect: the **9px floor** (4 elements) is
|
||||
below comfortable-read and fails WCAG minimums; 10–11px carries most of the UI,
|
||||
which is tight. **WARNING** — raise the 9px labels to ≥11px.
|
||||
|
||||
### Pillar 5 — Spacing · 2 (lowest)
|
||||
**20 distinct padding/margin values** including arbitrary `7px`, `9px`, `11px`,
|
||||
`13px` with no discernible 4/8 rhythm. Dense layout justifies *tightness*, not
|
||||
*arbitrariness* — the values look hand-tuned per element rather than drawn from a
|
||||
scale. **WARNING** — adopt a spacing scale (e.g. 2/4/8/12/16/24) and snap values
|
||||
to it; the visual result changes little but future edits stay consistent.
|
||||
|
||||
### Pillar 6 — Experience design · 4 (standout)
|
||||
This is where the dashboard earns its keep:
|
||||
- **Confirm-to-destruct**: EXIT/FLATTEN/PAUSE use an arm-then-confirm pattern
|
||||
(`armButton`, 8 sites) — no accidental market flatten.
|
||||
- **Full state coverage**: 38 loading/loader refs, 20 explicit empty states, 7
|
||||
error/connection-lost handlers ("stream lost — polling", "render error").
|
||||
- **Resilience**: SSE with polling fallback, degrade-gracefully panels.
|
||||
- **3 `@media` breakpoints** — responsive to phone/tablet.
|
||||
**No fix needed** — this pillar is a model for the rest.
|
||||
|
||||
### Cross-cutting — Accessibility (drags 2 & 6)
|
||||
**0 ARIA roles/labels, only 2 `:focus` styles** on a dashboard whose buttons
|
||||
submit real orders (Approve / Reject / EXIT / FLATTEN). Keyboard and
|
||||
screen-reader users cannot see focus or understand controls. Not a visual defect
|
||||
but a real usability gap for the money-path controls. **WARNING (borderline
|
||||
BLOCKER for keyboard operators).**
|
||||
|
||||
## Top 3 priority fixes
|
||||
|
||||
1. **Collapse duplicate semantic colors + migrate hardcoded values** (Pillar 3).
|
||||
Define `--green`/`--red` (+ one bright variant each), replace the 3 greens / 4
|
||||
reds and the 90 hardcoded hex. Highest consistency-per-effort. *WARNING.*
|
||||
2. **Add focus + ARIA to money-path controls** (Accessibility). `:focus-visible`
|
||||
outlines and `aria-label` on Approve/Reject/EXIT/FLATTEN. A keyboard operator
|
||||
currently flies blind on the destructive buttons. *WARNING → BLOCKER for
|
||||
keyboard use.*
|
||||
3. **Adopt a spacing scale** (Pillar 5). Snap the 20 ad-hoc px values to a
|
||||
2/4/8/12/16/24 ladder; raise the 9px type floor to ≥11px. *WARNING.*
|
||||
|
||||
## What NOT to touch
|
||||
Experience design (Pillar 6) and copywriting (Pillar 1) are strengths — don't
|
||||
regress the confirm-to-destruct flow or the explanatory empty states while
|
||||
refactoring color/spacing. The density is a deliberate, correct choice for the
|
||||
trading-terminal audience; do not "add whitespace" reflexively.
|
||||
|
||||
---
|
||||
*No UI-SPEC exists for this project. If the dashboard becomes a maintained
|
||||
surface, the highest-leverage next step is extracting the existing `:root` tokens
|
||||
into a small declared design contract so future changes have a baseline to audit
|
||||
against — the token system is 80% there already.*
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,177 @@
|
||||
# Momentum consolidation plan
|
||||
|
||||
_Scope-only plan produced 2026-06-23 by a read-only 4-agent mapping workflow (wf_3fb5d523). No code was changed. Items are for operator approval; execute in P0→P2 order._
|
||||
|
||||
## TL;DR
|
||||
|
||||
The single-Postgres migration is functionally COMPLETE at the connection layer (all live code flows through storage/db.py -> db_pg -> one DATABASE_URL Postgres; no live duckdb.connect exists), so the datastore work is cleanup, not architecture. The genuine open architectural item is the two entry chokepoints — but the maps are slightly stale in our favor: the VWAP gate (commit e3ac89c) and daily-entry cap (ac1ac50) are now ALSO on the slow/auto path, so the remaining fast-only gaps are narrower than stated: extension, day-extension, halt, liquidity sizing, and rank-concentration. The whole entry-path effort hinges on one unverified empirical claim — whether live fills originate from the fast/trigger path (which has all gates) or the slow auto path (which lacks the five above) — so the plan opens by PROVING that via the existing approved_by tag ('auto' vs 'orb_trigger'), since it determines whether those gaps are live or theoretical. Sequencing favors reversibility: P0 = the empirical fill-source check plus the zero-risk, zero-blast-radius wins (delete the untracked gitignored archive/ tree and 232MB zip; drop the orphaned mem_ test schema confirming the known PG-isolation landmine and close its recurrence). P1 = behaviour-neutral cleanups (delete the dead core/ and telemetry/ packages; remove vestigial .duckdb path-strings so a file backend can't be reintroduced; archive the two frozen pre-cutover .duckdb files AFTER a pg_dump; extract the fast path into a single _place_entry helper as a pure refactor). P2 = the higher-blast changes done last and behind a flag (enrich the signal_ready snapshot with live price/day_open/volume/rank — the structural root cause of the dead slow-path gates; route approve_order through the unified helper and choose ONE fill model; the Schwab delete-vs-repair product decision; and verifying the symbols/telemetry_events empty tables aren't silently-dead writers). Throughout, the non-negotiables are: keep every shared-state mutation (self._fresh_entries, self._armed, self._requested_symbols, self._exited_today, self._cooldown_until) inside @_locked methods, never touch the naked-stop protections in expire_stale_entries, preserve the auto-vs-manual approval exemption, and never block an entry on missing data (fail-open like the existing above_vwap handling). Key files: trading_execution.py (entry chokepoints + shared state), run_live_paper.py (fast-path call site + boot warning), storage/projections.py (ready-signal snapshot), research/schema.py + research/multi_schema.py (vestigial duckdb labels), storage/db_pg.py + tests/conftest.py (mem_ schema isolation), storage/pg_schema.sql (orphaned tables).
|
||||
|
||||
|
||||
> **Safety note (answered):** the P0 'which path produces fills' question is already settled — on 2026-06-23 **47/47 live entries were `approved_by="auto"` (the slow path)**. So the fast-path-only gates the map flags as missing on the live path — **extension / day-extension / halt-guard / liquidity-sizing / rank** — are a REAL current gap, not theoretical. (VWAP gate + daily-cap are now on both paths as of today's commits.)
|
||||
|
||||
|
||||
## Prioritized plan
|
||||
|
||||
### P0 — Confirm empirically which entry path produces live fills (instrumentation only, no logic change)
|
||||
- **blast radius:** low | **effort:** small
|
||||
- **action:** Before touching any gate, tag every fill with its origin: the slow/auto path already submits approve_order(approved_by='auto') (trading_execution.py:1037) and the fast path submits approve_order(approved_by='orb_trigger') (:970). Add a single grep/SQL over the live events table (events filtered to order-submit / approval events) for the last N trading days, bucketed by approved_by, to prove whether real fills originate from 'auto' (slow) or 'orb_trigger' (fast). Record the result in the plan ticket. Do NOT change behaviour.
|
||||
- **why:** The entire safety argument for the entry-path work hinges on an empirical claim the maps flag as unverified: 'whether live entries are protected depends on whether fills originate from the trigger thread.' If fills already flow through the fast/trigger path, the extension/halt/liquidity/rank gaps are theoretical and the high-risk unification can be deprioritized; if they flow through the slow auto path, those gaps are live and urgent. This one read-only check decides the priority of every downstream entry-path step and costs nothing.
|
||||
|
||||
### P0 — Delete untracked, gitignored clutter: archive/ tree and momentum-rebuilt.zip
|
||||
- **blast radius:** low | **effort:** small
|
||||
- **action:** Delete /mnt/c/code/momentum/archive/ (6641 .py files, gitignored per .gitignore:59, git ls-files archive/ = 0) and /mnt/c/code/momentum/momentum-rebuilt.zip (232 MB, gitignored). Both are local-only and not recoverable from git, so first confirm with the operator that nothing local is needed, then rm. Neither is referenced by any live import (grep for 'from archive'/'import archive' = 0).
|
||||
- **why:** Highest noise-reduction per unit risk. archive/ is a full duplicate codebase that inflates every repo-wide grep and is the single biggest source of troubleshooting noise; the 232 MB zip is pure disk clutter. Neither is in version control and nothing depends on them, so deletion cannot break the running system. This makes all subsequent investigation cleaner.
|
||||
|
||||
### P0 — Drop the orphaned mem_44381ad9a97c test schema from production Postgres and verify test isolation holds
|
||||
- **blast radius:** low | **effort:** small
|
||||
- **action:** Run DROP SCHEMA IF EXISTS mem_44381ad9a97c CASCADE on 192.168.1.5/momentum. This is a throwaway test-isolation schema (db_pg.py:203 creates mem_<uuid> for ':memory:' connections) left undropped — evidence that a test/script touched production PG. The auto-cleanup already exists: db_pg.py:165 drops the temp schema on close(), and tests/conftest.py:16 sweeps leftover mem_% schemas on teardown — so a single orphan implies a crashed/killed run, not a missing guard. After dropping, add a recurring sweep (the conftest query at conftest.py:33 reused as a doctor-CLI check) and confirm CI/tests set MOMENTUM_PG_SCHEMA or a non-prod DATABASE_URL so a missed cleanup can never write to the live events/bars tables.
|
||||
- **why:** This is the only finding in the maps rated medium risk in the datastore area, and it is the concrete confirmation of the known PG-isolation landmine: test code has at least once written to the production DB. Dropping the orphan is trivial and reversible; the real value is closing the recurrence path so a future un-isolated test cannot corrupt the 676k-row live events table. Low blast radius because dropping an unreferenced empty mem_ schema cannot affect the app's public-schema tables.
|
||||
|
||||
### P1 — Delete the dead core/ package (contracts.py, telemetry.py, validation.py, empty __init__.py) and the dead telemetry/ package
|
||||
- **blast radius:** low | **effort:** small
|
||||
- **action:** Delete /mnt/c/code/momentum/core/ (telemetry.py, validation.py, contracts.py, empty __init__.py) and /mnt/c/code/momentum/telemetry/ (events.py, empty __init__.py). Verified: repo-wide grep (incl run_live_paper.py and all tests) for 'import telemetry'/'from telemetry'/'core.telemetry'/'core.validation'/'core.contracts' returns ZERO live callers; contracts.py's only importers are telemetry.py and validation.py, which are themselves dead. The live loop uses storage/event_store + storage/event_schema (run_live_paper.py:66-67), a parallel superseding implementation. Do this as its own commit so it is trivially revertable. Optionally drop the orphaned telemetry_events table from storage/pg_schema.sql:415 (the dead telemetry/events.py was its only writer; the table has 0 rows) in a SEPARATE follow-up after the code delete is confirmed safe.
|
||||
- **why:** Pure dead-by-disuse code (imports cleanly but nobody calls it), so removing it cannot change runtime behaviour. It removes two competing telemetry implementations that are a constant source of 'which telemetry is real?' confusion during troubleshooting. Keep the code delete and the schema drop as separate steps so the reversible code change ships first and the schema change (slightly higher blast radius) is isolated.
|
||||
|
||||
### P1 — Collapse the vestigial DuckDB path-strings and logical-DB labels so a file backend can't be reintroduced
|
||||
- **blast radius:** low | **effort:** medium
|
||||
- **action:** In research/schema.py remove the literal DEFAULT_RESEARCH_DB='data/research/market.duckdb' (line 9) used as a default arg in get_research_connection (line 12) — it is never used as a real path because get_connection ignores db_path. In research/multi_schema.py collapse the DB_FILES map (line 19) and open_research_db (line 32)/research_db_path so callers route directly to storage.db.get_connection() with no '.duckdb' filename. Keep the function names/signatures stable (callers like run_live_paper.py:150-152 pass 'market'/'momentum') so this is a no-op at the call sites — only the dead path-string plumbing is removed. Confirm no duckdb.connect anywhere in live code first (already verified empty outside archive/tests).
|
||||
- **why:** Every datastore access already flows through storage/db.py -> db_pg -> the single Postgres; the 'market'/'news'/'signals' labels and '.duckdb' strings are vestigial and actively misleading (they imply a file backend that no longer exists). Removing the literals prevents someone from 'fixing' them back into a real file path and re-fragmenting the datastore. Low blast radius because the strings are inert today — but test thoroughly since these helpers are on the boot path (research_con in run_live_paper.py).
|
||||
|
||||
### P1 — Archive the two frozen live-dir .duckdb files after confirming Postgres covers their history
|
||||
- **blast radius:** low | **effort:** medium
|
||||
- **action:** data/momentum.duckdb (5.0MB, the pre-migration event store) and data/research/market.duckdb (5.5MB, pre-migration bars), both frozen at mtime 2026-06-16 00:24 (the cutover) and opened by no live code. Step 1: verify PG coverage — confirm PG events (676k rows) includes the date range of the duckdb events table, and PG daily_bars (38k)/minute_bars (1.46M) cover market.duckdb's range, by comparing min/max timestamps. Step 2: take a pg_dump of the relevant tables as the durable backup. Step 3: move both files to data/_archive/ (do NOT delete yet — they are the only on-disk copy of pre-2026-06-16 data). Delete only after the pg_dump exists and coverage is confirmed.
|
||||
- **why:** These are genuinely orphaned (no code path opens them post-cutover), so moving them cannot affect the running system. But they are the sole on-disk copy of pre-migration history, so the order matters: prove PG coverage and create a dump BEFORE removing them. Archiving rather than deleting keeps the step reversible. Lower urgency than the noise-clutter deletes because they are small and harmless where they sit.
|
||||
|
||||
### P1 — Refactor the fast path into a single internal _place_entry() helper (pure refactor, behaviour-preserving)
|
||||
- **blast radius:** medium | **effort:** medium
|
||||
- **action:** Extract a private _place_entry(symbol, entry_ref, trigger, stop, ctx) from the body of submit_breakout_now (trading_execution.py:868-1005), where ctx carries day_open, cum_volume, rank, halted, above_vwap, last_price. _place_entry runs the full fast-path gate set (dilution, geometry, extension :887-896, day-extension, halt :899-900, VWAP, rank :917/:922, liquidity sizing :903-904, gross-notional) and the shared post-submit arming. submit_breakout_now becomes a thin wrapper that builds ctx from its args and calls it. CRITICAL: keep all mutations of shared state (self._fresh_entries, self._requested_symbols, self._armed, self._exited_today, self._cooldown_until) inside @_locked methods under self._lock; do not touch expire_stale_entries (:1269) or the naked-stop grace/broker_held protections. Cover with existing fast-path tests; assert no behaviour change.
|
||||
- **why:** This is STEP 1 of the entry-unification and the prerequisite for closing the gate gaps, but on its own it changes NOTHING at runtime — it just relocates code into a callable helper. Doing it as an isolated, test-covered refactor de-risks the higher-blast-radius steps that follow (routing the auto path through the same helper). Marked P1 not P0 because P0 instrumentation must first confirm whether the slow path even produces live fills; if it doesn't, the gate gaps are not live.
|
||||
|
||||
### P2 — Enrich signal_ready / query_ready_signals_snapshot with trigger, day_open, cum_volume, rank, last_price
|
||||
- **blast radius:** medium | **effort:** large
|
||||
- **action:** Add trigger_price, day_open, cum_volume, rank, and a live/last price to the signal_ready event payload and to storage/projections.py query_ready_signals_snapshot (:223-233). The signal producer (watcher) must populate the new fields; until it does, the snapshot returns None and downstream gates must fail-open (never block on missing data, matching the existing above_vwap None-handling at trading_execution.py:660). This is the structural root cause of the dead slow-path gates: the slow snapshot has no live price/day_open/volume/rank, so extension/halt/liquidity/rank CANNOT be computed there regardless of code.
|
||||
- **why:** Without this, the auto/slow path is structurally incapable of running the anti-chase, halt, liquidity, and rank gates — so this is the true root cause of the dead gates, not the missing helper. It is P2 (not earlier) because it requires changing the signal producer and the projection contract, which is more invasive and must be validated end-to-end; and because if P0 shows live fills already flow through the fast/trigger path (which already has all this context), this enrichment is lower urgency. Risk is in the producer change, not the read side.
|
||||
|
||||
### P2 — Route approve_order through the unified _place_entry and pick ONE fill model behind a flag
|
||||
- **blast radius:** high | **effort:** large
|
||||
- **action:** After the snapshot is enriched (prior step), make approve_order (trading_execution.py:740) build a ctx from the enriched approval payload and call the SAME _place_entry instead of its inline executor.execute, so both paths share one gate set. PRESERVE the auto-vs-manual distinction: manual dashboard approvals (approved_by!='auto') intentionally bypass the daily cap (:754) and may bypass other gates — keep that exemption explicit in the helper. Then decide the single fill model (resting limit at signal entry, slow :713, vs marketable limit a hair above trigger, fast :929-931) and gate the change behind an env flag so it can be A/B'd against current behaviour and rolled back. Only after production logs confirm fills flow exclusively through _place_entry, delete the divergent inline blocks and the boot WARNING (run_live_paper.py:812-815). Note: VWAP gate (e3ac89c) and daily-cap (ac1ac50) are ALREADY on the auto path, so those two no longer need plumbing — the remaining gaps are extension, day-extension, halt, liquidity sizing, and rank.
|
||||
- **why:** This is the genuine architectural win — one entry chokepoint with one gate set — but it is the highest-risk change in the plan: it alters how live orders are placed and changes fill economics (resting vs marketable limit). It must come last, behind a flag, after the helper exists and the snapshot carries the needed context, and after P0 has shown where fills actually originate. The flag + A/B + 'delete only after production confirmation' sequencing is what keeps 'don't break what we have' true. Honest risk: get the shared-state locking or the manual-override exemption wrong and you either double-count the daily cap or strip a protection.
|
||||
|
||||
### P2 — Decide the fate of the dead Schwab integration (delete vs. repair) as an explicit product call
|
||||
- **blast radius:** medium | **effort:** medium
|
||||
- **action:** Present the operator a binary decision on schwab/ (23 files, ~1121 LOC) + milestone3_verify.py + tests/unit/schwab/ + the schwab_live Literal branch in trading_mode.py:8,21 + README/RUNBOOK Schwab rows. Facts: it is read-only (no order-placement code anywhere in schwab/orders or schwab/positions), it is isolated from the live loop (run_live_paper.py has zero schwab refs), and its credentials never load due to a confirmed env-var mismatch (schwab/settings.py:25-28 reads SCHWAB_MARKET_DATA_APP_KEY/SCHWAB_BROKER_APP_KEY with no fallback, but .env supplies SCHWAB_Market_APP_KEY/SCHWAB_Trading_APP_KEY). Option A (delete): remove all the above as one revertable commit. Option B (keep as future path): fix the env-var names and add a real order executor before it counts as anything. Recommend A unless Schwab is a committed roadmap item.
|
||||
- **why:** Schwab is the largest single block of troubleshooting noise after archive/, and it cannot trade (no executor, broken creds), so it is functionally dead today. But unlike the core/telemetry deletes it has a test file and is documented in README/RUNBOOK, so deletion is a product decision rather than mechanical cleanup — hence medium blast radius and P2. The cost of leaving it is ongoing confusion ('do we support Schwab?'); the cost of deleting wrongly is re-doing 1.1k LOC if Schwab is genuinely planned. Surface it as a decision rather than silently removing.
|
||||
|
||||
### P2 — Verify the two unpopulated PG tables (symbols=0, telemetry_events=0) are intentionally empty, not silently-dead writers
|
||||
- **blast radius:** low | **effort:** small
|
||||
- **action:** For storage/pg_schema.sql tables with 0 rows: telemetry_events is explained (its only writer is the dead telemetry/events.py being removed in the P1 dead-code step — drop the table there). For symbols=0, grep for any INSERT/writer against the symbols table; if a live code path is supposed to populate it (e.g. a symbol-universe loader) and isn't, that is a silently-dead writer to fix; if nothing should write it, drop the table from pg_schema.sql. Report which case applies; do not drop until the writer status is confirmed.
|
||||
- **why:** An empty table can be benign (reserved) or a symptom of a dead writer that is silently failing — the same failure class the observability overhaul exists to catch. Cheap to investigate (one grep + one row count) and it either closes a loose end or surfaces a real bug. Low blast radius and low urgency, so P2; bundle the telemetry_events drop with the P1 dead-code removal since they share a cause.
|
||||
|
||||
|
||||
## Appendix — Datastores map
|
||||
|
||||
- **[live / low risk] PostgreSQL (single canonical datastore)**
|
||||
- where: 192.168.1.5:5432/momentum via DATABASE_URL in /mnt/c/code/momentum/.env; connection layer /mnt/c/code/momentum/storage/db_pg.py; schema /mnt/c/code/momentum/storage/pg_schema.sql (24 public tables)
|
||||
- recommend: This IS the single-Postgres target; the migration is effectively done at the connection layer. Keep it. Note symbols=0 and telemetry_events=0 rows (tables exist but unpopulated) — verify those are intentionally empty vs a silently-dead writer.
|
||||
- **[live / low risk] get_connection / EventStore('momentum') / open_research_db('market') unification**
|
||||
- where: /mnt/c/code/momentum/storage/db.py -> db_pg.get_connection; /mnt/c/code/momentum/research/multi_schema.py (DB_FILES market/news/signals); /mnt/c/code/momentum/research/schema.py (DEFAULT_RESEARCH_DB)
|
||||
- recommend: Logical namespaces (market/news/signals) and the 'momentum' db_path are vestigial labels that all collapse to one DB — harmless but misleading. Optional cleanup: collapse open_research_db/get_research_connection to a single get_connection() call and drop the DB_FILES/DEFAULT_RESEARCH_DB '.duckdb' path strings so no one re-introduces a file backend.
|
||||
- **[dead / low risk] data/momentum.duckdb (root events DB)**
|
||||
- where: /mnt/c/code/momentum/data/momentum.duckdb
|
||||
- recommend: Orphaned leftover. Move to data/_archive/ or delete after confirming PG events table contains its history (PG events already has 676k rows). Keep one backup copy until a PG dump exists, since it is the only on-disk copy of pre-2026-06-16 events.
|
||||
- **[dead / low risk] data/research/market.duckdb (research/bars DB)**
|
||||
- where: /mnt/c/code/momentum/data/research/market.duckdb
|
||||
- recommend: Orphaned. Same disposition as root duckdb: archive/delete after confirming PG daily_bars+minute_bars cover its date range. Remove the literal '.duckdb' default in research/schema.py to prevent confusion.
|
||||
- **[dead / low risk] archive/data/*.duckdb (legacy multi-file research split)**
|
||||
- where: /mnt/c/code/momentum/archive/data/ (momentum.duckdb, momentum_events.duckdb, research/{market,news,signals,sec,rss,economic,evaluations,log}.duckdb, several test_*_events.duckdb, research.duckdb.wal)
|
||||
- recommend: Pure historical leftover. Safe to delete wholesale (or keep the archive/ dir as the migration snapshot). Not part of any single-Postgres step.
|
||||
- **[live / low risk] data/learned_params.json (nightly-tuner params cache)**
|
||||
- where: /mnt/c/code/momentum/data/learned_params.json
|
||||
- recommend: Legitimately a flat-file param cache, fine to leave as JSON (it is config, not transactional data). If you want truly single-Postgres, you could store it in a params table, but the file is simpler and the risk of putting boot-critical params behind a network DB (loop can't start if PG is down) argues for keeping it on local disk.
|
||||
- **[live / low risk] news / catalyst caches (now PG tables, not files)**
|
||||
- where: PG tables news_catalyst_cache, raw_news_items, raw_fetch_attempts, news_events (in /mnt/c/code/momentum/storage/pg_schema.sql); writers research/ingestion/news_enrichment.py; readers research/labeler.py, storage/projections.py
|
||||
- recommend: Already single-Postgres — nothing to migrate. No action.
|
||||
- **[live / low risk] Grafana datasource (read-only consumer)**
|
||||
- where: /mnt/c/code/momentum/grafana/provisioning/datasources/momentum.yaml + dashboards/momentum.json
|
||||
- recommend: Correct as-is. This is the intended end-state (Grafana + doctor CLI over one PG). No action.
|
||||
- **[dead / medium risk] Leftover mem_* test schema in production PG**
|
||||
- where: schema mem_44381ad9a97c inside the live 192.168.1.5/momentum database
|
||||
- recommend: Drop it: DROP SCHEMA mem_44381ad9a97c CASCADE. Then guard against recurrence — ensure tests set MOMENTUM_PG_SCHEMA or use a dedicated non-prod DATABASE_URL so a missed cleanup can never pollute production. Periodically sweep for mem_* schemas (the conftest already attempts this on teardown).
|
||||
|
||||
_notes:_ Concrete single-Postgres status: the migration is functionally COMPLETE at the connection layer. Every datastore access in live code (run_live_paper.py, trading_execution.py, runtime/triggers, research ingestion, projections, dashboard api, telegram_bot) flows through storage/db.py -> storage/db_pg.py -> psycopg2 to the one DATABASE_URL Postgres. No live code calls duckdb.connect (verified by grep excluding archive+tests). The remaining 'work' is cleanup, not architecture: (1) delete/archive the two orphaned live-dir .duckdb files (data/momentum.duckdb, data/research/market.duckdb, both frozen 2026-06-16 at cutover) and the archive/data/*.duckdb legacy split; (2) remove the vestigial '.duckdb' path-string defaults in research/schema.py and the logical DB_FILES map in research/multi_schema.py so no one re-introduces a file backend; (3) DROP the orphaned mem_* test schema in prod and harden test isolation. Two flat files remain on-disk by design and are reasonable to keep: data/learned_params.json (boot params; local-disk so the loop can start even if PG is down) and the data/*.log files (live_run.log, trade_journal.log via journal.sh, dashboard.log — plain text logs, not a datastore). Risk note: making params/boot depend on the remote LAN Postgres would couple loop startup to network availability; the current split (transactional data in PG, a tiny boot-config JSON on disk) is defensible. The single real hazard found is the leftover mem_ schema confirming test code has at least once touched production PG.
|
||||
|
||||
## Appendix — Entry-paths map
|
||||
|
||||
- **[duplicate / high risk] Two entry chokepoints; live entries use the SLOW path (approve_order). FAST path (submit_breakout_now) is the trigger-thread counterpart carrying the richer anti-chase gates**
|
||||
- where: trading_execution.py:607 request_approvals_for_ready_signals, :740 approve_order, :1031 process_auto_approvals (SLOW/LIVE) vs :809 submit_breakout_now (FAST)
|
||||
- recommend: Pick ONE submit chokepoint. Keep the trigger thread as signal source (it has live price/day_open/rank/cum_volume) but route both paths through a single internal _place_entry(symbol, entry_ref, trigger, stop, ctx) helper so gates live in one function. Verify in production logs which path actually produces fills before deleting either.
|
||||
- **[live / low risk] Daily fresh-entry cap (max_fresh_entries_per_day=5)**
|
||||
- where: FAST trading_execution.py:844-852; SLOW :754-761; increments :793-799 and :989-991
|
||||
- recommend: Already unified via shared state. Use as the model: a unified helper should keep the auto-vs-manual distinction.
|
||||
- **[dead / medium risk] Rank-concentration / rank risk-haircut (concentrate_top_n, rank_risk_factor)**
|
||||
- where: FAST only: trading_execution.py:841-843 skip-if-rank_factor<=0, :917 risk*rank_factor, :922 max_risk_dollars*rank_factor. SLOW has no rank concept.
|
||||
- recommend: To make it live: emit rank onto signal_ready, add it to query_ready_signals_snapshot, apply rank_risk_factor inside the unified helper. Until then keep TRADING_CONCENTRATE_TOP_N=0 and keep the boot warning.
|
||||
- **[partial / medium risk] VWAP selection gate (require_above_vwap=1)**
|
||||
- where: SLOW only: trading_execution.py:659-680. FAST has no VWAP check.
|
||||
- recommend: Plumb above_vwap (or session VWAP) into the trigger path, or funnel all live entries through the unified chokepoint. Easiest: compute above_vwap in the unified helper from live price + a vwap provider.
|
||||
- **[live / low risk] Dilution veto (_dilution_vetoed, NEWS_DILUTION_VETO_ENABLED=1)**
|
||||
- where: BOTH: SLOW trading_execution.py:644-648; FAST :864-867
|
||||
- recommend: No change. Template for unification: one shared predicate called from both paths.
|
||||
- **[partial / low risk] Bad-geometry check (stop>=entry, trigger<=0, bad price)**
|
||||
- where: SLOW: trading_execution.py:651 (not entry or not stop or stop>=entry). FAST: :872-878 (bad_price, then stop>=entry_ref or trigger<=0).
|
||||
- recommend: In the unified helper validate geometry against the same reference (the price the order will actually use) and keep the bad_price guard. Low risk to converge.
|
||||
- **[dead / high risk] Extension / anti-chase guard (extension_max_pct=0.15) + day-extension guard (day_extension_max_pct=0.30)**
|
||||
- where: FAST only: trading_execution.py:887-896 (over_extended / over_extended_day). SLOW has neither.
|
||||
- recommend: Most safety-relevant gap. Either ensure live fills happen on the fast/trigger path, or carry trigger+day_open+live price into the unified chokepoint and apply the same extension math. Do not leave the live entry point without anti-chase protection.
|
||||
- **[dead / high risk] Halt guard (halt_guard_enabled, halt_max_bar_gap_min)**
|
||||
- where: FAST only: trading_execution.py:899-900 (skipped='halted_symbol'). SLOW has no halt check.
|
||||
- recommend: Compute or pass the halt flag inside the unified chokepoint. Until unified, slow-path entries can buy into a halt.
|
||||
- **[partial / medium risk] Sizing: liquidity cap (liquidity_max_volume_pct=0.025)**
|
||||
- where: FAST only: trading_execution.py:903-904 max_shares=int(liq*cum_volume) -> calculate_position_size(max_shares=...). SLOW :687-697 passes no max_shares.
|
||||
- recommend: Carry cum_volume (or a recent-volume estimate) onto the signal/projection so the unified helper can pass max_shares.
|
||||
- **[live / low risk] Sizing core: equity / risk% / max_position_pct / max_gross_notional / max_risk_dollars**
|
||||
- where: BOTH: SLOW trading_execution.py:682-697; FAST :902-923
|
||||
- recommend: Keep the shared sizing core. Move rank_factor and max_shares into the unified helper's context so the single calculate_position_size call covers all paths.
|
||||
- **[duplicate / medium risk] Fill mechanism diverges: resting limit (slow) vs marketable limit (fast)**
|
||||
- where: SLOW: trading_execution.py:713 order_type=self.settings.entry_order_type (default 'limit' resting at signal entry). FAST: :929-931 marketable limit = max(entry_ref,trigger)*(1+trigger_slippage_pct).
|
||||
- recommend: Decide one fill model. The marketable-limit (fast) model is intended for runners and needs live price, which is another reason the unified chokepoint should keep the trigger thread as price source.
|
||||
- **[live / low risk] Shared post-submit arming + backout (expire_stale_entries) already unified**
|
||||
- where: BOTH arm into self._armed (SLOW :781-792, FAST :976-988); expire_stale_entries :1269, _note_position_exits :1232, _register_backout :1207 use shared state.
|
||||
- recommend: Model the entry unification on this already-unified exit side: bring the entry decision+submit into one _place_entry the same way.
|
||||
|
||||
_notes:_ UNIFICATION PLAN (concrete, ordered, per-step risk). Goal: one entry chokepoint _place_entry(symbol, entry_ref, trigger, stop, ctx) that both the trigger thread and the auto-approval flow call, eliminating the dead/partial gates. ctx carries day_open, cum_volume, rank, halted, above_vwap, last_price. STEP 1 (low risk): Extract a private _place_entry from submit_breakout_now's body (lines 868-1005) that takes a context object and runs ALL gates (dilution, geometry, extension, day-extension, halt, VWAP, rank, liquidity sizing, gross-notional). submit_breakout_now becomes a thin wrapper that builds ctx from its args and calls it. Pure refactor, no behaviour change; covered by existing fast-path tests. STEP 2 (medium risk): Enrich the signal_ready event payload + query_ready_signals_snapshot (storage/projections.py:223-233) to carry trigger_price, day_open, cum_volume, rank, and a live/last price. Without this the slow path structurally cannot run the extension/halt/liquidity/rank gates (root cause of the dead gates). Risk: signal producer (watcher) must populate the new fields; until it does, gates fail-open. STEP 3 (medium): Make approve_order build a ctx from the enriched approval payload and call the SAME _place_entry instead of the inline executor.execute (trading_execution.py:774). Now both paths share one gate set. Keep approve_order's manual-override exemption (manual dashboard approvals skip the daily cap and possibly other gates by design). STEP 4 (high risk, do last and behind a flag): Decide the single fill model. If the marketable-limit model wins, the slow path must price off live price (needs STEP 2's last price). Gate behind an env flag so you can A/B against the current resting-limit behaviour and roll back. STEP 5 (cleanup): Once verified in production that fills flow only through the unified helper, delete the divergent inline blocks and the boot WARNING at run_live_paper.py:810-815. What MUST NOT break: shared state under self._lock (self._fresh_entries, self._requested_symbols, self._armed, self._exited_today, self._cooldown_until, self._held_ret) — keep all mutations inside @_locked methods; the naked-stop protections in expire_stale_entries (grace window, broker_held check) — leave untouched; the auto-vs-manual approval distinction. SAFETY NOTE: today, because live entries take the slow path, the anti-chase extension guards, day-extension guard, halt guard, VWAP gate (on fast), liquidity sizing, and rank-concentration are NOT all applied to the actual live entry. The trigger thread DOES apply them when it fires, so whether live entries are protected depends on whether fills originate from the trigger thread or the slow auto-approval flow — confirm this empirically before relying on those guards. Files: /mnt/c/code/momentum/trading_execution.py, /mnt/c/code/momentum/run_live_paper.py, /mnt/c/code/momentum/storage/projections.py, /mnt/c/code/momentum/strategy/risk/position_sizing.py, /mnt/c/code/momentum/.env.
|
||||
|
||||
## Appendix — Dead-code map
|
||||
|
||||
- **[dead / low risk] telemetry/ package (telemetry/events.py + empty telemetry/__init__.py)**
|
||||
- where: /mnt/c/code/momentum/telemetry/events.py, /mnt/c/code/momentum/telemetry/__init__.py
|
||||
- recommend: Delete telemetry/events.py and telemetry/__init__.py (both git-tracked). Optionally drop the now-orphaned `telemetry_events` table from storage/pg_schema.sql:415 (no live writer/reader).
|
||||
- **[dead / low risk] core/telemetry.py**
|
||||
- where: /mnt/c/code/momentum/core/telemetry.py
|
||||
- recommend: Delete core/telemetry.py (git-tracked). It is a parallel, unused telemetry implementation superseded by storage/event_store + storage/event_schema.
|
||||
- **[dead / low risk] core/validation.py**
|
||||
- where: /mnt/c/code/momentum/core/validation.py
|
||||
- recommend: Delete core/validation.py (git-tracked).
|
||||
- **[dead / low risk] core/contracts.py**
|
||||
- where: /mnt/c/code/momentum/core/contracts.py
|
||||
- recommend: Delete core/contracts.py together with core/telemetry.py and core/validation.py. The entire core/ package (3 files + empty core/__init__.py) can be removed; __init__.py is 0 bytes with no re-exports.
|
||||
- **[dead / medium risk] Schwab integration (schwab/ package, 23 files / ~1121 LOC)**
|
||||
- where: /mnt/c/code/momentum/schwab/ (auth/, market/, orders/, positions/, health/, streaming/, settings.py)
|
||||
- recommend: Non-functional and isolated from the live loop. Either (a) delete schwab/ + milestone3_verify.py + tests/unit/schwab/ + the schwab_live branch in trading_mode.py + Schwab rows in README/RUNBOOK, or (b) if Schwab is a deliberate future path, fix the env-var names in schwab/settings.py and add an actual order executor before keeping it. As-is it is pure troubleshooting noise.
|
||||
- **[dead / medium risk] milestone3_verify.py (Schwab health-walk script)**
|
||||
- where: /mnt/c/code/momentum/milestone3_verify.py
|
||||
- recommend: Delete alongside the schwab/ package (they live and die together). Remove its README/RUNBOOK rows.
|
||||
- **[live / high risk] Standalone research/sweep CLIs (diagnose_entries.py, sweep_exits.py, sweep_filters.py, comprehensive_backtest.py, fetch_minute_bars.py, backtest_cli.py)**
|
||||
- where: /mnt/c/code/momentum/diagnose_entries.py, sweep_exits.py, sweep_filters.py, comprehensive_backtest.py, fetch_minute_bars.py, backtest_cli.py
|
||||
- recommend: KEEP. Do not delete -- these are documented operator/research entry points, just not part of the trading runtime.
|
||||
- **[dead / low risk] archive/ directory (6641 .py files, ~older full copies of the codebase)**
|
||||
- where: /mnt/c/code/momentum/archive/
|
||||
- recommend: Safe to delete the local archive/ directory entirely -- it is gitignored (not in version control, nothing depends on it). Removing it dramatically cleans up search/grep results. Verify nothing local is needed first since it is not recoverable from git.
|
||||
- **[dead / low risk] momentum-rebuilt.zip (232 MB stray artifact)**
|
||||
- where: /mnt/c/code/momentum/momentum-rebuilt.zip
|
||||
- recommend: Safe to delete the local file (gitignored, not tracked, no references).
|
||||
- **[dead / low risk] data/_backtest_multiday.py, data/_sip_backfill.py (underscore-prefixed scratch modules)**
|
||||
- where: /mnt/c/code/momentum/data/_backtest_multiday.py, /mnt/c/code/momentum/data/_sip_backfill.py
|
||||
- recommend: Safe to delete (local-only, untracked, no callers). Confirm they aren't a backfill you still need to run, since they're not in git.
|
||||
|
||||
_notes:_ Method: read entry points (run_live_paper.py imports at lines 38-68 use alpaca_paper + research.ingestion + runtime + storage + trading_execution -- NOT core/ or telemetry/ or schwab/), then grepped the whole repo (excluding the gitignored archive/ and .venv) for each suspect, explicitly including run_live_paper.py and all of tests/. Verified imports actually load with system python3 (core.contracts/validation/telemetry and telemetry.events all import OK), so the memory's 'reported broken' label is stale -- these are dead-by-disuse, not syntactically broken. Schwab is the higher-risk delete: it's a 23-file/~1.1k-LOC read-only integration with a confirmed env-var mismatch (settings.py reads SCHWAB_MARKET_DATA_APP_KEY/SCHWAB_BROKER_APP_KEY with no fallback; .env supplies SCHWAB_Market_APP_KEY/SCHWAB_Trading_APP_KEY) and no order-placement code, so it can't trade live -- but it has a test file and README/RUNBOOK docs, so deletion is a product decision (medium risk). The sweep/diagnose CLIs are LIVE research tools (documented, not runtime) and must NOT be deleted (marked high risk to prevent accidental removal). Biggest noise reducers with lowest risk: delete the untracked, gitignored archive/ tree (6641 .py files) and momentum-rebuilt.zip (232MB). I MODIFIED NOTHING -- read-only inspection only.
|
||||
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,100 @@
|
||||
# Momentum-ignition setup — shadow scope
|
||||
|
||||
_Scope produced 2026-06-24 by a read-only 4-agent mapping workflow (wf_55582b6d). No code changed.
|
||||
A SECOND, shadow-only lane to catch PLSM-class vertical squeezes the pullback-ORB scorer is blind to._
|
||||
|
||||
## Why
|
||||
The live setup is `impulse → pullback → breakout` (`strategy/evaluation/structure.py` `classify_setup`).
|
||||
A true vertical squeeze gives **no pullback and no formed opening range**, so `classify_setup` returns
|
||||
NONE and the scorer blocks it — verified: PLSM was evaluated **1,895× on 2026-06-24 and blocked ~30%
|
||||
every time**. This lane is the missing discipline: ride the vertical, not the base.
|
||||
|
||||
## What it detects
|
||||
An intraday vertical ignition with no consolidation: a fresh **high-of-day above the prior all-time
|
||||
high (blue sky)**, **accelerating price velocity** (consecutive green bars + an X% move over K minutes),
|
||||
and an **absolute volume burst**. It deliberately catches the pattern that makes `classify_setup`
|
||||
return NONE.
|
||||
|
||||
## Detection rules — all computable TODAY (binary, no fitting at this n)
|
||||
| signal | rule | source |
|
||||
|---|---|---|
|
||||
| **Blue-sky fresh-HOD** | `current.high > prior_ATH` (max `daily_bars.high` before today) AND this bar is the running session-high HOD. *Don't* reuse `detect_hod_break` (needs consolidation a vertical never gives). | `daily_bars.high` + `minute_bars.high` |
|
||||
| **Price velocity** | `pct_move(close[t]/close[t-N]-1) ≥ 0.08` over N=5–10 RTH min, AND ≥4 of last 5 bars green, AND higher-highs. (PLSM 12:02–12:16: 10/14 green, +13.9% in 14 min.) | `minute_bars` OHLC, pure arithmetic |
|
||||
| **Absolute volume burst** | `last_bar_vol / mean(prior K=5 bars) ≥ 3.0` AND cumulative session volume ≥ 100k shares (or a $-volume floor). **Absolute floors, NOT RVOL** — `scan_gappers` RVOL zeros out on thin IPOs (adv=0), and `rolling_avg_volume_20d` is NULL. | `minute_bars.volume` |
|
||||
| catalyst *(tag only)* | attach max-conviction bullish row (`sentiment>0`, `is_dilutive=False`); never a gate (Ollama gated off / can be down). PLSM "Soaring" → 0.9 conviction. | `news_catalyst_cache` |
|
||||
| VWAP *(tag only)* | log `above_vwap` + `dist_from_vwap` as grading context (the one validated ~1.5× signal). | `compute_key_levels.vwap` |
|
||||
| float *(placeholder)* | **DO NOT GATE** — `symbols` table is empty, `float_shares` NULL. Log `None` so a later float backfill can re-grade historical shadows. | `symbols.float_shares` (empty) |
|
||||
|
||||
## Hook point
|
||||
A new zero-arg closure **`step_shadow_ignition()`** in `run_live_paper.py` beside `step_watch` (~L528),
|
||||
registered via `scheduler.add('shadow_ignition', …, enabled=_flag('SHADOW_IGNITION_ENABLED','0'))` —
|
||||
**ships dark**. Pure-read against `research_con` (single psycopg2 conn — not thread-safe), emit-only,
|
||||
never references `rt['execution']`. Reuses the watcher provider's `get_candidates`/`get_bars` (same
|
||||
symbols/bars) but with its OWN wider caps (`SHADOW_PRICE_MAX≈100`, no gap ceiling) so it validates the
|
||||
very names live config excludes. Detection is a new pure `detect_momentum_ignition(bars)` in
|
||||
`strategy/evaluation/ignition.py` — **NOT** wired into `classify_setup`/`evaluate_setup` (the pullback
|
||||
criteria would reject it and `STRATEGY_SETUPS=opening_range_break` would filter it).
|
||||
|
||||
## Shadow mechanism — structurally cannot trade
|
||||
New `EventType.SHADOW_SIGNAL` + `ShadowSignalEvent` (symbol, trigger, would-be stop, signal_values
|
||||
dict, session_date). It is **invisible to execution**: `query_ready_signals_snapshot` filters strictly
|
||||
on `event_type='signal_ready'`, the only thing `TradingExecutionService.tick` reads — a `shadow_signal`
|
||||
is never seen, and the lane never calls `submit_breakout_now`/`request_approvals`. (Precedent: the VWAP
|
||||
gate already shadow-logs via `RiskRuleTriggeredEvent action_taken=shadow_logged`.) Safety: wrap the
|
||||
step in try/except (a shadow failure can't kill the loop), once-per-session per-symbol debounce (so one
|
||||
squeeze logs once, not 1,895×), never emit `signal_ready`, never set a symbol "ready".
|
||||
|
||||
## Measurement — reuse existing forward-outcome machinery (zero new metric code)
|
||||
1. **Labeler**: add `build_ignition` to `research/labeler.py` (`setup_name=momentum_ignition`,
|
||||
`setup_version=ign_v1`) reusing the identical label block (`reached_1r/2r_before_minus_1r`,
|
||||
`max_upside_5/15/60m`, `time_to_max`, `failed_breakout_flag`=the fizzle trap, `held_vwap`). `report`
|
||||
/`lift` filtered by version give runner-vs-trap separation per signal.
|
||||
2. **eod_replay**: add `score_shadow_signals` — after the close, read the day's actual `shadow_signal`
|
||||
events, walk `minute_bars` strictly after each, classify **runner (+1R) / fizzle / halt-down-trap**
|
||||
(the FCUV mode: a forward bar gapping far below the prior low, or R ≤ −1 within minutes). Run nightly.
|
||||
|
||||
## Promotion bar — PRE-REGISTER before shadowing (so the bar can't move to fit noise)
|
||||
- `reached_1r` rate **materially above** the `failed_breakout` trap rate, AND
|
||||
- halt-down-trap rate **below a ceiling**, AND
|
||||
- **distinct runner instances** above the thin-n floor (≥30, or rate×count ≥8) — counted in distinct
|
||||
runners, NOT calendar days (the ~17–19 days are autocorrelated, one regime).
|
||||
- Decision hangs on the **eod_replay as-fired** numbers (what a live lever would actually capture).
|
||||
|
||||
## Build phases
|
||||
| phase | blast | what |
|
||||
|---|---|---|
|
||||
| **P0** | none | `detect_momentum_ignition(bars)` pure + unit-tested (synthetic bars + the PLSM 12:02–12:16 window). No wiring. |
|
||||
| **P1** | low | `SHADOW_SIGNAL`/`ShadowSignalEvent` + `step_shadow_ignition` (default OFF, own caps, debounce, try/except). Test asserts it NEVER emits `signal_ready` / never calls execution. |
|
||||
| **P2** | low | `build_ignition` (labeler) + `score_shadow_signals` (eod_replay), nightly. DB-unreachable tolerant; tests use `:memory:`/`MOMENTUM_PG_SCHEMA`. |
|
||||
| **P3** | none | Pre-register the promotion thresholds; shadow N weeks; accumulate. Zero capital, zero trading-path edits. |
|
||||
| **P4** | high | *Only after P3 separation is proven* — a tiny live lever behind `require_ignition_confirmed` (default OFF, mirrors `require_above_vwap`), minimally sized, bounded by ALL shipped catastrophe controls (enforced stops, anti-chase, BE@+5%, halt guard). The ORB book + anti-chase guards untouched — strictly additive. |
|
||||
|
||||
## Open questions for you
|
||||
- **Stop definition** for the R denominator: last-higher-low vs bar-low vs −8%? Changes every reached_1r/trap rate — pick ONE and pre-register. (Maps lean last-higher-low.)
|
||||
- **Shadow gap ceiling**: drop entirely or cap ~80%? Dropping surfaces the most extreme/trap-prone names (fine for shadow).
|
||||
- **Catalyst tag** needs Ollama running that day — force-enable it for the shadow grading pass so news-backed vs not can be measured?
|
||||
- **Distinct-runner-n ≥30** may be unrealistic given runner sparsity — a Bayesian interval at low n instead of a point rate?
|
||||
|
||||
## ⚠️ P0 reality-check (2026-06-25) — READ BEFORE P1
|
||||
The P0 detector is built + unit-tested (fires correctly on a clean synthetic vertical). But running
|
||||
it over the **real** PLSM 2026-06-24 bars fired **0 times — correctly** — and the *why* reshapes this
|
||||
whole effort:
|
||||
- PLSM's session HIGH (12.85) was made at the **9:35 ET open** (it gapped from ~$4 **pre-market**); RTH
|
||||
was then a day-long **fade**. The "+400% to $20" the trader cites happened **pre-market**, which the
|
||||
bot does not ingest (RTH-only `minute_bars`) or trade.
|
||||
- PLSM was **halted for 228 of ~390 RTH minutes** — its RTH move is **halt-resumption GAPS**
|
||||
(+3.6%, −12.5%, −17.3%, −14.4% …), NOT a smooth consecutive-bar vertical. A detector keyed on
|
||||
consecutive green bars + smooth velocity **structurally cannot** see a halt-gapped move.
|
||||
|
||||
**Implication:** the momentum-ignition opportunity is largely a **pre-market + halt-resumption**
|
||||
phenomenon, not an RTH smooth-vertical one. Capturing it would need (a) **pre-market/extended-hours bar
|
||||
ingestion**, (b) **halt-aware, gap-up detection** (resumption gaps, not consecutive bars), and (c) the
|
||||
bot to **trade pre-market / halt resumptions** — which runs straight into the **halt-down blowup risk**
|
||||
(FCUV) the anti-chase guards exist to prevent. That is a far bigger lift than an RTH detector, and an
|
||||
operator decision before P1: invest in pre-market+halt data/logic, or shelve the ignition lane and bank
|
||||
the ORB/float work instead. The P0 detector + tests remain a correct foundation either way.
|
||||
|
||||
## Hard constraints (non-negotiable)
|
||||
Shadow-only until forward data proves it · never touches the ORB book or anti-chase guards · ships dark
|
||||
(`SHADOW_IGNITION_ENABLED=0`) · binary rules, no fitting · do NOT gate on RVOL or float (both broken for
|
||||
exactly these names) · tests use `:memory:`/`MOMENTUM_PG_SCHEMA` (never the live DB).
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,6 @@
|
||||
# Historical documents
|
||||
|
||||
Point-in-time plans, phase writeups, and runbooks from earlier eras of the
|
||||
project (DuckDB backend, WSL/systemd-only deployment, pre-migration
|
||||
architecture pushes). Kept for archaeology; **do not follow them** — the
|
||||
living docs are the root README.md, RUNBOOK.md, and deploy/README.md.
|
||||
File renamed without changes.
@@ -0,0 +1,227 @@
|
||||
# RUNBOOK
|
||||
|
||||
Operational guide for running the momentum system. This file replaces the
|
||||
original MILESTONE*.md documents, which were unrecoverable during the
|
||||
rebuild.
|
||||
|
||||
---
|
||||
|
||||
## 1. Why live paper trading never worked (root causes, now fixed)
|
||||
|
||||
1. **Minute bars were never ingested.** The watcher reads 1-minute bars from
|
||||
`data/research/market.duckdb`, but the old ingestion only wrote *daily*
|
||||
bars for a handful of mega-caps. Fix: `research/ingestion/market_data.py`
|
||||
ingests 1-minute Alpaca IEX bars continuously, and
|
||||
`run_live_paper.py` schedules it every `LIVE_BARS_INTERVAL_SECONDS`.
|
||||
|
||||
2. **Session flags were hardcoded.** Every old row had
|
||||
`is_regular_hours = TRUE`. Fix: flags are derived from the bar timestamp
|
||||
in US/Eastern (premarket 04:00–09:30, regular 09:30–16:00, afterhours
|
||||
16:00–20:00) — see `classify_session()` and its tests.
|
||||
|
||||
3. **The candidate filter excluded everything ingested.** The query had a
|
||||
hardcoded `close BETWEEN 1 AND 20` while only mega-caps (price ≫ 20) were
|
||||
in the database, so the watchlist was always empty. Fix: the band is
|
||||
configurable (`WATCHER_PRICE_MIN` / `WATCHER_PRICE_MAX`) and the default
|
||||
symbol list now actually gets ingested.
|
||||
|
||||
4. **No orchestrator.** Ingestion, watcher, broker sync, and execution were
|
||||
never wired into one loop. Fix: `run_live_paper.py`.
|
||||
|
||||
5. **Broken source files.** `runtime/providers/registry.py`,
|
||||
`schwab/market/models.py`, `option_chain_service.py`, and
|
||||
`schwab/streaming/client.py` had syntax errors that crashed imports.
|
||||
All repaired/rewritten.
|
||||
|
||||
6. **Schwab tokens were committed to git.** `data/schwab_tokens.json` is now
|
||||
git-ignored and written `chmod 0600`. **Action required:** if that file
|
||||
exists anywhere in your repo history, revoke the refresh token in the
|
||||
Schwab developer portal and re-run the OAuth flow. Scrub history with
|
||||
`git filter-repo --path data/schwab_tokens.json --invert-paths` if the
|
||||
repo was ever shared.
|
||||
|
||||
---
|
||||
|
||||
## 2. Environment setup
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Minimum for paper trading: `ALPACA_API_KEY`, `ALPACA_SECRET_KEY`
|
||||
(free keys are fine — the system uses the IEX feed). Everything else has a
|
||||
sensible default; see `.env.example` for the full catalog.
|
||||
|
||||
Key knobs:
|
||||
|
||||
| Variable | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `TRADING_EXECUTION_MODE` | `alpaca_paper` | `schwab_live` routes to real money — leave on paper |
|
||||
| `TRADING_AUTO_APPROVE` | `0` | `1` = orders submit without dashboard approval (or pass `--auto-approve`) |
|
||||
| `TRADING_MAX_ORDERS_PER_TICK` | `1` | throttle new entries |
|
||||
| `TRADING_MAX_CONCURRENT_POSITIONS` | `3` | cap on simultaneous open positions |
|
||||
| `TRADING_RISK_PER_TRADE_PCT` | `0.01` | 1% of equity risked per trade (sizing = risk / (entry − stop)) |
|
||||
| `TRADING_REWARD_MULTIPLE` | `2.0` | profit target = entry + this × risk (2R by default) |
|
||||
| `TRADING_ENTRY_ORDER_TYPE` | `limit` | `limit` rests at the trigger so an unfilled entry can be cancelled; `market` fills instantly |
|
||||
| `TRADING_ENTRY_TIMEOUT_BARS` | `2` | cancel an unfilled entry after this many minutes resting (wall-clock; `0` disables the time box) |
|
||||
| `TRADING_ENTRY_INVALIDATE_PCT` | `0.0` | cancel an unfilled entry if the **live last-trade price** trades this fraction below the trigger (`0.0` = any break below entry; `0.005` = 0.5% below; negative disables) |
|
||||
| `TRADING_ENTRY_GUARD_INTERVAL_SECONDS` | `5` | how often the fast guard re-checks armed entries against the live price — lower = faster break detection, more quote calls |
|
||||
| `TRADING_MAX_DAILY_LOSS_PCT` | `0.03` | daily-loss circuit breaker |
|
||||
| `WATCHER_SYMBOLS` | `AAPL,TSLA,…` | static watchlist; `--discover` adds screener actives |
|
||||
| `WATCHER_PRICE_MIN/MAX` | `1` / `20` | candidate price band |
|
||||
| `WATCHER_EVENT_DB_PATH` | `./data/momentum.duckdb` | the event store |
|
||||
| `DASHBOARD_HOST/PORT` | `127.0.0.1` / `8010` | UI address |
|
||||
|
||||
---
|
||||
|
||||
## 3. Start commands
|
||||
|
||||
**Full live paper loop (recommended):**
|
||||
|
||||
```bash
|
||||
python run_live_paper.py
|
||||
# dashboard: http://127.0.0.1:8010
|
||||
# flags: --once --symbols SNDL,COSM --auto-approve --discover --no-dashboard
|
||||
```
|
||||
|
||||
Each tick prints one status line:
|
||||
|
||||
```
|
||||
[09:31:02] ingest: 240 minute rows | watch: evaluated=5 ready=['SNDL'] blocked=3 | sync: ok | execute: approvals_requested=['…'] auto_executed=0
|
||||
```
|
||||
|
||||
**Standalone read-only dashboard** (e.g. to review a past session):
|
||||
|
||||
```bash
|
||||
python dashboard_api.py
|
||||
```
|
||||
|
||||
Approve/reject/exit buttons require the orchestrator (it attaches the
|
||||
execution service to the same server); the standalone runner returns 503 on
|
||||
actions by design.
|
||||
|
||||
**Other tools:**
|
||||
|
||||
```bash
|
||||
python fetch_minute_bars.py SNDL COSM --lookback 390 # ad-hoc ingestion
|
||||
python research_cli.py symbols --date 2026-06-11 # what's in the DB
|
||||
python research_cli.py gappers --date 2026-06-11 # gap scan → scanner_snapshots
|
||||
python backtest_cli.py --date 2026-06-10 # backtest a stored session
|
||||
python milestone3_verify.py # Schwab health walk
|
||||
python -m pytest # full test suite
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. The trading day, end to end
|
||||
|
||||
1. **Ingest** — every 60 s, 1-minute IEX bars for the watchlist land in
|
||||
`data/research/market.duckdb` with correct session flags. On boot, a
|
||||
30-day daily backfill gives gap % and relative volume a baseline.
|
||||
2. **Watch** — every 30 s the watcher pulls in-band candidates, evaluates
|
||||
nine weighted criteria (gap, RVOL, impulse, pullback, pullback volume,
|
||||
VWAP, candle quality, breakout, data sufficiency) plus structure
|
||||
detection, and drives `discovered → watching → ready | blocked | late`.
|
||||
A `signal_ready` is emitted at most once per symbol per session.
|
||||
3. **Sync** — Alpaca account, positions, and orders snapshot into the event
|
||||
store so projections and risk guards see broker truth.
|
||||
4. **Execute** — ready signals become orders sized at `risk_per_trade_pct`
|
||||
of equity, with the stop from the setup and the target at
|
||||
entry + `reward_multiple` × risk (2R by default). Manual mode parks them
|
||||
in the dashboard queue; **auto mode (`--auto-approve`) submits
|
||||
immediately, no approval.** Guards: max concurrent positions, no re-entry
|
||||
into held symbols, one request per symbol per session.
|
||||
5. **Manage** — approve/reject from the dashboard; **Exit** sends a market
|
||||
sell for the synced quantity.
|
||||
|
||||
### The entry lifecycle (auto-arm → fill or back out)
|
||||
|
||||
The earlier system would "see" a setup as something that had *already
|
||||
happened* rather than a trade about to happen, and never pull the trigger.
|
||||
The execution model now separates recognition from conviction from
|
||||
discipline:
|
||||
|
||||
1. **Recognize** — when all the criteria pass, a `signal_ready` fires. The
|
||||
setup is evaluated against the **last bar's timestamp**, not the wall
|
||||
clock, so a live 9:40 breakout is judged against the 9:40 cutoff instead
|
||||
of being dismissed as hours stale.
|
||||
2. **Arm with conviction** — in auto mode the order is submitted right away
|
||||
as a **bracket** (entry + stop + target placed atomically). With
|
||||
`TRADING_ENTRY_ORDER_TYPE=limit` (default) the entry rests at the trigger,
|
||||
so "unfilled" is a real, observable state.
|
||||
3. **Back out at a defined point** — a fast **guard loop**
|
||||
(`TRADING_ENTRY_GUARD_INTERVAL_SECONDS`, default 5 s) re-checks every
|
||||
armed-but-unfilled entry and cancels it if either:
|
||||
- the **live last-trade price** (pulled tick-by-tick from Alpaca's data
|
||||
API, not a bar close) trades back below the trigger by
|
||||
`TRADING_ENTRY_INVALIDATE_PCT`, or
|
||||
- it has rested longer than `TRADING_ENTRY_TIMEOUT_BARS` minutes
|
||||
(wall-clock — running the guard often does **not** shorten this window).
|
||||
A cancel frees the risk budget and the concurrent-position slot, and emits
|
||||
a `risk_rule_triggered` (`entry_backout`) you can see in the activity feed.
|
||||
Once the order **fills**, it stops being tracked and the bracket's stop and
|
||||
target manage the open position — no second-guessing.
|
||||
|
||||
Tuning cheat-sheet: react to a fakeout faster → lower
|
||||
`TRADING_ENTRY_GUARD_INTERVAL_SECONDS` (e.g. `2`); more patience on the fill →
|
||||
raise `TRADING_ENTRY_TIMEOUT_BARS`; give the breakout room to wobble → set
|
||||
`TRADING_ENTRY_INVALIDATE_PCT` to something like `0.005`; chase fills instead
|
||||
of resting → `TRADING_ENTRY_ORDER_TYPE=market` (note: market fills can't time
|
||||
out); bigger winners → raise `TRADING_REWARD_MULTIPLE`.
|
||||
|
||||
> **Quote rate limits.** The guard makes one latest-trade call per armed
|
||||
> symbol per pass (a ~2 s cache collapses duplicates). At the 5 s default with
|
||||
> a few open entries you're well inside Alpaca's free-tier 200 req/min. If you
|
||||
> drop the interval to 1 s with several armed entries, watch the budget —
|
||||
> bars ingestion and account sync also draw from it.
|
||||
|
||||
### Reading the dashboard
|
||||
|
||||
- **Badge** — blue `PAPER · ALPACA` vs red `LIVE · SCHWAB`. If it's red and
|
||||
you didn't mean it, stop and check `TRADING_EXECUTION_MODE`.
|
||||
- **P&L strip** (top) — session realized / unrealized / total, win rate,
|
||||
average R, open/closed counts. Green/red signed.
|
||||
- **Approval Queue** — pending entries with a **risk preview** ($ risk,
|
||||
shares, notional, stop %) and Approve/Reject. Keyboard: `A` / `R` act on the
|
||||
top row.
|
||||
- **Ready Signals / Open Positions** — each row has a live **sparkline** (last
|
||||
~40 one-minute closes) with dashed entry (blue) and stop (red) lines.
|
||||
- **Watch States** — click any row to expand the **nine-criteria breakdown**
|
||||
(✓ pass / ✕ fail / · not-evaluated). The add-symbol box (or `/`) injects a
|
||||
ticker onto the watchlist for the rest of the session.
|
||||
- **Activity Feed** — submissions and fills as they happen.
|
||||
- **Order Lifecycle** — per-order status from the event store.
|
||||
- **Connection dot** (top right) — green pulse = live SSE stream; it falls
|
||||
back to 4 s polling automatically if the stream drops.
|
||||
- **Sound** — toggle the button or press `S` for an audio + toast alert when a
|
||||
new signal goes ready. Press `?` for the shortcut list.
|
||||
|
||||
Streams from `GET /api/stream` (SSE); also exposes `/api/snapshots`,
|
||||
`/api/criteria?symbol=`, `/api/bars?symbol=&minutes=`, and
|
||||
`POST /api/watch/add`. Charts require the research DB, which the orchestrator
|
||||
wires in automatically; the standalone `dashboard_api.py` has no bars source,
|
||||
so sparklines there show "no bars".
|
||||
|
||||
---
|
||||
|
||||
## 5. Troubleshooting
|
||||
|
||||
| Symptom | Likely cause / fix |
|
||||
|---|---|
|
||||
| `watch: evaluated=0` all day | No minute bars for today. Check keys; run `python research_cli.py symbols`. Outside market hours there are no new bars — backfill with `fetch_minute_bars.py --lookback 1440`. |
|
||||
| Candidates exist but never `ready` | Score < 60%. Watch the `Watch States` panel score column; criteria details are in `criteria_evaluated` events. |
|
||||
| `ingest: … 403` errors | Free keys are IEX-only — ensure `ALPACA_DATA_FEED=iex` (default) and that the request window ends ≥30 s in the past (handled automatically). |
|
||||
| Approve button does nothing | You're on `dashboard_api.py` (read-only). Use `run_live_paper.py`. |
|
||||
| Port 8010 busy | Set `DASHBOARD_PORT`, or `--no-dashboard`. |
|
||||
| Schwab shows `SCHWAB-UNAUTH` | Expected without OAuth. Run the flow in `schwab/auth/oauth2_flow_manager.py`; verify with `milestone3_verify.py`. |
|
||||
| Start fresh | Delete `data/momentum.duckdb` (events) and/or `data/research/` (bars). Both rebuild automatically. |
|
||||
|
||||
---
|
||||
|
||||
## 6. Going live on Schwab (when ready)
|
||||
|
||||
Paper-trade until the stats earn it. Then: complete OAuth
|
||||
(`milestone3_verify.py` all green), set `TRADING_EXECUTION_MODE=schwab_live`,
|
||||
keep `TRADING_AUTO_APPROVE=0`, and start with the smallest viable
|
||||
`TRADING_RISK_PER_TRADE_PCT`. The dashboard badge turns red in live mode.
|
||||
File renamed without changes.
@@ -0,0 +1,293 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Ideation: Next Enhancement Milestone — Momentum</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #fafaf8; --fg: #1a1d21; --muted: #5c6470; --line: #dcdfe4;
|
||||
--card: #ffffff; --accent: #0f6b54; --accent-soft: #e6f2ee; --accent-text: #0b5442;
|
||||
--warn: #9a5b00; --warn-soft: #fdf3e3; --info: #274f8f; --info-soft: #e9effa;
|
||||
--danger: #a33; --danger-soft: #faecec; --chip: #eef0f3; --code: #f2f3f5;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root {
|
||||
--bg: #14171a; --fg: #e6e8ea; --muted: #9aa3ad; --line: #2c3238;
|
||||
--card: #1b1f23; --accent: #4fbf9f; --accent-soft: #14312a; --accent-text: #7dd6bc;
|
||||
--warn: #e0a94f; --warn-soft: #33270f; --info: #7ea6e0; --info-soft: #17233a;
|
||||
--danger: #e08080; --danger-soft: #351b1b; --chip: #262b30; --code: #22272c;
|
||||
}
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; background: var(--bg); color: var(--fg);
|
||||
font: 15px/1.55 -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; }
|
||||
.page { max-width: 920px; margin-inline: auto; padding: 2.2rem 1.4rem 4rem; }
|
||||
p, li, dd { max-width: 72ch; }
|
||||
h1 { font-size: 1.7rem; line-height: 1.25; margin: .2rem 0 .4rem; }
|
||||
h2 { font-size: 1.25rem; margin: 2.4rem 0 .8rem; padding-top: 1rem; border-top: 1px solid var(--line); }
|
||||
h3 { font-size: 1.05rem; margin: 1.6rem 0 .5rem; }
|
||||
code { background: var(--code); padding: .1em .35em; border-radius: 4px; font-size: .88em; }
|
||||
a { color: var(--info); }
|
||||
.eyebrow { font-size: .72rem; letter-spacing: .12em; text-transform: uppercase; color: var(--muted); }
|
||||
.meta { display: flex; flex-wrap: wrap; gap: .4rem .6rem; margin: .6rem 0 0; padding: 0; list-style: none; }
|
||||
.meta li { background: var(--chip); border-radius: 999px; padding: .18rem .7rem; font-size: .8rem; color: var(--muted); }
|
||||
.meta li b { color: var(--fg); font-weight: 600; }
|
||||
.stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: .7rem; margin: 1.6rem 0; }
|
||||
.stat { background: var(--card); border: 1px solid var(--line); border-radius: 10px; padding: .7rem .9rem; }
|
||||
.stat .n { font-size: 1.35rem; font-weight: 700; }
|
||||
.stat .l { font-size: .78rem; color: var(--muted); }
|
||||
article.idea { background: var(--card); border: 1px solid var(--line); border-radius: 12px;
|
||||
padding: 1.1rem 1.3rem; margin: 1.1rem 0; }
|
||||
article.idea h3 { margin: 0 0 .5rem; }
|
||||
.chips { display: flex; flex-wrap: wrap; gap: .35rem; margin: 0 0 .8rem; }
|
||||
.chips span { border-radius: 999px; padding: .14rem .6rem; font-size: .74rem; font-weight: 600; }
|
||||
.chip-axis { background: var(--info-soft); color: var(--info); }
|
||||
.chip-conf { background: var(--accent-soft); color: var(--accent-text); }
|
||||
.chip-cx { background: var(--chip); color: var(--muted); }
|
||||
.chip-cx.high { background: var(--warn-soft); color: var(--warn); }
|
||||
dl.fields { margin: 0; }
|
||||
dl.fields dt { font-size: .72rem; letter-spacing: .1em; text-transform: uppercase; color: var(--muted); margin-top: .7rem; }
|
||||
dl.fields dd { margin: .15rem 0 0; }
|
||||
.jump { background: var(--card); border: 1px solid var(--line); border-radius: 10px; padding: .8rem 1.1rem; }
|
||||
.jump ol { margin: .3rem 0 0 1.2rem; padding: 0; }
|
||||
table { border-collapse: collapse; width: 100%; font-size: .88rem; }
|
||||
th, td { border: 1px solid var(--line); padding: .45rem .6rem; text-align: left; vertical-align: top; }
|
||||
th { background: var(--chip); font-size: .78rem; letter-spacing: .05em; text-transform: uppercase; }
|
||||
.tablewrap { overflow-x: auto; }
|
||||
figure { margin: 1rem 0 .3rem; }
|
||||
figcaption { font-size: .78rem; color: var(--muted); margin-top: .3rem; }
|
||||
footer.composition-signal { margin-top: 3rem; padding-top: 1rem; border-top: 1px solid var(--line);
|
||||
font-size: .78rem; color: var(--muted); }
|
||||
svg text { font-family: inherit; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="page">
|
||||
|
||||
<div class="eyebrow">Ideation · repo-grounded</div>
|
||||
<h1>Next Enhancement Milestone — Momentum</h1>
|
||||
<ul class="meta">
|
||||
<li><b>date</b> 2026-07-13</li>
|
||||
<li><b>topic</b> next-enhancement-milestone</li>
|
||||
<li><b>mode</b> repo-grounded</li>
|
||||
<li><b>repo</b> momentum @ d54a2a9</li>
|
||||
<li><b>focus</b> read everything, brainstorm enhancements, pick the highest-leverage build</li>
|
||||
</ul>
|
||||
|
||||
<div class="stats">
|
||||
<div class="stat"><div class="n">48</div><div class="l">raw ideas · 5 frames</div></div>
|
||||
<div class="stat"><div class="n">14</div><div class="l">merged candidates</div></div>
|
||||
<div class="stat"><div class="n">14/14</div><div class="l">verified sound (fresh-context spot-checks)</div></div>
|
||||
<div class="stat"><div class="n">7</div><div class="l">survivors · 5/5 axes covered</div></div>
|
||||
</div>
|
||||
|
||||
<h2 id="grounding-context">Grounding Context</h2>
|
||||
<p><strong>Codebase context.</strong> Event-sourced small-cap momentum day-trading system (gap-and-go / opening-range breakout). Alpaca <em>paper</em> account over the SIP feed; single PostgreSQL datastore (append-only <code>events</code> table + projections, ~19M minute bars); production is a Docker Compose stack on TrueNAS SCALE (postgres 18, trading loop, dashboard :8765, admin :8767, telegram bot, nightly scheduler); 376 pytest tests against throwaway <code>mem_*</code> schemas. Solo owner-operator.</p>
|
||||
<p>Grounding came from an 8-agent recon sweep (2026-07-13) that read every subsystem — live loop, strategy, storage, research/data, dashboard, ops/deploy, brokers, tests — and produced ~95 evidence-cited findings, plus the in-repo external research pass <code>docs/research/entry_conversion_research.md</code> (academic + backtest literature on breakout entry conversion, with a ranked experiment shortlist). A fresh-context verifier then spot-checked every load-bearing claim against HEAD with ~25 targeted reads; <strong>every checked citation matched</strong>.</p>
|
||||
<p><strong>Settled constraints honored throughout:</strong> the validated entry population stays ($2–20, static open gap 10–35%, real volume, ≥150%-gap & rvol≥5 squeeze exception); seven selection levers are refuted at year scale and are not re-proposed; scale-out partials and retest/pullback entries are refuted; the :8765 dashboard is the only UI to extend; the system's proven bottleneck is <strong>entry-conversion</strong>, not selection.</p>
|
||||
|
||||
<h2 id="topic-axes">Topic Axes</h2>
|
||||
<ul>
|
||||
<li><strong>money-path-correctness</strong> — session lifecycle, broker-truth handling, order reconcile, entry-path drift, exit-manager safety</li>
|
||||
<li><strong>learning-flywheel</strong> — tuning/replay scheduling, look-ahead & population parity, dataset growth, experiment machinery</li>
|
||||
<li><strong>entry-conversion</strong> — execution experiments on the proven bottleneck (time-stop, probe-add, asymmetric exit), shadow-track infrastructure</li>
|
||||
<li><strong>operability-delivery</strong> — CI gate, alerting/dead-man, backup, retention, env drift, dead-weight deletion</li>
|
||||
<li><strong>operator-surface</strong> — :8765 dashboard, :8767 admin, telegram: repairs, visibility, manual controls</li>
|
||||
</ul>
|
||||
|
||||
<h2 id="ranked-ideas">Ranked Ideas</h2>
|
||||
<div class="jump">
|
||||
<div class="eyebrow">Jump to</div>
|
||||
<ol>
|
||||
<li><a href="#i1">Experiment OS — pre-registered, auto-adjudicated counterfactual experiments over logged fires</a></li>
|
||||
<li><a href="#i2">Learning-flywheel resurrection — one population function + scheduled tune/replay</a></li>
|
||||
<li><a href="#i3">Entry-path unification + decision receipts</a></li>
|
||||
<li><a href="#i4">Money-path hardening — self-restart, positive-evidence contract, EOD cage count</a></li>
|
||||
<li><a href="#i5">Production safety rails — CI gate, dead-man's switch, automated backup</a></li>
|
||||
<li><a href="#i6">Operator control plane — dashboard repair, kill switch, effective-config panel</a></li>
|
||||
<li><a href="#i7">Dead-weight deletion sweep — every table names its reader or dies</a></li>
|
||||
</ol>
|
||||
</div>
|
||||
|
||||
<article class="idea" id="i1">
|
||||
<h3>1. Experiment OS — pre-registered, auto-adjudicated counterfactual experiments over logged fires</h3>
|
||||
<div class="chips"><span class="chip-axis">entry-conversion</span><span class="chip-conf">Confidence 75%</span><span class="chip-cx high">Complexity High</span></div>
|
||||
<dl class="fields">
|
||||
<dt>Description</dt>
|
||||
<dd>One parametric experiment harness replacing the four hand-built shadow-track triples in <code>research/labeler.py</code> (vwap_reclaim / runner / liquid / premarket — ~1,467 lines of structurally near-identical build/report/validate code). An experiment becomes a config row: population, intervention (an entry or exit policy variant), pre-registered pass/fail bars, minimum n, and futility criteria. A replay engine runs every logged live fire counterfactually against its recorded bar path — the Tesla shadow-mode pattern — including a <em>counterfactual exit ledger</em> (every real trade re-run under each exit variant). A verdict engine auto-adjudicates at the pre-registered thresholds (the clinical-trial DSMB pattern, with futility stops) and emits PASS / FAIL / INCONCLUSIVE verdict events that flow to Telegram and a dashboard experiment-ledger panel. First cohort ships with the harness: the three literature-backed entry-conversion experiments (idea card behind them in <code>docs/research/entry_conversion_research.md</code>): time-boxed entries + 10-minute time-stop on unconfirmed positions, probe-then-add 0.5×+0.5× sizing, asymmetric exit (volatility-fractional stop with a spread floor, no target, trail to EOD) — plus the rank-concentration "one-bullet" counterfactual and an immediate adjudication of pm1.</dd>
|
||||
<dt>Basis</dt>
|
||||
<dd><code>direct:</code> <code>data/premarket_eval_history.tsv</code> latest row (2026-07-09) shows pm1 at n=2377 with rate_1r=0.443 vs the pre-registered >0.50 bar and median_up_60m=0.0433 vs the ≥0.10 bar — failing 3 of 4 pre-registered criteria while the code's own docstring still frames it as "n=21 is far too small… watch n climb". Seven prior levers each required a human to notice a threshold crossing in a log file. <code>external:</code> the three first-cohort experiments carry academic + replicated-backtest support (Zarattini/Barbon/Aziz SSRN 4729284, Osler NY Fed SR150, Mesfin arXiv 2605.04004) per the in-repo research doc. Verifier confirmed none of the harness/experiment machinery exists yet.</dd>
|
||||
<dt>Rationale</dt>
|
||||
<dd>The system's bottleneck is proven to be entry-conversion, and the three highest-priority interventions are already researched, pre-registered, and testable for <em>free</em> against fires the system has already logged — zero live risk. The blocker is machinery, not ideas: every new experiment today costs a hand-built ~300-line triple plus a human reading a TSV. Build the machinery once and every future experiment (premarket module included) drops to a config row with an automatic verdict.</dd>
|
||||
<dt>Downsides</dt>
|
||||
<dd>Biggest build in the set; replay fidelity is a real risk (fills, spreads, halts must be modeled honestly — the tight-stop zero-slippage artifact has already fooled two shadow tracks here, so the harness must enforce adverse-fill floors by construction). Requires the population module from idea 2 to avoid baking look-ahead into every experiment.</dd>
|
||||
</dl>
|
||||
<figure>
|
||||
<svg viewBox="0 0 860 190" role="img" aria-label="Experiment OS flow">
|
||||
<defs><marker id="arr" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#8892a0"/></marker></defs>
|
||||
<rect x="10" y="55" width="150" height="80" rx="10" fill="var(--info-soft)" stroke="var(--info)"/>
|
||||
<text x="85" y="88" text-anchor="middle" font-size="13" font-weight="600" fill="var(--fg)">Logged fires</text>
|
||||
<text x="85" y="106" text-anchor="middle" font-size="11" fill="var(--muted)">+ recorded bar paths</text>
|
||||
<rect x="230" y="20" width="170" height="60" rx="10" fill="var(--card)" stroke="var(--line)"/>
|
||||
<text x="315" y="45" text-anchor="middle" font-size="12.5" font-weight="600" fill="var(--fg)">Experiment configs</text>
|
||||
<text x="315" y="62" text-anchor="middle" font-size="11" fill="var(--muted)">population · policy · bars · n</text>
|
||||
<rect x="230" y="105" width="170" height="65" rx="10" fill="var(--accent-soft)" stroke="var(--accent)"/>
|
||||
<text x="315" y="132" text-anchor="middle" font-size="12.5" font-weight="600" fill="var(--fg)">Replay engine</text>
|
||||
<text x="315" y="149" text-anchor="middle" font-size="11" fill="var(--muted)">counterfactual P&L per fire</text>
|
||||
<rect x="470" y="70" width="160" height="65" rx="10" fill="var(--warn-soft)" stroke="var(--warn)"/>
|
||||
<text x="550" y="97" text-anchor="middle" font-size="12.5" font-weight="600" fill="var(--fg)">Verdict engine</text>
|
||||
<text x="550" y="114" text-anchor="middle" font-size="11" fill="var(--muted)">pre-registered bars · futility</text>
|
||||
<rect x="700" y="20" width="150" height="55" rx="10" fill="var(--card)" stroke="var(--line)"/>
|
||||
<text x="775" y="43" text-anchor="middle" font-size="12" font-weight="600" fill="var(--fg)">Telegram ping</text>
|
||||
<text x="775" y="60" text-anchor="middle" font-size="11" fill="var(--muted)">PASS / FAIL / INCONCLUSIVE</text>
|
||||
<rect x="700" y="115" width="150" height="55" rx="10" fill="var(--card)" stroke="var(--line)"/>
|
||||
<text x="775" y="138" text-anchor="middle" font-size="12" font-weight="600" fill="var(--fg)">Ledger panel</text>
|
||||
<text x="775" y="155" text-anchor="middle" font-size="11" fill="var(--muted)">:8765 dashboard</text>
|
||||
<line x1="160" y1="95" x2="222" y2="130" stroke="#8892a0" stroke-width="1.6" marker-end="url(#arr)"/>
|
||||
<line x1="315" y1="80" x2="315" y2="98" stroke="#8892a0" stroke-width="1.6" marker-end="url(#arr)"/>
|
||||
<line x1="400" y1="130" x2="462" y2="108" stroke="#8892a0" stroke-width="1.6" marker-end="url(#arr)"/>
|
||||
<line x1="630" y1="88" x2="692" y2="55" stroke="#8892a0" stroke-width="1.6" marker-end="url(#arr)"/>
|
||||
<line x1="630" y1="115" x2="692" y2="138" stroke="#8892a0" stroke-width="1.6" marker-end="url(#arr)"/>
|
||||
</svg>
|
||||
<figcaption>Every logged fire becomes reusable evidence; every experiment becomes a config row with an automatic verdict.</figcaption>
|
||||
</figure>
|
||||
</article>
|
||||
|
||||
<article class="idea" id="i2">
|
||||
<h3>2. Learning-flywheel resurrection — one population function + scheduled tune/replay</h3>
|
||||
<div class="chips"><span class="chip-axis">learning-flywheel</span><span class="chip-conf">Confidence 90%</span><span class="chip-cx">Complexity Medium</span></div>
|
||||
<dl class="fields">
|
||||
<dt>Description</dt>
|
||||
<dd>Extract the live population gate into one shared pure module with an as-of-time contract (open-knowable inputs only: static open gap, price band, premarket-knowable volume, squeeze exception) and make every offline consumer — <code>learn_params.py</code>, the sweeps, <code>eod_replay.py</code>, the labeler — consume it instead of re-implementing it. Then actually schedule the flywheel in production: <code>eod_replay</code> after each close (unbiased whole-market dataset growth), <code>nightly_tune</code> after it (atomic tmp+rename write, staleness alert if <code>learned_params.json</code> ages past 2 days).</dd>
|
||||
<dt>Basis</dt>
|
||||
<dd><code>direct:</code> verifier-confirmed at HEAD: the TrueNAS scheduler runs <em>only</em> <code>research.premarket_eval</code>; <code>data/learned_params.json</code> is stamped as_of 2026-06-17 — 26 days stale — while the boot log still claims self-tuning; <code>learn_params.py:61,64</code> admits day-end <code>day_rvol</code> (outcome-time look-ahead); <code>eod_replay.py:44-48</code> screens $1–5 under a "live verified config" comment while production trades $2–20 (<code>truenas-compose.yml:105-106</code>); <code>eod_replay.sh</code> points at a dead venv path; <code>nightly_tune.py:91-92</code> writes non-atomically.</dd>
|
||||
<dt>Rationale</dt>
|
||||
<dd>The project memory says "nightly_tune self-tunes params, applied on boot" — that story is currently fiction in production, and the offline population drift means even a re-enabled tuner would fit a universe the bot doesn't trade. This is the data-integrity substrate under idea 1: experiments adjudicated against a look-ahead population are worthless. The owner has already been burned twice by exactly this class (the 2026-07-08 look-ahead correction).</dd>
|
||||
<dt>Downsides</dt>
|
||||
<dd>Population-module extraction touches the live gate — needs characterization tests before refactor. A deliberate alternative is to <em>delete</em> the tuner and stop claiming self-tuning; the wrong option is the current silent limbo.</dd>
|
||||
</dl>
|
||||
<figure>
|
||||
<svg viewBox="0 0 860 175" role="img" aria-label="One population module fanning out to five consumers">
|
||||
<defs><marker id="arr2" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#8892a0"/></marker></defs>
|
||||
<text x="115" y="20" text-anchor="middle" font-size="11.5" font-weight="700" fill="var(--danger)">TODAY: 5 diverging copies</text>
|
||||
<rect x="30" y="32" width="170" height="26" rx="6" fill="var(--danger-soft)" stroke="var(--danger)"/><text x="115" y="49" text-anchor="middle" font-size="11" fill="var(--fg)">live gate — $2–20, open gap</text>
|
||||
<rect x="30" y="64" width="170" height="26" rx="6" fill="var(--danger-soft)" stroke="var(--danger)"/><text x="115" y="81" text-anchor="middle" font-size="11" fill="var(--fg)">tuner — day-end rvol (look-ahead)</text>
|
||||
<rect x="30" y="96" width="170" height="26" rx="6" fill="var(--danger-soft)" stroke="var(--danger)"/><text x="115" y="113" text-anchor="middle" font-size="11" fill="var(--fg)">eod_replay — $1–5, no squeeze</text>
|
||||
<rect x="30" y="128" width="170" height="26" rx="6" fill="var(--danger-soft)" stroke="var(--danger)"/><text x="115" y="145" text-anchor="middle" font-size="11" fill="var(--fg)">sweeps · labeler — own filters</text>
|
||||
<text x="580" y="20" text-anchor="middle" font-size="11.5" font-weight="700" fill="var(--accent-text)">TARGET: one as-of-time module</text>
|
||||
<rect x="360" y="65" width="190" height="56" rx="10" fill="var(--accent-soft)" stroke="var(--accent)"/>
|
||||
<text x="455" y="88" text-anchor="middle" font-size="12.5" font-weight="600" fill="var(--fg)">population.py</text>
|
||||
<text x="455" y="105" text-anchor="middle" font-size="11" fill="var(--muted)">open-knowable inputs only</text>
|
||||
<rect x="660" y="28" width="170" height="24" rx="6" fill="var(--card)" stroke="var(--line)"/><text x="745" y="44" text-anchor="middle" font-size="11" fill="var(--fg)">live gate</text>
|
||||
<rect x="660" y="58" width="170" height="24" rx="6" fill="var(--card)" stroke="var(--line)"/><text x="745" y="74" text-anchor="middle" font-size="11" fill="var(--fg)">tuner + sweeps</text>
|
||||
<rect x="660" y="88" width="170" height="24" rx="6" fill="var(--card)" stroke="var(--line)"/><text x="745" y="104" text-anchor="middle" font-size="11" fill="var(--fg)">eod_replay (scheduled)</text>
|
||||
<rect x="660" y="118" width="170" height="24" rx="6" fill="var(--card)" stroke="var(--line)"/><text x="745" y="134" text-anchor="middle" font-size="11" fill="var(--fg)">labeler / Experiment OS</text>
|
||||
<line x1="550" y1="80" x2="652" y2="42" stroke="#8892a0" stroke-width="1.5" marker-end="url(#arr2)"/>
|
||||
<line x1="550" y1="90" x2="652" y2="70" stroke="#8892a0" stroke-width="1.5" marker-end="url(#arr2)"/>
|
||||
<line x1="550" y1="100" x2="652" y2="100" stroke="#8892a0" stroke-width="1.5" marker-end="url(#arr2)"/>
|
||||
<line x1="550" y1="110" x2="652" y2="130" stroke="#8892a0" stroke-width="1.5" marker-end="url(#arr2)"/>
|
||||
</svg>
|
||||
<figcaption>Every backtest, tune, and experiment verdict becomes trustworthy by construction when all consumers share the live gate.</figcaption>
|
||||
</figure>
|
||||
</article>
|
||||
|
||||
<article class="idea" id="i3">
|
||||
<h3>3. Entry-path unification + decision receipts</h3>
|
||||
<div class="chips"><span class="chip-axis">money-path-correctness</span><span class="chip-conf">Confidence 85%</span><span class="chip-cx">Complexity Medium</span></div>
|
||||
<dl class="fields">
|
||||
<dt>Description</dt>
|
||||
<dd>Collapse the two drifted entry paths — slow auto-approval (<code>request_approvals_for_ready_signals</code>) and fast trigger (<code>submit_breakout_now</code>) — onto one shared pure gate function. Every gate evaluation emits a compact <em>decision receipt</em> event (gate name, inputs, pass/fail), so "why did/didn't it trade X?" becomes a query instead of a forensic session.</dd>
|
||||
<dt>Basis</dt>
|
||||
<dd><code>direct:</code> verifier-confirmed: <code>_population_reject</code>/<code>_outside_session</code>/<code>_anti_chase_skip</code> are already shared by both paths, but the VWAP check exists only in the slow path (~<code>trading_execution.py:938</code>) and <code>rank_risk_factor</code> only in the fast path (<code>:1228</code>); the boot log itself warns "rank-concentration is enforced ONLY on the dead fast-trigger path, NOT the live auto-approval path" (<code>run_live_paper.py:1026-1028</code>). The daily-cap counted 44-vs-5 incident (fixed ac1ac50) was this same drift class.</dd>
|
||||
<dt>Rationale</dt>
|
||||
<dd>Half the convergence is already done — this finishes it so every future gate (including any Experiment-OS winner promoted to live) lands on both paths by construction. Receipts double as the raw material Experiment OS replays and the operator reads on the dashboard.</dd>
|
||||
<dt>Downsides</dt>
|
||||
<dd>Touches the live money path — needs the CI gate (idea 5) and characterization tests first. Receipt events add volume to the events table (bounded; they replace ad-hoc diagnostic events that retention already prunes).</dd>
|
||||
</dl>
|
||||
</article>
|
||||
|
||||
<article class="idea" id="i4">
|
||||
<h3>4. Money-path hardening — self-restart, positive-evidence contract, EOD cage count</h3>
|
||||
<div class="chips"><span class="chip-axis">money-path-correctness</span><span class="chip-conf">Confidence 95%</span><span class="chip-cx">Complexity Low</span></div>
|
||||
<dl class="fields">
|
||||
<dt>Description</dt>
|
||||
<dd>Three small, high-severity fixes sharing one principle — never act on assumed state. (a) <strong>Daily self-restart</strong>: <code>exit(0)</code> after a successful EOD flatten and let compose's <code>restart: unless-stopped</code> resurrect a fresh process — the process <em>is</em> the trading session. (b) <strong>Positive-evidence contract</strong>: no destructive market action on an ambiguous broker read — the exit manager must distinguish "orders call failed / silently empty" from "position confirmed naked" before market-flattening, and a lost order-POST must reconcile via <code>client_order_id</code> lookup before journaling "cancelled". (c) <strong>EOD cage count</strong>: a nightly reconcile that diffs broker positions/orders/fills against event-store truth and alerts on any discrepancy, plus SIGTERM handled as clean shutdown (<code>docker stop</code> currently skips the shutdown path).</dd>
|
||||
<dt>Basis</dt>
|
||||
<dd><code>direct:</code> verifier-confirmed: <code>_eod_done</code>/<code>_catchup_done</code> (<code>run_live_paper.py:1064,1113</code>) and <code>_session_closed</code>/<code>_daily_loss_breach</code> (<code>trading_execution.py:398,612</code>) never reset — an uninterrupted day-2 takes zero entries and has no EOD flatten, masked today only by frequent redeploys; <code>exit_manager.py:71-72</code> returns None on any exception and treats it identically to "naked", flattening after ~24s grace; <code>execution.py:124-129</code> journals lost POSTs as cancelled with no reconcile (no <code>by_client_order_id</code> method exists in the client); the 2026-07-06 duplicate-resting-order incident is corroborated by d54a2a9's own commit message; recovery-on-boot (<code>_rebuild_daily_state</code>) already exists and is tested.</dd>
|
||||
<dt>Rationale</dt>
|
||||
<dd>Highest severity-to-effort ratio in the whole set. (a) alone converts an entire recurring bug class (session gate, after-hours submissions, bench state — each patched separately before) into a non-issue by construction, in ~5 lines plus tests.</dd>
|
||||
<dt>Downsides</dt>
|
||||
<dd>Self-restart must fire only after a <em>successful</em> flatten (exit-on-failure would orphan positions — though broker-side stop legs keep them protected); the positive-evidence branch adds one more state to exit-manager logic that is already subtle.</dd>
|
||||
</dl>
|
||||
</article>
|
||||
|
||||
<article class="idea" id="i5">
|
||||
<h3>5. Production safety rails — CI gate, dead-man's switch, automated backup</h3>
|
||||
<div class="chips"><span class="chip-axis">operability-delivery</span><span class="chip-conf">Confidence 95%</span><span class="chip-cx">Complexity Low</span></div>
|
||||
<dl class="fields">
|
||||
<dt>Description</dt>
|
||||
<dd>The "vacation test" bundle: (a) CI runs the 376-test suite (plus <code>ruff check --select F</code> for the untested script layer) <em>before</em> the image workflow pushes <code>:latest</code> to GHCR/Docker Hub; (b) the always-up qabot container gets a dead-man's switch — alert when the event-store heartbeat goes stale during market hours, plus a nightly OK-ping (the loop can't report its own death; the qabot is the independent observer); (c) a scheduled <code>pg_dump</code> (or documented ZFS snapshot task) — production currently has <strong>no automated backup</strong>.</dd>
|
||||
<dt>Basis</dt>
|
||||
<dd><code>direct:</code> verifier-confirmed: <code>.github/workflows/docker.yml</code> has zero test step and ships on every master push; <code>api/main.py</code> at HEAD references <code>os</code>/<code>hmac</code> without importing them (AST-confirmed) — an untested regression that already shipped; loop-down currently means alerts-down (the notifier lives inside the loop process, <code>run_live_paper.py:1034</code>); <code>pg_dump</code> appears only in the one-time migration script.</dd>
|
||||
<dt>Rationale</dt>
|
||||
<dd>This is the lever that makes every other lever safe to ship. Every subsequent idea in this list lands as an image push; today each push is an untested bet against the production money path.</dd>
|
||||
<dt>Downsides</dt>
|
||||
<dd>CI needs a Postgres service container for the suite (~3–4 min per push). Dead-man thresholds need market-calendar awareness to avoid weekend false alarms — the holiday-aware calendar already exists in <code>runtime/market_calendar.py</code>.</dd>
|
||||
</dl>
|
||||
</article>
|
||||
|
||||
<article class="idea" id="i6">
|
||||
<h3>6. Operator control plane — dashboard repair, kill switch, effective-config panel</h3>
|
||||
<div class="chips"><span class="chip-axis">operator-surface</span><span class="chip-conf">Confidence 85%</span><span class="chip-cx">Complexity Medium</span></div>
|
||||
<dl class="fields">
|
||||
<dt>Description</dt>
|
||||
<dd>Make the production :8765 surface tell the truth and accept commands. Repairs: fix the POST <code>NameError</code> (unimported <code>os</code>/<code>hmac</code>); wire the research DB into the standalone dashboard so sparklines work in production; gate action buttons on the <code>has_execution</code> flag the API already ships; derive the LIVE badge from broker truth (it currently keys on a <code>schwab_live</code> literal the admin go-live flow never sets — it would read PAPER while trading real money). Capability: manual flatten (all / per-symbol) and pause-entries as guarded <em>operator-command events</em> through the store — the loop polls its own 5s guard cadence and executes with the existing <code>cancel_protective_and_close</code> helper; dashboard and qabot become thin command emitters. Efficiency: replace the per-client per-second fresh-connection + full-DDL + <code>COUNT(*)</code> SSE change detector with one shared cached signal. Visibility: an effective-config panel showing the merged config per service with per-key provenance (baked env → compose env → <code>overrides.env</code> → learned params) and drift-vs-inventory warnings.</dd>
|
||||
<dt>Basis</dt>
|
||||
<dd><code>direct:</code> verifier-confirmed: <code>telegram_bot.py:29</code> hard-refuses action words; <code>dashboard_api.py:36</code> builds the production dashboard with <code>execution_service=None</code> — no manual flatten path exists anywhere in the production UI; <code>dashboard.html:1061</code> keys LIVE on <code>schwab_live</code> while <code>admin_api.py:276-295</code> only swaps Alpaca URL/keys; SSE poll confirmed at <code>api/main.py:355</code>; concurrency-gauge default drift (3 vs 6) confirmed at <code>trading_execution.py:75</code> vs <code>projections.py:869</code>.</dd>
|
||||
<dt>Rationale</dt>
|
||||
<dd>The operator's two moments of need — "something is wrong, get me flat NOW" and "what is this thing actually running?" — both currently require shelling into a container. Command-as-event reuses the store as the bus (no new API surface into the loop) and leaves an audit trail by construction.</dd>
|
||||
<dt>Downsides</dt>
|
||||
<dd>A kill switch is a live-money control: needs HMAC/token guard, idempotency, and a test that a replayed or stale command cannot re-fire. Command latency is bounded by the 5s guard cadence (acceptable; broker stops remain the fast layer).</dd>
|
||||
</dl>
|
||||
</article>
|
||||
|
||||
<article class="idea" id="i7">
|
||||
<h3>7. Dead-weight deletion sweep — every table names its reader or dies</h3>
|
||||
<div class="chips"><span class="chip-axis">operability-delivery</span><span class="chip-conf">Confidence 90%</span><span class="chip-cx">Complexity Low</span></div>
|
||||
<dl class="fields">
|
||||
<dt>Description</dt>
|
||||
<dd>Finish the deletion d54a2a9 claimed: <code>schwab/</code> is still fully tracked (23 files) despite the commit message. Then the rest of the confirmed-dead surface: <code>comprehensive_backtest.py</code> (crashes on import — <code>research/connection.py</code> doesn't exist), <code>strategy/regime.py</code>, <code>universe.py</code>, <code>backtest/outcomes.py</code>, the ignition detector, the <code>grafana/</code> stack (points at the decommissioned pre-migration DB with committed plaintext dev credentials), the <code>telemetry/</code> husk, <code>docker-publish.yml</code> (fires on a nonexistent <code>main</code> branch, pushes a wrong image name), stale <code>uv.lock</code>, the empty <code>test_alpaca_client.py</code>, 12 zero-reference tables and 6 dead projections. Standing rule going forward: every table names its reader or dies — <code>quote_snapshots</code> gets retention or deletion; <code>float_cache</code>/<code>dilution_events</code> get wired into the labeler's features or stop being collected nightly.</dd>
|
||||
<dt>Basis</dt>
|
||||
<dd><code>direct:</code> verifier-confirmed each item at HEAD, including <code>git ls-files schwab/</code> (23 tracked files) vs d54a2a9's message, the import crash, the grafana datasource credentials, and grep-confirmed zero readers for the write-only tables.</dd>
|
||||
<dt>Rationale</dt>
|
||||
<dd>Every dead artifact taxes every future session ("which one is real?") and the grafana credentials are a small standing security smell. Deletion is the cheapest enhancement in the set and makes every subsequent plan smaller.</dd>
|
||||
<dt>Downsides</dt>
|
||||
<dd>The schwab/ decision should be explicit (its only non-delete rationale is the dormant human-vs-bot comparison); deleting tables needs a backup first (idea 5c) and a one-session soak.</dd>
|
||||
</dl>
|
||||
</article>
|
||||
|
||||
<h2 id="rejection-summary">Rejection Summary</h2>
|
||||
<p>Nothing was refuted: the fresh-context verifier confirmed all 14 merged candidates sound, and all 48 raw ideas were absorbed into the 7 survivors. The table records the merges and the judgment calls.</p>
|
||||
<div class="tablewrap">
|
||||
<table>
|
||||
<tr><th>#</th><th>Idea</th><th>Disposition</th></tr>
|
||||
<tr><td>1</td><td>Daily self-restart (raw #1, 9, 18, 25, 33 — five frames converged independently)</td><td>Merged into survivor 4a — strongest single convergence in the run</td></tr>
|
||||
<tr><td>2</td><td>Broker-ambiguity fixes / MCAS rule / "empty is not naked" (raw #2, 26, 37)</td><td>Merged into survivor 4b as one shared contract</td></tr>
|
||||
<tr><td>3</td><td>EOD cage-count reconciliation (raw #27)</td><td>Folded into survivor 4c — thinnest-sourced candidate (analogy + one past incident); kept as backstop, not standalone</td></tr>
|
||||
<tr><td>4</td><td>Verdict watchdog / DSMB / experiment ledger / experiment-as-config / Tesla shadow mode / counterfactual exit ledger (raw #4, 13, 22, 30, 34, 43, 44)</td><td>Merged into survivor 1 (Experiment OS) — the run's flagship synthesis</td></tr>
|
||||
<tr><td>5</td><td>Entry-conversion shortlist experiments (raw #5, 38) and one-bullet rank counterfactual (raw #48)</td><td>Merged into survivor 1 as the first experiment cohort</td></tr>
|
||||
<tr><td>6</td><td>Population parity / flywheel scheduling / "replay whole market nightly" (raw #3, 11, 12, 20, 21, 28, 35, 47)</td><td>Merged into survivor 2</td></tr>
|
||||
<tr><td>7</td><td>Kill switch variants: qabot /flatten, fire handle, command-bus events (raw #8, 15, 32, 36)</td><td>Merged into survivor 6 — command-as-event chosen as the unifying mechanism</td></tr>
|
||||
<tr><td>8</td><td>LISTEN/NOTIFY dashboard change signal (raw #16)</td><td>Demoted to implementation option inside survivor 6 — shared cached signal may be enough (lazy option first)</td></tr>
|
||||
<tr><td>9</td><td>Effective-config panel (raw #46)</td><td>Merged into survivor 6</td></tr>
|
||||
<tr><td>10</td><td>CI gate / vacation test / eye-in-the-sky (raw #6, 7, 17, 31, 41, 42)</td><td>Merged into survivor 5</td></tr>
|
||||
<tr><td>11</td><td>Deletion sweeps (raw #14, 24, 39)</td><td>Merged into survivor 7</td></tr>
|
||||
<tr><td>—</td><td>Axis coverage</td><td>All five axes have survivors; no recovery dispatch was needed</td></tr>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<footer class="composition-signal">Composed 2026-07-13 by ce-ideate (repo-grounded) from an 8-agent recon sweep + 5-frame ideation fleet + fresh-context basis verification, at momentum commit <code>d54a2a9</code>. Scratch: <code>/tmp/compound-engineering/ce-ideate/a3f9c2e1/</code></footer>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,142 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<mxfile host="drawio" version="26.0.0">
|
||||
<diagram name="Current architecture">
|
||||
<mxGraphModel dx="1420" dy="900" grid="1" gridSize="10" page="1" pageWidth="1460" pageHeight="920">
|
||||
<root>
|
||||
<mxCell id="0" />
|
||||
<mxCell id="1" parent="0" />
|
||||
|
||||
<mxCell id="title" value="Momentum — current production architecture (TrueNAS compose, 2026-07 @ d54a2a9)" style="text;html=1;fontSize=18;fontStyle=1;align=left;" vertex="1" parent="1">
|
||||
<mxGeometry x="40" y="0" width="900" height="30" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="browser" value="Operator browser" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1">
|
||||
<mxGeometry x="60" y="60" width="180" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="telegram" value="Telegram" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1">
|
||||
<mxGeometry x="300" y="60" width="160" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="feeds" value="Data feeds
Alpaca news · RSS · FMP float · SEC EDGAR" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1">
|
||||
<mxGeometry x="700" y="60" width="240" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="alpaca" value="Alpaca Paper API
orders · account · SIP bars/quotes" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="1">
|
||||
<mxGeometry x="1040" y="60" width="240" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="compose" value="TrueNAS SCALE — Docker Compose stack (image: fez5587/momentum-app)" style="swimlane;startSize=30;html=1;fillColor=none;strokeColor=#666666;fontStyle=1;" vertex="1" parent="1">
|
||||
<mxGeometry x="40" y="180" width="1340" height="580" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="dashboard" value="Dashboard :8765
static JS + SSE
(read-only projections)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="compose">
|
||||
<mxGeometry x="30" y="60" width="210" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="admin" value="Admin :8767
config overrides · profiles
service control" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="compose">
|
||||
<mxGeometry x="30" y="170" width="210" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="qabot" value="Telegram qabot
status Q&A (read-only)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="compose">
|
||||
<mxGeometry x="30" y="280" width="210" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="pg" value="PostgreSQL 18
events (append-only)
minute_bars (~19M) · research tables" style="shape=cylinder3;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;boundedLbl=1;" vertex="1" parent="compose">
|
||||
<mxGeometry x="480" y="160" width="240" height="130" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="nightly" value="Nightly scheduler
premarket_eval 20:00 ET
retention · shadow tracks · caches" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="compose">
|
||||
<mxGeometry x="480" y="380" width="240" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="loop" value="Trading loop — run_live_paper.py" style="swimlane;startSize=30;html=1;fillColor=#f8f8f8;strokeColor=#6c8ebf;fontStyle=1;" vertex="1" parent="compose">
|
||||
<mxGeometry x="810" y="50" width="490" height="350" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="schedthread" value="Scheduler thread
discover · ingest · watch · arm · guard" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="loop">
|
||||
<mxGeometry x="20" y="40" width="210" height="70" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="trigthread" value="Trigger thread (4s)
live cross → submit_breakout_now" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="loop">
|
||||
<mxGeometry x="260" y="40" width="210" height="70" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="exitmgr" value="LiveExitManager (12s)
BE ratchet · trail · EOD flatten" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="loop">
|
||||
<mxGeometry x="20" y="140" width="210" height="70" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="execsvc" value="TradingExecutionService
gates · sizing · bracket orders" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="loop">
|
||||
<mxGeometry x="260" y="140" width="210" height="70" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="syncsvc" value="AlpacaPaperSync (60s)
account · positions · orders · journal" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="loop">
|
||||
<mxGeometry x="140" y="250" width="210" height="70" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="e1" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="browser" target="dashboard">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="e2" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.1;exitY=1;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="browser" target="admin">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="50" y="390" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="e3" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="telegram" target="qabot">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="380" y="500" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="e4" value="projections (read)" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.25;entryDx=0;entryDy=0;" edge="1" parent="1" source="dashboard" target="pg">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="e5" value="reads" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.75;entryDx=0;entryDy=0;" edge="1" parent="1" source="qabot" target="pg">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="e6" value="events · bars" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=0;exitY=0.5;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="loop" target="pg">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="e7" value="writes · purges" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="1" source="nightly" target="pg">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="e8" value="REST: bars · quotes · orders · fills" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="alpaca" target="loop">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="e9" value="ingest" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.1;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="feeds" target="loop">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<mxPoint as="offset" x="-10" y="0" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="e10" value="overrides.env · restart (docker sock)" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;dashed=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=1;exitY=0.9;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="1" source="admin" target="loop">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="460" y="690" />
|
||||
<mxPoint x="1100" y="690" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="legend" value="Legend" style="rounded=0;whiteSpace=wrap;html=1;fillColor=none;strokeColor=#666666;verticalAlign=top;fontStyle=1;" vertex="1" parent="1">
|
||||
<mxGeometry x="60" y="790" width="200" height="120" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="leg1" value="" style="rounded=0;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="legend">
|
||||
<mxGeometry x="10" y="26" width="30" height="14" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="leg1t" value="Service / thread" style="text;html=1;align=left;verticalAlign=middle;fontSize=11;" vertex="1" parent="legend">
|
||||
<mxGeometry x="50" y="24" width="140" height="18" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="leg2" value="" style="rounded=0;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="legend">
|
||||
<mxGeometry x="10" y="48" width="30" height="14" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="leg2t" value="Datastore" style="text;html=1;align=left;verticalAlign=middle;fontSize=11;" vertex="1" parent="legend">
|
||||
<mxGeometry x="50" y="46" width="140" height="18" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="leg3" value="" style="rounded=0;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="legend">
|
||||
<mxGeometry x="10" y="70" width="30" height="14" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="leg3t" value="Scheduled job" style="text;html=1;align=left;verticalAlign=middle;fontSize=11;" vertex="1" parent="legend">
|
||||
<mxGeometry x="50" y="68" width="140" height="18" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="leg4" value="" style="rounded=0;html=1;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="legend">
|
||||
<mxGeometry x="10" y="92" width="30" height="14" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="leg4t" value="Broker API (money path)" style="text;html=1;align=left;verticalAlign=middle;fontSize=11;" vertex="1" parent="legend">
|
||||
<mxGeometry x="50" y="90" width="140" height="18" as="geometry" />
|
||||
</mxCell>
|
||||
</root>
|
||||
</mxGraphModel>
|
||||
</diagram>
|
||||
</mxfile>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 82 KiB |
@@ -0,0 +1,117 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<mxfile host="drawio" version="26.0.0">
|
||||
<diagram name="Milestone feature set">
|
||||
<mxGraphModel dx="1420" dy="900" grid="1" gridSize="10" page="1" pageWidth="1540" pageHeight="900">
|
||||
<root>
|
||||
<mxCell id="0" />
|
||||
<mxCell id="1" parent="0" />
|
||||
|
||||
<mxCell id="title" value="Momentum — next-milestone feature set: 7 workstreams and their dependencies" style="text;html=1;fontSize=18;fontStyle=1;align=left;" vertex="1" parent="1">
|
||||
<mxGeometry x="40" y="0" width="960" height="30" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="ph0" value="FOUNDATIONS — ship first" style="text;html=1;fontSize=12;fontStyle=1;fontColor=#888888;align=center;" vertex="1" parent="1">
|
||||
<mxGeometry x="60" y="50" width="300" height="20" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="ph1" value="SUBSTRATE — trust the data" style="text;html=1;fontSize=12;fontStyle=1;fontColor=#888888;align=center;" vertex="1" parent="1">
|
||||
<mxGeometry x="500" y="50" width="300" height="20" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="ph2" value="FLAGSHIP" style="text;html=1;fontSize=12;fontStyle=1;fontColor=#888888;align=center;" vertex="1" parent="1">
|
||||
<mxGeometry x="920" y="50" width="280" height="20" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="ph3" value="OPERATOR SURFACE" style="text;html=1;fontSize=12;fontStyle=1;fontColor=#888888;align=center;" vertex="1" parent="1">
|
||||
<mxGeometry x="1280" y="50" width="220" height="20" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="ws5" value="WS5 · Production safety rails [S]

CI pytest gate before :latest
qabot dead-man's switch
scheduled pg_dump backup" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;align=left;spacingLeft=10;verticalAlign=top;spacingTop=6;" vertex="1" parent="1">
|
||||
<mxGeometry x="60" y="90" width="300" height="120" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="ws7" value="WS7 · Dead-weight deletion [S]

schwab/ · grafana/ · telemetry/ · dead strategy modules
12 zero-ref tables · 6 dead projections
rule: every table names its reader or dies" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;align=left;spacingLeft=10;verticalAlign=top;spacingTop=6;" vertex="1" parent="1">
|
||||
<mxGeometry x="60" y="280" width="300" height="130" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="ws4" value="WS4 · Money-path hardening [S]

daily self-restart (exit 0 after EOD flatten)
positive-evidence contract (empty ≠ naked;
lost POST → by_client_order_id reconcile)
EOD broker-vs-events cage count · SIGTERM" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#ffe6cc;strokeColor=#d79b00;align=left;spacingLeft=10;verticalAlign=top;spacingTop=6;" vertex="1" parent="1">
|
||||
<mxGeometry x="60" y="480" width="300" height="140" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="ws2" value="WS2 · Learning-flywheel resurrection [M]

ONE population module (as-of-time, open-knowable)
consumers: live gate · tuner · sweeps · replay · labeler
schedule eod_replay + nightly_tune in production
atomic learned_params write + staleness alert" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;align=left;spacingLeft=10;verticalAlign=top;spacingTop=6;" vertex="1" parent="1">
|
||||
<mxGeometry x="500" y="120" width="330" height="150" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="ws3" value="WS3 · Entry-path unification [M]

one shared gate function, both entry paths
decision-receipt event per gate pass/fail
(closes VWAP/quality vs rank drift)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;align=left;spacingLeft=10;verticalAlign=top;spacingTop=6;" vertex="1" parent="1">
|
||||
<mxGeometry x="500" y="420" width="330" height="130" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="ws1" value="WS1 · Experiment OS [L]

experiment = config row (population · intervention ·
pre-registered bars · min-n · futility)
counterfactual replay of every logged fire
counterfactual exit ledger (all exit variants per trade)
auto verdicts: PASS / FAIL / INCONCLUSIVE

first cohort: 10-min time-stop · probe-then-add
asymmetric exit (spread floor) · one-bullet rank · pm1 verdict" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;align=left;spacingLeft=10;verticalAlign=top;spacingTop=6;fontStyle=1" vertex="1" parent="1">
|
||||
<mxGeometry x="920" y="220" width="320" height="230" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="ws6" value="WS6 · Operator control plane [M]

dashboard POST fix · sparklines · LIVE badge truth
kill switch: operator-command events
(flatten all / symbol · pause entries)
experiment ledger panel
effective-config panel (per-key provenance)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;align=left;spacingLeft=10;verticalAlign=top;spacingTop=6;" vertex="1" parent="1">
|
||||
<mxGeometry x="1340" y="230" width="190" height="210" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="d1" value="trusted population" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.25;entryDx=0;entryDy=0;" edge="1" parent="1" source="ws2" target="ws1">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="d2" value="decision receipts" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.75;entryDx=0;entryDy=0;" edge="1" parent="1" source="ws3" target="ws1">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="d3" value="verdict events" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="ws1" target="ws6">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<mxPoint as="offset" x="0" y="-18" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="d4" value="safe ships" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;dashed=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="ws5" target="ws2">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="d5" value="smaller surface" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;dashed=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="ws7" target="ws3">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="d6" value="promoted winners land on the unified gate" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;dashed=1;labelBackgroundColor=#ffffff;fontSize=11;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="1" source="ws1" target="ws3">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="1090" y="620" />
|
||||
<mxPoint x="665" y="620" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="note4" value="WS4 is independent — ship in week 1 alongside WS5/WS7" style="text;html=1;fontSize=11;fontColor=#888888;align=left;" vertex="1" parent="1">
|
||||
<mxGeometry x="60" y="630" width="330" height="20" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="legend" value="Legend" style="rounded=0;whiteSpace=wrap;html=1;fillColor=none;strokeColor=#666666;verticalAlign=top;fontStyle=1;" vertex="1" parent="1">
|
||||
<mxGeometry x="500" y="680" width="360" height="140" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="lg1" value="" style="rounded=0;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="legend">
|
||||
<mxGeometry x="10" y="26" width="30" height="14" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="lg1t" value="Foundation (ops / hygiene)" style="text;html=1;align=left;verticalAlign=middle;fontSize=11;" vertex="1" parent="legend">
|
||||
<mxGeometry x="50" y="24" width="300" height="18" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="lg2" value="" style="rounded=0;html=1;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="legend">
|
||||
<mxGeometry x="10" y="48" width="30" height="14" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="lg2t" value="Money-path correctness" style="text;html=1;align=left;verticalAlign=middle;fontSize=11;" vertex="1" parent="legend">
|
||||
<mxGeometry x="50" y="46" width="300" height="18" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="lg3" value="" style="rounded=0;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="legend">
|
||||
<mxGeometry x="10" y="70" width="30" height="14" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="lg3t" value="Substrate (data / gate integrity)" style="text;html=1;align=left;verticalAlign=middle;fontSize=11;" vertex="1" parent="legend">
|
||||
<mxGeometry x="50" y="68" width="300" height="18" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="lg4" value="" style="rounded=0;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="legend">
|
||||
<mxGeometry x="10" y="92" width="30" height="14" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="lg4t" value="Flagship build (Experiment OS)" style="text;html=1;align=left;verticalAlign=middle;fontSize=11;" vertex="1" parent="legend">
|
||||
<mxGeometry x="50" y="90" width="300" height="18" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="lg5" value="" style="rounded=0;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="legend">
|
||||
<mxGeometry x="10" y="114" width="30" height="14" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="lg5t" value="Operator surface (:8765 / :8767 / qabot)" style="text;html=1;align=left;verticalAlign=middle;fontSize=11;" vertex="1" parent="legend">
|
||||
<mxGeometry x="50" y="112" width="300" height="18" as="geometry" />
|
||||
</mxCell>
|
||||
</root>
|
||||
</mxGraphModel>
|
||||
</diagram>
|
||||
</mxfile>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 132 KiB |
@@ -0,0 +1,29 @@
|
||||
# Data-source key inventory (2026-07-18)
|
||||
|
||||
All keys below are in `.env` (gitignored). Most were provisioned but never
|
||||
wired into code — the system uses only Alpaca (bars/quotes), FMP (float),
|
||||
EDGAR/SEC full-text (filings), and RSS+Ollama (news). This catalogs what's
|
||||
available so a future need pulls the right one instead of assuming absence.
|
||||
|
||||
**Triage — what's worth wiring, and what's redundant.** Do NOT wire all of
|
||||
these; more sources = more complexity, not more edge (news was already only
|
||||
weakly useful). Wire a source only when a specific hypothesis needs it.
|
||||
|
||||
| Key | What it is | Verdict |
|
||||
|---|---|---|
|
||||
| `DATABENTO_API_KEY` | **Nasdaq TotalView L2 depth (MBP-10)**, pay-as-you-go + free credits | **GAME-CHANGER — unblocks plan 017 Phase 0 at ~$0. Verified live 2026-07-18: covers the real gapper universe, fractions of a cent per fire.** |
|
||||
| `POLYGON_API_KEY` | aggregated bars/quotes, some depth | Nice-to-have: cross-check / fallback for Alpaca bars; not urgent |
|
||||
| `SEC_API_KEY` | sec-api.io — STRUCTURED filings (vs raw EDGAR full-text the dilution feed uses now) | Minor upgrade to `research/ingestion/sec_dilution.py` if that feed ever needs form-subtype precision; the raw feed already works |
|
||||
| `Financial_Modeling_Prep_Api_Key` | FMP — float/fundamentals | **Already wired** (float_cache) |
|
||||
| `FINNHUB_API_KEY` | news + fundamentals + some float | Redundant with FMP/news pipe |
|
||||
| `Alpha_Vantage_Api_Key`, `Twelve_Data_Api_Key`, `Marketstack_Api_Key`, `EOD_Historical_Data_Api_Key` | bars/fundamentals/float, free tiers | Redundant — float refuted as a gate; bars come from Alpaca SIP |
|
||||
| `NewsAPI_Api_Key`, `NewsData.io_Api_Key`, `Marketaux_Api_Key`, `MediaStack_Api_Key` | news feeds, free tiers | SKIP for now — 4 news APIs is complexity, not edge; RSS+Ollama already covers catalyst. Revisit only if a news-LATENCY hypothesis is registered |
|
||||
|
||||
**Bottom line:** one key (Databento) materially advances the project's founding
|
||||
question (can premarket data give autonomous selection confidence — via the
|
||||
order book). The rest are redundancy or unproven-need. Discipline over
|
||||
accumulation: wire on demand, not on availability.
|
||||
|
||||
**Security:** these keys were pasted into a chat session 2026-07-18 — rotate
|
||||
Databento + Polygon (billing feeds) when convenient. `.env` is gitignored;
|
||||
never commit this file's real values (it holds none — names only).
|
||||
@@ -0,0 +1,156 @@
|
||||
# Entry-conversion research: what demonstrably improves breakout entry conversion in thin small-caps
|
||||
|
||||
**Date:** 2026-07-02
|
||||
**Method:** deep-research fan-out — 5 parallel search angles (entry timing, confirmation filters, position management, stop placement, breakout-failure microstructure), ~20 primary sources fetched, load-bearing claims adversarially re-verified against primary sources.
|
||||
**Context this feeds:** the bot's proven bottleneck is ENTRY-CONVERSION (7 refuted selection levers; 76–84% of ORB fires poke-then-stop; the one +EV subset found in-house is breakouts confirming within ~10–15 min of the open, +0.42R).
|
||||
|
||||
**Evidence-quality labels used throughout:** `ACADEMIC` (peer-reviewed or serious preprint), `BACKTEST#` (non-academic but real numbers, reproducible-ish), `GURU` (practitioner claim, no verifiable numbers, often selling something), `SEO` (content-farm fluff).
|
||||
|
||||
---
|
||||
|
||||
## Headline synthesis
|
||||
|
||||
Three independent literatures converge on one mechanism for why our fires poke-then-die:
|
||||
|
||||
> The breakout level is where resting **supply** is maximal (take-profit clusters, ATM-offering desks, distributing pumpers), while stop-loss orders cluster just beyond it. Merely *reaching* the level most often produces a rejection — the poke is the statistically expected outcome — while a genuine *crossing* that absorbs the shelf ignites a real trend. The reach-vs-cross question resolves fast (rejections play out within ~30 min).
|
||||
|
||||
- Osler (NY Fed, `ACADEMIC`, numbers verified against the paper): take-profits cluster ON round-number levels (9.9% of TP orders vs 3.8% baseline), stop-losses just beyond (14.3% of stop-buys priced in the 01–10 digit band vs 6.9% in 90–99). Prices reverse abnormally often upon merely *reaching* a level (59.3% vs 54.8% at arbitrary prices), and that rejection completes within ~30 min; genuine crossings, by contrast, trend significantly for **at least two hours** (FX data). Tradable content: poke-rejection is fast and the base case; a confirmed cross carries durable momentum. [sr150.pdf](https://www.newyorkfed.org/medialibrary/media/research/staff_reports/sr150.pdf)
|
||||
- SmallCapLab (~3,000 small-cap gap-up events, `BACKTEST#`, short-biased vendor; verified with caveats): 64.2% of gap-ups close below their RTH open, ~72% below VWAP, **46.6% of highs-of-day print in the first 15 min, ~70% by 10:30** (the page's "85%+ by 10:30" caption contradicts its own cumulative chart at 69.6%; ~86% by 13:00); fade-severity buckets imply ~65% collapse ≥20% off HOD. The fade is the base case; the whole long game is the open. Universe caveat: gap ≥45% vs prior close, price ≥$0.30 — *bigger* gaps than the bot's ≤35%-gap universe, so directional only. [smallcaplab.com/research](https://www.smallcaplab.com/research)
|
||||
- Zarattini/Barbon/Aziz (`ACADEMIC` working paper): 5-min opening range Sharpe 2.81 vs 15-min 1.43 vs 30-min 0.21 — the edge **decays fast with later entry**. [SSRN 4729284](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4729284)
|
||||
|
||||
All three externally corroborate the in-house +0.42R early-confirm finding. The literature's answer to "how do winners enter?" is *not* a cleverer trigger — every credibly profitable published system enters **immediately on the break** — it is (a) only trading the first minutes, (b) an asymmetric exit structure (tight-ish stop, no target, let the fat tail run to EoD), and (c) not paying full risk on unconfirmed pokes.
|
||||
|
||||
---
|
||||
|
||||
## 1. Breakout entry timing: on-break vs retest vs first-pullback
|
||||
|
||||
### What the evidence says
|
||||
|
||||
**Enter-on-break is the only variant with credible supporting numbers.**
|
||||
|
||||
- Zarattini/Barbon/Aziz 2024 ([SSRN 4729284](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4729284), `ACADEMIC` working paper, verified against source): resting stop order AT the 5-min OR high — no confirmation, no retest. Top-20 opening-RVOL "Stocks in Play", 2016–2023, 7,000+ stocks: **+1,637% net, Sharpe 2.81, ann. alpha ~36%**. Independent QuantConnect replication: Sharpe 2.40, **~17% per-trade win rate** (profit rides rare 10R+ winners) ([QuantConnect study](https://www.quantconnect.com/research/18444/opening-range-breakout-for-stocks-in-play/), `BACKTEST#`). Caveats: universe is price>$5 / >1M sh avg volume / ATR>$0.50 — *liquid* stocks-in-play, not our thin sub-$5 tier; **zero slippage assumed**; both authors sell trading education.
|
||||
- The companion QQQ paper ([SSRN 4416622](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4416622)) likewise enters at the open of the 2nd 5-min candle with zero confirmation. Baseline: 24% win rate, +0.13R/trade.
|
||||
|
||||
**Retest / wait-for-pullback entries: the only direct head-to-head found says they are much worse.**
|
||||
|
||||
- Mesfin 2026 ([arXiv 2605.04004](https://arxiv.org/abs/2605.04004), `ACADEMIC` preprint, falsification-style with friction; numbers verified exact): MNQ futures, 947 days, 5-min bars, 25-min opening range. Immediate ORB entry: 51.9% win (fails net of friction; best immediate variant +2.82 pts at bar+15, still fails T≥2). **Pullback/retest entry: N=83, 19.3% win rate, 80.7% stop-out rate, −4.44 pts.** Paper's verdict: pullback entries are "systematically wrong" because "a high proportion of apparent breakouts simply fail and reverse." Caveats: index futures, unconditioned signal, small N, and the comparison is not perfectly apples-to-apples (pullback variant used a 20-pt stop while immediate variants used fixed-horizon exits) — but it is the only rigorous break-vs-retest comparison found anywhere.
|
||||
- Bulkowski (`BACKTEST#`, informal daily-bar pattern stats): throwbacks/pullbacks occur after ~60–74% of breakouts and patterns **with** throwbacks *underperform* those without — waiting for the retest adversely selects the weak breakouts and misses the ~30–40% strongest moves that never come back ([thepatternsite.com](https://thepatternsite.com/fallwedge.html)).
|
||||
- Mechanistically (Mesfin): entering a bar or more after the trigger is structurally post-exhaustion — expansion-bar continuation tested *actively wrong* (T=−10.96 at bar+1; measured in the Asia session, so context caveat applies); the move is largely contained in the trigger bar. Entry latency consumes the edge.
|
||||
|
||||
**First-pullback / bull-flag advocacy is evidence-free.** All numeric claims circulating for intraday small-cap flags (~52–75% win rates) trace to `SEO`/`GURU` content with no reproducible methodology. Warrior Trading's Ross Cameron publishes CPA-reviewed *aggregate* P&L ($18.8M 2017–2025 from $583) but **no audited per-setup statistics** — and the FTC settled with Warrior Trading for **$3M in 2022 over deceptive earnings claims** ([warriortrading.com earnings page](https://www.warriortrading.com/ross-camerons-verified-day-trading-earnings/), [FTC action]) — n=1 discretionary trader, course-seller conflict. This is consistent with our own no-chase VWAP-pullback backtest coming out negative after cost, and with the VWAP-reclaim shadow track's do-not-promote verdict.
|
||||
|
||||
### Verdict
|
||||
| Variant | Evidence | Direction |
|
||||
|---|---|---|
|
||||
| On-break (stop order at trigger) | ACADEMIC + replication | Only variant with a documented net edge (when paired with selection + exit structure) |
|
||||
| Wait-for-retest | ACADEMIC (1 study) + BACKTEST# | Materially worse; adverse selection |
|
||||
| First-pullback (flag) | GURU only | Untested anywhere credible; our own test negative |
|
||||
|
||||
**Testable rule (mostly a stop-doing rule):** do not build retest/pullback entry logic; keep the on-break trigger and attack conversion via the time window, exit structure, and sizing below.
|
||||
|
||||
---
|
||||
|
||||
## 2. Confirmation at the moment of entry (1-min OHLCV computable)
|
||||
|
||||
**The only confirmation filter with credible published numbers is opening relative volume**, and it is a *selection* filter, not a trigger filter: Zarattini et al. define RVOL = first-5-min volume ÷ 14-day average first-5-min volume. Trades with RVOL<100% averaged **−0.02R**; RVOL>100% averaged **+0.08R** (`ACADEMIC`, verified). Computable: yes (needs 14 days of per-symbol history). Note: this exact definition (same-minutes-of-day baseline) may differ from the bot's current RVOL — worth checking the implementation matches before assuming this box is ticked.
|
||||
|
||||
**Bar-close-above-trigger vs intrabar poke: empirically OPEN.** No published head-to-head with numbers exists despite targeted searching. Every "wait for the close above" claim traces to unbacktested blog assertions (`SEO`). Meanwhile the best-documented ORB system fills *intrabar* on the poke and works. Given Mesfin's latency finding (one bar late = post-exhaustion), waiting 1 minute for a close plausibly costs more than it saves — but this is cheaply testable in-house on logged fires.
|
||||
|
||||
**Close-location-in-range (CLR = (close−low)/(high−low)) carries a warning label:** on daily bars in liquid stocks, closing near the high is a *mean-reversion sell* signal — IBS>0.8 fade has worked for two decades (Alvarez, `BACKTEST#`, [IBS post](https://alvarezquanttrading.com/blog/internal-bar-strength-for-mean-reversion/)). High-RVOL gapper minutes may flip the sign, but "strong close = confirmation" is not a market universal.
|
||||
|
||||
**Academic base rate for volume confirmation is directional only:** Lo/Mamaysky/Wang 2000 and Gervais/Kaniel/Mingelgrin 2001 (`ACADEMIC`, peer-reviewed) show volume carries incremental information — at day-to-month horizons on 1962–1996 liquid stocks. No intraday entry-bar numbers. The practitioner heuristics (breakout-bar volume ≥1.2–1.3× 20-bar average; price above rising VWAP; CLR in top 30%) are coherent mechanics with **zero published before/after deltas** (`GURU`/`SEO`).
|
||||
|
||||
**Testable rules (pre-register, shadow only):**
|
||||
1. `confirm_close`: require the trigger-minute bar to CLOSE above trigger (vs current intrabar fire) — measure fill-price cost vs whipsaw avoided on logged fires.
|
||||
2. `vol_surge`: breakout-bar volume ≥ 2× median of prior 20 one-min bars.
|
||||
3. `clr`: trigger-bar CLR ≥ 0.7.
|
||||
Each judged on the standard bars (+1R rate, expectancy net of cost, vs same-day ORB baseline). Expectation set LOW: these are folklore until proven on our fires.
|
||||
|
||||
---
|
||||
|
||||
## 3. Position management: scaling in/out, probes, pyramids
|
||||
|
||||
**Scale-OUT (Warrior-style "sell partials into strength") is the best-refuted idea in this whole research pass.** Every source with actual numbers says it reduces expectancy; it buys variance-reduction and win-rate optics:
|
||||
|
||||
- Howard Bandy via Bulkowski (`BACKTEST#`, 4,176 trades, 72 liquid tickers 2003–2008): the closer the first target, the more average profit is reduced; BE-stop-after-partial cut profits to **one-third**. Bulkowski: "If you want to make less profit, then scale out of a trade." ([thepatternsite.com/ScalingOut.html](https://thepatternsite.com/ScalingOut.html))
|
||||
- Alvarez (`BACKTEST#`): scaling out → "large drop in CAR, no change or larger MDD" ([post](https://alvarezquanttrading.com/blog/adding-stops-and-scaling-out-to-a-mean-reversion-strategy/)).
|
||||
- Kevin Davey (`BACKTEST#`, 567k exit backtests, 40 futures markets): all six *combination* exits were consistently worse than simple single exits ([kjtradingsystems.com](https://kjtradingsystems.com/algo-trading-exits.html)).
|
||||
- The arithmetic identity: taking half off at +1R turns 2R winners into 1.5R; it only pays if edge *shrinks* as the move extends. Our book is the opposite regime — rare fat winners pay for everything (runner-grade forensics) — which makes the anti-scale-out math *stronger* for us. Warrior Trading's scale-out advice optimizes a discretionary human's psychology, not a bot's expectancy; they publish no backtest.
|
||||
|
||||
**Scale-IN (half-size probe at trigger + add on confirmation): theoretically right, empirically unpublished.** Kelly logic supports staged betting when confirmation genuinely raises P(win); trend-following pyramiding literature (Concretum, `BACKTEST#`, multi-decade futures) finds adds lower win rate but raise profit factor via 2–3× larger winners. **No credible published backtest of probe-then-add for intraday small-cap momentum exists** — either way. But it attacks exactly our measured problem: paying full risk on the 76–84% of fires that are unconfirmed pokes, while our own data says confirmation-within-10–15-min flips the subset to +0.42R.
|
||||
|
||||
**Testable rule (`probe_add`):** enter 0.5× normal size at trigger; add the remaining 0.5× only if within 10 min the position (a) has not touched the stop and (b) prints a new high above the trigger bar's high (or holds above VWAP). Compare blended R and net expectancy vs all-in baseline on the same fires. Trivially implementable with 1-min bars + two orders.
|
||||
|
||||
---
|
||||
|
||||
## 4. Stop placement for whipsaw-prone names
|
||||
|
||||
**Theory anchor (Kaminski & Lo, `ACADEMIC`, J. Financial Markets 2014):** under a random walk any stop-loss rule strictly lowers expected return; under positive serial correlation (momentum) stops add value ([SSRN 968338](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=968338)). So Alvarez's famous "stops hurt" results (`BACKTEST#`, mean-reversion daily bars: every 3–15% stop reduced CAR — [post](https://alvarezquanttrading.com/blog/maximum-loss-stops-do-you-really-need-them/)) do NOT transfer to our momentum entries; expect the opposite sign. Keep stops; the question is width and type.
|
||||
|
||||
**The best-evidenced production configuration is: tight volatility-fractional stop + NO profit target + end-of-day exit.**
|
||||
- Stocks-in-play paper: stop = **10% of 14-day ATR**, no target, EoD exit, size = 1% risk ÷ stop distance with 4× leverage cap → Sharpe 2.81. Per-trade win rate is LOW (~17% in the replication); the system is a low-win-rate, fat-right-tail machine. EoD-exit beat every fixed R-target in both Zarattini papers.
|
||||
- **The tight-stop artifact is published and author-admitted:** in the QQQ paper's sweep, the best cell was the tight end of the stop range (5% of ATR ≈ $0.08 on TQQQ, +9,350%) under **zero slippage**, and the authors themselves flag it verbatim as "unrealistic as our model assumed no slippage… will likely be exceeded." This is exactly the artifact we hit with the VR track. Any in-house tight-stop test must enforce a spread-aware floor: `stop_width ≥ max(k_ATR × ATR14, m × spread_proxy)` where spread_proxy for thin names can be estimated from 1-min high-low of quiet bars.
|
||||
- Note the accounting identity: with fixed-fractional sizing, stop width and position size are the same knob until a leverage/notional cap binds. "Tight stop, small dollar loss, huge size" is what makes the Zarattini machine work on liquid names — and what slippage destroys on thin ones. A calibrated middle (e.g. 20–30% of ATR14 with the spread floor) is the honest test.
|
||||
|
||||
**Breakeven moves: near-noise in published tests** (0.3% of trades touched in one study; "heavily cuts profitability" in another — `BACKTEST#`/`GURU` mixed). On whipsaw small caps a BE@+5% will scratch far more often — our shipped BE rule is worth measuring, not assuming.
|
||||
|
||||
**Time stops: evidence gap.** No credible published study of "exit if no progress in N minutes" for intraday small caps. EoD exit is the only well-evidenced time stop. But Osler's fast poke-rejection window (~30 min) + SmallCapLab's HOD-in-first-15-min curve + our own +0.42R early-confirm subset all point the same direction, making this the highest-prior in-house experiment:
|
||||
|
||||
**Testable rules:**
|
||||
1. `time_stop`: exit at market if the trade has not reached +0.5R (alt: has not printed a new post-entry high) within 10 min of fill. Measures: expectancy delta, avg loss on scratched trades vs their counterfactual stop-outs.
|
||||
2. `atr_stop_floor`: stop = max(0.25 × ATR14_daily_fraction, 3 × spread_proxy), no profit target, trail-to-EoD — Zarattini structure adapted with an honest fill floor; pre-register against the tight-stop artifact (require the edge to survive +1 tick adverse fill on entry AND stop).
|
||||
3. `eod_runner`: for confirmed trades (passed the 10-min gate), remove the profit target; hold with trailed structure stop into the last hour (Gao et al. intraday momentum: payoff concentrates late day — `ACADEMIC`, index-level caveat).
|
||||
|
||||
---
|
||||
|
||||
## 5. Why breakouts fail (microstructure) and what precedes the ones that hold
|
||||
|
||||
- **Reach-then-reject is the base case; real crossings trend** (Osler, `ACADEMIC`): stops cluster just beyond salient levels, take-profits sit on them. Prices that merely *reach* a level reverse abnormally often, with the rejection completing within ~30 min; prices that genuinely *cross* keep trending for hours (FX). The whipsaw is not noise — it is the statistically dominant outcome of touching a supply shelf, and the fast resolution window is why early confirmation is the +EV subset. (Note: an earlier read of this paper as "cascade fuel gone by 60 min" was checked against the source and inverted — post-crossing trends persist; it is the *rejection* that is fast.)
|
||||
- **The level itself is a supply shelf**: take-profits cluster ON round numbers (Osler); in dilution-overhung small caps, ATM desks sell into spikes (DilutionTracker, `GURU`, unquantified but mechanistically sound — and EDGAR S-3/424B5 presence is checkable, ties to the existing EDGAR-dilution roadmap item); in pumped names the breakout print *is* the manipulator's exit liquidity (Korea order-book study, `ACADEMIC`, [SSRN 4173353](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4173353)); heavily retail-herded names average **−4.7%** 20-day abnormal returns (Barber et al. JF 2022, `ACADEMIC`).
|
||||
- **Base rates say fade is the default** (SmallCapLab, `BACKTEST#`, vendor-biased, gap ≥45% universe): 64% of gappers close red vs open; only 41.5% ever break the premarket high; bigger gaps collapse harder; **higher premarket volume ⇒ harder fade** (56.2% → 65.6% → 71.5% across PM-volume buckets, small n in places) — externally corroborates our refuted RVOL/gap selection levers.
|
||||
- **Conditions preceding holds, computable from our bars:** minutes-since-open (dominant), premarket-high break (binary level of real significance), distance of trigger to round numbers ($0.50/$1.00), gap size (inverse), price tier ($10+ most resilient, sub-$1 fades hardest). Conditions that do NOT predict holding: premarket volume, RVOL beyond the >100% floor, float — consistent with all seven in-house refutations.
|
||||
|
||||
**Testable rule (`round_number`):** log distance from trigger price to nearest $0.50/$1.00 increment on every fire; test whether fires triggering just *below* a round number (into the take-profit shelf) underperform fires triggering just *above* one (through the stop cluster). Free to log, zero risk.
|
||||
|
||||
---
|
||||
|
||||
## Ranked shortlist: 3 most promising, implementable on 1-min OHLCV
|
||||
|
||||
**1. Time-boxed confirmation: entries only in the opening window + a 10-minute time-stop on unconfirmed positions.**
|
||||
Strongest convergence in the entire pass: our own +0.42R early-confirm subset, Osler's ~30-min poke-rejection resolution (`ACADEMIC`), SmallCapLab's 47%-of-HODs-in-first-15-min curve (`BACKTEST#`), and Zarattini's 5-min≫15-min≫30-min Sharpe decay (`ACADEMIC`). Rule: take ORB fires only ~9:30–9:50; exit at market any position that hasn't made a new post-entry high (or +0.5R) within 10 min. Directly converts the known 76–84% poke-then-die population from full-stop losses into small scratches. Cheapest to test: pure replay of logged fires.
|
||||
|
||||
**2. Probe-then-add sizing: 0.5× at trigger, add 0.5× only on 10-minute confirmation.**
|
||||
The only technique that structurally exploits the discovered +EV confirm-subset without needing to *predict* it. Theory-consistent (Kelly staged betting, pyramiding profit-factor results); no published intraday small-cap evidence either way, so shadow-first with pre-registered bars. Halves risk on the ~80% dead pokes at the cost of a worse blended entry on winners — the trade-off is measurable on existing fire logs before any live change.
|
||||
|
||||
**3. Asymmetric exit restructure: volatility-fractional stop with a spread floor, NO profit target, trail to EoD; kill any scale-out.**
|
||||
The published edge (Sharpe 2.81, replicated) lives in the exit asymmetry, not the entry: tight-ish stop, ~17–25% win rate, fat right tail runs to the close. Scale-out is refuted by every numeric source (Bandy/Alvarez/Davey). MUST be tested with an adverse-fill/spread floor (`stop ≥ max(0.25×ATR14, 3×spread_proxy)`, +1-tick adverse fills) — the tightest-stop-wins result is a documented zero-slippage artifact, the same artifact class that killed the VR and liq1 reads.
|
||||
|
||||
**Explicit do-nots from this pass:** no retest or first-pullback entry logic (refuted/evidence-free; matches our negative in-house test); no scale-out partials; no adoption of close-above-trigger or volume-multiple confirmation from folklore without an in-house pre-registered test.
|
||||
|
||||
---
|
||||
|
||||
## Source list
|
||||
|
||||
| Source | Type | URL |
|
||||
|---|---|---|
|
||||
| Zarattini, Barbon & Aziz — Profitable Day Trading Strategy (stocks-in-play ORB) | Academic WP | https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4729284 |
|
||||
| Zarattini & Aziz — Can Day Trading Really Be Profitable? (QQQ ORB) | Academic WP | https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4416622 |
|
||||
| QuantConnect replication of stocks-in-play ORB | Backtest | https://www.quantconnect.com/research/18444/opening-range-breakout-for-stocks-in-play/ |
|
||||
| Mesfin — ORB/retest falsification study, MNQ | Academic preprint | https://arxiv.org/abs/2605.04004 |
|
||||
| Osler — Stop-Loss Orders and Price Cascades (NY Fed SR150) | Academic | https://www.newyorkfed.org/medialibrary/media/research/staff_reports/sr150.pdf |
|
||||
| Kaminski & Lo — When Do Stop-Loss Rules Stop Losses? | Academic | https://papers.ssrn.com/sol3/papers.cfm?abstract_id=968338 |
|
||||
| Gao, Han, Li & Zhou — Intraday Momentum | Academic | https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2552752 |
|
||||
| Heston, Korajczyk & Sadka — intraday periodicity | Academic | https://papers.ssrn.com/sol3/papers.cfm?abstract_id=1107590 |
|
||||
| Lo, Mamaysky & Wang — Foundations of Technical Analysis | Academic | https://papers.ssrn.com/sol3/papers.cfm?abstract_id=228099 |
|
||||
| Gervais, Kaniel & Mingelgrin — High-Volume Return Premium | Academic | https://papers.ssrn.com/sol3/papers.cfm?abstract_id=146468 |
|
||||
| Barber, Huang, Odean & Schwarz — Attention-Induced Trading (Robinhood herding) | Academic | https://papers.ssrn.com/sol3/papers.cfm?abstract_id=3715077 |
|
||||
| Lee/Lee/Kim — pump-and-dump order-book study (Korea) | Academic | https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4173353 |
|
||||
| Bulkowski — Scaling Out (incl. Bandy study) | Backtest | https://thepatternsite.com/ScalingOut.html |
|
||||
| Alvarez — Adding Stops and Scaling Out (MR) | Backtest | https://alvarezquanttrading.com/blog/adding-stops-and-scaling-out-to-a-mean-reversion-strategy/ |
|
||||
| Alvarez — Maximum Loss Stops | Backtest | https://alvarezquanttrading.com/blog/maximum-loss-stops-do-you-really-need-them/ |
|
||||
| Alvarez — Internal Bar Strength | Backtest | https://alvarezquanttrading.com/blog/internal-bar-strength-for-mean-reversion/ |
|
||||
| Davey — exit study (567k backtests) | Backtest | https://kjtradingsystems.com/algo-trading-exits.html |
|
||||
| Concretum — pyramiding vs vol targeting | Backtest | https://concretumgroup.com/position-sizing-in-trend-following-comparing-volatility-targeting-volatility-parity-and-pyramiding/ |
|
||||
| SmallCapLab — gapper base-rate research | Vendor backtest | https://www.smallcaplab.com/research |
|
||||
| Warrior Trading — verified earnings page | Guru (aggregate audited) | https://www.warriortrading.com/ross-camerons-verified-day-trading-earnings/ |
|
||||
| CXO Advisory — ORB review | Review | https://www.cxoadvisory.com/technical-trading/day-trading-with-an-opening-range-breakout-strategy/ |
|
||||
@@ -0,0 +1,181 @@
|
||||
<!-- Distilled 2026-07-17 from books/order_flow_microstructure/ (9 sources; O'Hara
|
||||
image-only PDF unreadable). 9 parallel reader agents + adversarial synthesis, hard-constrained
|
||||
to data the system actually has (1-min SIP bars, 15s quote snapshots) and fenced from the
|
||||
8 refuted levers. Companion to regime_notes.md (plan 011's source). Next step: pick
|
||||
hypotheses, register as pre-registered experiments (experiments/*.json), let nightly
|
||||
adjudication decide. -->
|
||||
|
||||
|
||||
> **VERDICTS (2026-07-17, first adjudication under deployed exit config, n=2044 v1 fires):**
|
||||
> - **H1 trigger-erasure: FAIL, mechanism INVERTED here.** 76% of all fires close back below
|
||||
> the trigger within 5 bars — on $2-20 gappers the recross is NORMAL price action, not a
|
||||
> death signature. The rule clipped 46% of eventual winners (kill bar: 15%); delta −0.87R.
|
||||
> Fast recross-invalidation is proven HARMFUL on this population. This also pre-kills H5's
|
||||
> stall-tightening leg unless it behaves fundamentally differently.
|
||||
> - **H2 flow-persistence: formal PASS (+0.107R) — REFUTED IN SUBSTANCE by its own
|
||||
> pre-registered checks.** The kept population still loses (−0.19R/fire vs zeroed −0.24 —
|
||||
> barely separated), and within BOTH gap strata the sign INVERTS (choppy beats persistent:
|
||||
> 10-20% −0.03 vs −0.09; 20%+ −0.06 vs −0.26). The pooled tercile gradient is composition,
|
||||
> not signal (Simpson's). Do not promote. STRUCTURAL LESSON: on a −0.21R baseline, ANY gate
|
||||
> that zeroes ~44% of fires clears the +0.05 bar mechanically (908/2069 x 0.24 = +0.105);
|
||||
> keep-or-zero experiments need a random-gate null, not just expR_delta_min.
|
||||
> - **H4 volume-spike: FAIL clean** (−0.008R; concentration gate separates nothing).
|
||||
> - **H3 spread-expansion: INCONCLUSIVE (n=29), accruing ~30 fires/day by design** — the
|
||||
> skip-on-no-coverage guard is working; decisive in ~2 weeks.
|
||||
> Running tally: conversion levers tested 12, surviving 0 (the regime gate remains the only
|
||||
> OOS survivor, and it is session-level, not conversion). The transitory-impact mechanism
|
||||
> (convergent mechanism #1) is looking less like a filterable signature and more like the
|
||||
> population's base nature.
|
||||
|
||||
# Microstructure Shelf → ORB Entry-Conversion: Synthesis Note
|
||||
|
||||
The bot's proven #1 problem is entry conversion: triggers fire but only ~35-48% of breakouts reach +1R before -1R, and eight selection-side levers have been refuted — the population is fine, what happens AT and AFTER the breakout is the bottleneck. This shelf (Hasbrouck, Bouchaud et al., Cartea et al., Cont-Kukanov-Stoikov, Dalton, Bellafiore, Neill, Cetin-Danilova) is unusually well-matched to that problem, because market microstructure is precisely the study of which price moves stick (permanent, information-driven impact) and which revert (transitory, liquidity-driven impact). The academic and practitioner books converge, from opposite directions, on the same handful of observable signatures — and critically, most are computable from 1-min bars and 15s quote snapshots without tick data. Everything below is either a conditioning gate on already-fired entries or an exit-config variant; nothing re-opens the refuted selection axis.
|
||||
|
||||
## Convergent mechanisms
|
||||
|
||||
**1. Failed breakouts are transitory impact; the default fate of a one-shot volume burst is reversion.** (Hasbrouck ch.8/9/15; Bouchaud ch.12.4; Cartea ch.4/10.4) Price moves decompose into a permanent (information) component and a transitory (liquidity) component that mean-reverts once the initiating flow stops. Bouchaud is bluntest: after a metaorder completes, impact decays abruptly — reversion is the empirically *normal* outcome. A ~50-65% breakout failure rate is not a bug in the setup; it is the base rate for one-shot bursts. Continuation requires *sustained* flow.
|
||||
|
||||
**2. Return into the broken range is the structural invalidation signal, and it arrives before the stop.** (Dalton — balance-area re-entry; Bellafiore — fake-breakout recross at AIG $25/AMZN $86.60; Neill — false moves; Hasbrouck — the transitory signature *is* the round-trip) Four independent traditions say the same thing: once price trades back through the level it broke, the participants who drove the break are gone and the trade thesis is dead — waiting for the fixed stop is paying extra for the same information.
|
||||
|
||||
**3. Continuation is fed by persistent same-side flow, not by the size of the trigger print.** (Bouchaud ch.10 — long-memory order flow from order splitting/herding; Cartea ch.4/12 — same-side MO clustering, imbalance regimes with 0.97-0.98 persistence; Hasbrouck ch.9 — momentum-feedback impulse responses that keep rising; Neill — volume growing on the advance) The discriminator between real and fake breakouts is whether one-directional participation was *ongoing* into the trigger, versus a single isolated spike.
|
||||
|
||||
**4. Spread dynamics around the trigger carry real-time information — but the books disagree on the sign.** (Cartea ch.2, Hasbrouck ch.5: widening = Glosten-Milgrom adverse-selection pricing = informed flow = bullish for follow-through. Bouchaud ch.22, Cont-Kukanov-Stoikov (β~1/D), Bellafiore: widening = liquidity pulling back / thin book / chop-zone fragility = bearish.) Five sources agree the spread's *relative* behavior at trigger is informative; the direction is an empirical question and must be pre-registered two-sided.
|
||||
|
||||
**5. Speed and shape of the initial rejection through the level grades conviction.** (Dalton — Open-Drive vs Open-Auction taxonomy, tails "made by time" carry no conviction; Neill — gradual advance on steady volume = durable buying, vertical spike = short-covering; Bellafiore — held-bid absorption precedes real breaks) A decisive drive that doesn't look back differs measurably, at the moment it happens, from a limp tag of the level — this is same-bar shape, not delayed confirmation.
|
||||
|
||||
**6. Heavy volume with no price progress = exhaustion, at any timescale.** (Neill — "churning"; Dalton — time-beyond-level without acceptance; Bellafiore — chop zones) A *joint* volume-high + price-stall signature — explicitly distinct from the refuted volume-decline fade.
|
||||
|
||||
**7. The open is a structurally thin, high-impact regime; every quote-based feature must be normalized to the name's own trailing baseline.** (Bouchaud ch.4.2 intraday U-shape; CKS — depth 2x lower, impact 5x higher 9:30-10:00; Bellafiore — time-of-day regimes) Not a lever itself — a normalization requirement that guards every hypothesis below against re-deriving the refuted liquidity floor.
|
||||
|
||||
## Ranked hypotheses
|
||||
|
||||
### H1 — Trigger-erasure / recross fast invalidation (exit_config)
|
||||
|
||||
**Convergence: 5 books (Dalton, Bellafiore, Neill, Hasbrouck, Bouchaud). Strongest on the shelf.**
|
||||
|
||||
*Statement:* Fires where a post-entry 1-min bar closes back below the ORB trigger price within the first 5 minutes are exhibiting the transitory-impact / balance-re-entry / false-move signature and are disproportionately headed to -1R; exiting on the first such close (rather than riding to the stop) reduces average realized loss without materially truncating winners.
|
||||
|
||||
*Computable features (1-min bars only):* `trigger_price` from entry-gate receipts; per post-fill bar, `erased = close < trigger_price`; `first_erasure_bar_index` within a 5-bar window; secondary variant `recross_count` = number of bars whose low trades back through trigger. Trigger must be defined from the OR-high, independent of the stop price (no tautology).
|
||||
|
||||
*Draft pre-registration:*
|
||||
```
|
||||
population: {min_entry_gap_pct: 0.10, min_day_volume: 300000, price_min: 2.0, price_max: 20.0}
|
||||
intervention: {kind: exit_config, rule: "exit at market on first 1-min bar CLOSE < trigger_price within 5 bars post-fill; else standing exit stack", sweep: {window_bars: [3, 5], condition: [close_below, low_through]}}
|
||||
bars: {expR_delta_min: 0.05, min_n: 300}
|
||||
fidelity: {atr_frac: 0.25, spread_mult: 3, adverse_ticks: 1}
|
||||
```
|
||||
|
||||
*Expected direction:* Erasure fires convert to +1R at a materially lower rate; the early-exit variant lifts fire-weighted expR by shrinking the loser tail.
|
||||
|
||||
*Kill criteria:* expR delta < +0.05 across the sweep; OR lost-winner rate > 15% of eventual +1R trades exited by the rule (erasure-then-recovery fakeouts eating the gain); OR the rule's benefit vanishes when BE@+5% ratchet is active (redundant with existing protection).
|
||||
|
||||
*Confound guards:* Family resemblance to refuted volume-fade exits — but this is price-structure, not volume, and must independently clear the same bar. Measure lost-winner rate explicitly, not just loss reduction. Use consolidated bar closes, not quote mids, to avoid bid-ask bounce on thin names. Report separately by OR-width tercile so wide-range chop doesn't drive the pooled result.
|
||||
|
||||
### H2 — Pre-trigger order-flow persistence gate (keep-or-zero)
|
||||
|
||||
**Convergence: 4 books (Bouchaud, Cartea, Neill, Hasbrouck).**
|
||||
|
||||
*Statement:* Fires preceded by persistent same-direction bar-level flow (volume-weighted sign agreement over the run-up window) are the signature of an ongoing metaorder/herd and convert better than fires whose pre-trigger flow was choppy/balanced — the trigger is a mid-stream observation of sustained flow, not a one-off spike.
|
||||
|
||||
*Computable features (1-min bars only):* Over the last 15 closed pre-trigger bars: `sign_i = sign(close_i − open_i)`; `imbalance_persistence = |Σ sign_i · vol_i| / Σ vol_i`; secondary `agree_frac` = fraction of bars with sign matching breakout direction. No look-ahead into the trigger bar.
|
||||
|
||||
*Draft pre-registration:*
|
||||
```
|
||||
population: {min_entry_gap_pct: 0.10, min_day_volume: 300000, price_min: 2.0, price_max: 20.0}
|
||||
intervention: {kind: keep_or_zero_gate, rule: "zero fires in the bottom imbalance_persistence tercile", stratify_by: [gap_pct_bucket, rvol_bucket]}
|
||||
bars: {expR_delta_min: 0.05, min_n: 300}
|
||||
fidelity: {atr_frac: 0.25, spread_mult: 3, adverse_ticks: 1}
|
||||
```
|
||||
|
||||
*Expected direction:* Top tercile beats bottom tercile on +1R-before--1R rate; zeroing the bottom tercile lifts kept-population fire-weighted expR ≥ +0.05.
|
||||
|
||||
*Kill criteria:* Monotonicity fails across terciles; OR the spread between top and bottom tercile collapses inside gap%/RVOL strata (i.e., it was a refuted selection lever in disguise); OR fewer than min_n fires per tercile.
|
||||
|
||||
*Confound guards:* Bar-sign is a crude tick-rule proxy — accept the noise, demand the effect survive stratification by gap% and RVOL (the refuted axes) before believing it. Check redundancy against the surviving runner-grade before shipping as an independent lever.
|
||||
|
||||
### H3 — Spread-expansion-ratio at trigger (keep-or-zero, two-sided)
|
||||
|
||||
**Convergence: 5 books cite the mechanism (Cartea, Hasbrouck, Bouchaud, CKS, Bellafiore) — but they split on direction, so this is registered two-sided.**
|
||||
|
||||
*Statement:* The spread's behavior at trigger, normalized to the name's own trailing baseline, discriminates converting from failing breakouts. Adverse-selection theory (Cartea/Hasbrouck) predicts widening = informed flow = keep; fragility theory (Bouchaud/CKS/Bellafiore) predicts widening = liquidity vacuum = zero. The test determines which regime dominates $2-20 gappers.
|
||||
|
||||
*Computable features (15s quote snapshots):* `spread_baseline = median((ask−bid)/mid) over the 10 min pre-trigger`; `spread_at_trigger` from the last snapshot at/before trigger; `expansion_ratio = spread_at_trigger / spread_baseline`; secondary post-fill variant `post_ratio(1min) = spread at t0+1min / baseline`.
|
||||
|
||||
*Draft pre-registration:*
|
||||
```
|
||||
population: {min_entry_gap_pct: 0.10, min_day_volume: 300000, price_min: 2.0, price_max: 20.0}
|
||||
intervention: {kind: keep_or_zero_gate, rule: "two-sided: compare top vs bottom expansion_ratio tercile; zero the losing tercile only if pre-registered margin met", normalize: per_name_baseline, stratify_by: [price_bucket, rvol_bucket, minutes_since_open]}
|
||||
bars: {expR_delta_min: 0.05, min_n: 300}
|
||||
fidelity: {atr_frac: 0.25, spread_mult: 3, adverse_ticks: 1}
|
||||
```
|
||||
|
||||
*Expected direction:* Two-sided by design; a tercile spread in conversion rate that is stable across the backtest window and survives stratification, in either direction.
|
||||
|
||||
*Kill criteria:* No stable tercile ordering across time halves of the window; OR the effect disappears within price/RVOL strata (= liquidity floors relitigated → discard permanently); OR >20% of fires lack a usable pre-trigger snapshot baseline.
|
||||
|
||||
*Confound guards:* Ratio-to-own-baseline only, never absolute bps (absolute spread = refuted liquidity floor). Stratify by minutes-since-open (Bouchaud/CKS U-shape mechanically widens everyone's open spread). Test for independent value vs H1 — post-fill spread blowout and price erasure are correlated symptoms of the same reversal.
|
||||
|
||||
### H4 — Volume ramp vs isolated-spike trigger (keep-or-zero)
|
||||
|
||||
**Convergence: 3 books (Bouchaud, Neill, Hasbrouck).**
|
||||
|
||||
*Statement:* Fires where volume built progressively across the pre-trigger bars (a ramp — consistent with an unfinished metaorder still executing) convert better than fires where nearly all volume arrived in the single trigger bar with no build-up (a completed one-shot burst whose impact, per Bouchaud, reverts once done; per Neill, the vertical short-covering spike that "dies at the top").
|
||||
|
||||
*Computable features (1-min bars only):* `concentration_ratio = trigger_bar_volume / Σ(volume of preceding 5 bars)`; `ramp_slope = sign of linear volume trend over the last 5 pre-trigger bars`. Strictly pre-trigger — no post-entry volume conditioning (keeps it categorically distinct from the refuted volume-fade exit).
|
||||
|
||||
*Draft pre-registration:*
|
||||
```
|
||||
population: {min_entry_gap_pct: 0.10, min_day_volume: 300000, price_min: 2.0, price_max: 20.0}
|
||||
intervention: {kind: keep_or_zero_gate, rule: "zero fires in the top concentration_ratio tercile (isolated-spike triggers)", stratify_by: [rvol_bucket, gap_pct_bucket]}
|
||||
bars: {expR_delta_min: 0.05, min_n: 300}
|
||||
fidelity: {atr_frac: 0.25, spread_mult: 3, adverse_ticks: 1}
|
||||
```
|
||||
|
||||
*Expected direction:* Isolated-spike (high concentration) fires convert worse; ramp fires convert better; zeroing spikes lifts kept expR.
|
||||
|
||||
*Kill criteria:* No monotone relation across concentration terciles; OR effect collapses within RVOL strata; OR feature is >0.7 correlated with H2's imbalance_persistence on the fire population (redundant — keep only the stronger).
|
||||
|
||||
*Confound guards:* Must survive RVOL stratification (day-level RVOL is a refuted axis; this is intrabar structure). Check against how the gap formed premarket (instant-gap vs premarket-ramp names) to avoid re-deriving runner-grade. Single odd-lot block prints on thin names can fake a spike — winsorize trigger-bar volume at a per-name percentile.
|
||||
|
||||
### H5 — Post-entry churn/stall exhaustion tightening (exit_config)
|
||||
|
||||
**Convergence: 3 books (Neill, Dalton, Bellafiore).**
|
||||
|
||||
*Statement:* Positions showing, within the first 3 minutes post-fill, either (a) no new 1-min bar high above the entry bar's high (Dalton: no range extension = weak other-timeframe control), or (b) volume spiking above baseline while price makes no net progress (Neill: churning), are disproportionately failures; tightening the stop to breakeven-minus-buffer on that signal cuts loser size without materially cutting winners.
|
||||
|
||||
*Computable features (1-min bars only):* `entry_bar_high`; `first_new_high_bar_index` within 3 bars; joint churn flag on bars t0+1, t0+2: `vol_ratio = bar_vol / trailing_15bar_mean > 2.0 AND |close − entry_price|/entry_price < 0.5 × trailing_bar_range_mean`.
|
||||
|
||||
*Draft pre-registration:*
|
||||
```
|
||||
population: {min_entry_gap_pct: 0.10, min_day_volume: 300000, price_min: 2.0, price_max: 20.0}
|
||||
intervention: {kind: exit_config, rule: "on stall (no new high in N bars) OR churn flag, tighten stop to entry − 0.25×ATR; sweep N in [2,3,4]", baseline: current_stop_stack}
|
||||
bars: {expR_delta_min: 0.05, min_n: 300}
|
||||
fidelity: {atr_frac: 0.25, spread_mult: 3, adverse_ticks: 1}
|
||||
```
|
||||
|
||||
*Expected direction:* Average realized loss on eventual losers shrinks; eventual-winner capture rate stays within 10% of baseline; net fire-weighted expR ≥ +0.05.
|
||||
|
||||
*Kill criteria:* Winner-capture drops >10% at every N in the sweep (whipsaw cost exceeds loss savings); OR the churn leg alone reproduces the refuted volume-fade result (volume component fires without the price-stall component adding anything — then drop the volume leg and keep only price-stall, or kill entirely); OR gains are fully redundant with H1 erasure exits on the same trades.
|
||||
|
||||
*Confound guards:* This is a *joint* high-volume+stall or no-new-high signature, never volume-decline (the refuted fade). Run H1 first; only fires H1 doesn't already catch count toward H5's incremental value. Sensitivity-sweep N — real winners on small caps routinely chop 2-3 bars before extending.
|
||||
|
||||
## Rejected during synthesis
|
||||
|
||||
- **Gap-still-open-at-trigger gate (Dalton)** — single source; heavily confounded with time-since-open and gap% (refuted axis); clean but keep on the bench behind the five above.
|
||||
- **Round-number proximity gate (Cartea)** — single source; evidence from a $450 large-cap at penny resolution, transfer to $2-20 names unvalidated; mechanically confounded with price level.
|
||||
- **Time-since-open depth-regime gate (CKS)** — single source; collides with the dead 10:30-cutoff history and with early-trigger/gap% correlation; refuted-adjacent selection flavor.
|
||||
- **Pre-trigger quote-size imbalance (Cartea/CKS)** — hard dependency on bid_size/ask_size existing in the snapshot logs; unverified = dead on arrival; the paper's OFI variable explicitly needs queue data we don't have.
|
||||
- **Volume-node thickness at trigger (Bellafiore)** — single source; volume-at-price concentration is RVOL in new clothes without careful shape-normalization; weakest power.
|
||||
- **Overhead same-day resistance distance (Neill)** — single source, and the book itself flags naive levels as unreliable "until confirmed by the action of the stock."
|
||||
- **Bid-erosion early exit (Bellafiore)** — single source; 15s sampling too coarse for "held bid"; largely restates mark-to-market drawdown; folded conceptually into H1/H3.
|
||||
- **Impulse decay half-life adaptive stops (Hasbrouck)** — 5 noisy data points per fire can't support a fitted decay rate; strictly a refinement of H1 if H1 shows signal.
|
||||
- **Trigger-bar close-location drive gate (Dalton) / trigger-bar volume conviction (Neill)** — too close to the refuted close-above-trigger confirmation family; their live content (shape/participation quality) is already carried by H2 and H4 with better mechanism backing.
|
||||
- **Everything from Cetin-Danilova** — existence-proof mathematics over unobservable signed order flow; zero computable features.
|
||||
|
||||
## Data upgrades that would unlock more
|
||||
|
||||
1. **Verify whether bid_size/ask_size are in the 15s quote snapshots** — if yes, the CKS order-flow-imbalance family (the highest-R² predictor on this shelf) becomes partially computable; costs one grep.
|
||||
2. Tighten quote snapshot cadence to ~5s for watched names in the 5 minutes around triggers — halves the aliasing that weakens H3.
|
||||
3. Log NBBO at the exact trigger/fill instant in the entry receipts (not nearest-15s) — removes the largest measurement error in every spread hypothesis.
|
||||
4. Per-minute uptick/downtick trade counts (available from SIP aggregates on some feeds) would upgrade H2's crude bar-sign proxy to a real tick-rule imbalance.
|
||||
5. Tick-by-tick trades remain the true unlock for trade-sign imbalance and held-bid detection — everything above is the best available proxy without them.
|
||||
@@ -0,0 +1,59 @@
|
||||
# The regime gate: what it is, and why "it fails everything" is the right question
|
||||
|
||||
**Owner's question (2026-07-18):** "I don't know if we need the regime thing if
|
||||
everything just fails the regime as well. What's the whole point of that?"
|
||||
|
||||
Honest answer, in three parts.
|
||||
|
||||
## What it actually is
|
||||
|
||||
The regime gate scores each session `P(our ORB makes money today)` from free
|
||||
pre-open features (premarket breadth, SPY momentum/vol, the bot's own recent
|
||||
trailing results), using a small walk-forward model. Below the threshold, it
|
||||
sits the whole day out. Config: `TRADING_REGIME_GATE_ENABLED=1`,
|
||||
`TRADING_REGIME_GATE_P_MIN=0.5`. It is the **one** research lever out of ~13
|
||||
that survived out-of-sample testing.
|
||||
|
||||
## Why it "fails everything" right now — and why that's honest, not broken
|
||||
|
||||
Recent sessions genuinely scored bad (Friday 07-17 = −0.9R realized). The gate
|
||||
isn't malfunctioning when it sits out — it's correctly reporting that the
|
||||
recent regime is one the ORB loses in. Over the full 266-session backtest it
|
||||
did NOT sit out everything: it traded ~32% of days and took the strategy from
|
||||
−0.11R/session (bleeding) to roughly flat. So "always no" is a recent-weeks
|
||||
artifact of a genuinely bad stretch, not the gate's normal behavior.
|
||||
|
||||
## The real limitation you're pointing at (and you're right)
|
||||
|
||||
**A gate that only ever says "no" is indistinguishable from turning the bot
|
||||
off.** That is the honest ceiling: the regime gate is **damage control, not an
|
||||
edge source.** It cannot create profit that isn't there — it can only allocate
|
||||
the strategy's existing (slightly negative) edge onto the least-bad days. Its
|
||||
best possible outcome on a losing base strategy is **flat**. Never gain.
|
||||
|
||||
So its value is entirely **conditional**:
|
||||
|
||||
- **If the base entry has no positive edge** (where we are now — 12 conversion
|
||||
levers refuted, ~20% win rate), the regime gate is just a sophisticated "don't
|
||||
trade much." Flat-vs-bleeding is a real but unexciting win, and yes — at the
|
||||
limit it's a kill switch with extra steps. Your instinct is correct.
|
||||
- **If a positive entry edge is found** (the whole point of the plan-017 L2
|
||||
depth work), the regime gate flips from damage-control to **force
|
||||
multiplier**: trade the good NAMES on the good DAYS, and the two signals
|
||||
compound instead of one merely limiting losses.
|
||||
|
||||
## Big-picture relevance
|
||||
|
||||
The regime gate is not the thing that makes money. It's the thing that stops
|
||||
the bot bleeding **while** we search for the thing that makes money. Keep it on
|
||||
because flat beats bleeding for free — but judge it correctly: it is a seatbelt,
|
||||
not an engine. The engine, if it exists, is an entry signal that separates
|
||||
winners at the trigger (plan 017 / order-book depth). **The day an entry edge
|
||||
clears the bar, the regime gate becomes genuinely valuable. Until then, it's a
|
||||
holding pattern — and a holding pattern is the correct posture for a strategy
|
||||
still looking for its edge.**
|
||||
|
||||
If plan-017 Phase 0 shows the order book separates winners, the regime gate
|
||||
earns its keep. If not, the honest move is the owner-selector fork, and the
|
||||
regime gate reverts to a simple "trade fewer days" toggle you can leave on or
|
||||
off with little consequence.
|
||||
@@ -0,0 +1,55 @@
|
||||
# Regime-detection research notes — distilled for the momentum bot
|
||||
|
||||
**Date:** 2026-07-15
|
||||
**Method:** 6 books in `books/market_regime_state_detection/` distilled by parallel readers (Ang & Timmermann NBER survey; Kritzman *Regime Shifts*; Zucchini/MacDonald/Langrock *HMM for Time Series*; Chan *Hands-On AI Trading* & *Quantitative Trading*; Kakkar et al *Trading Decoded*). Hamilton 1989 is a scanned image (no extractable text); its Markov-switching method is covered by the others.
|
||||
**Problem this feeds:** the bot is ~breakeven with no entry edge after honest fills; the one thing untried is **regime awareness** — it did well 07-08/09 (a runner regime) and sits out / bleeds otherwise. Goal: a FREE-data daily regime signal that gates/sizes ORB entries, harness-backtestable over ~260 sessions.
|
||||
|
||||
---
|
||||
|
||||
## The one insight that reframes everything (Chan's metalabeling)
|
||||
|
||||
> "The key to successfully apply AI/ML to finance is to focus on **metalabeling** — finding the probability of profit of *your own* simple trading strategy, and **not** to use it to predict the market directly. Your own strategy's past track record is private; no one else is trying to predict it." — Chan, *Quantitative Trading*
|
||||
|
||||
Don't detect abstract "market regime." **Predict `P(our ORB makes money today)`** from free pre-open features, using our OWN per-session expR (which the plan-006 harness already computes) as the label. This targets exactly what matters (does *our* edge work today), sidesteps the "high-vol lumps runner-days with crash-days" problem, reuses the harness, is free, and is a *private* signal nobody arbs away. Every practical book (Chan CPO, Trading Decoded RF) converges here; the academic survey (Ang-Timmermann) calls the same thing "regime-conditioned position sizing."
|
||||
|
||||
## The method stack (what to build, in escalation order — ponytail)
|
||||
|
||||
1. **Threshold baseline first** (Kritzman, Trading Decoded both insist): a trivial rule — e.g. `SPY 5d momentum > 0 AND SPY 20d vol < median`, or Kritzman **Financial Turbulence** (`sqrt((r−μ)Σ⁻¹(r−μ)')` on ~11 sector-ETF returns) below its 75th percentile. If the fancy model can't beat this net-of-costs out-of-sample, ship the rule.
|
||||
2. **Metalabel classifier** (the primary): `GradientBoostingClassifier` / `RandomForest` (free sklearn — the books' `predictnow.ai`/`gbdt` replaced) predicting `P(profitable ORB session)` from ~5–8 free features. Gate/size on the probability.
|
||||
3. **HMM regime as a FEATURE** (not the whole model): a 2-state Markov-switching state on SPY (`statsmodels MarkovRegression(spy_returns, k_regimes=2, switching_variance=True)`) or on the turbulence series — one column feeding the classifier, or a standalone gate to compare.
|
||||
|
||||
## Free features (all pre-open, all we have or can ingest free)
|
||||
|
||||
| Feature | Source | Note |
|
||||
|---|---|---|
|
||||
| SPY 5d/20d momentum + 20d realized vol (VIX proxy) | already ingesting SPY (plan 009) | the level-1 input every book uses |
|
||||
| IWM (small-cap index) + sector-ETF momentum | Alpaca/yfinance, **free** | small-caps specifically; sector rotation |
|
||||
| Kritzman turbulence (Mahalanobis on ~11 sector ETFs) | sector-ETF daily returns, **free** | "prices behaving uncharacteristically" |
|
||||
| Breadth = % of the gapper universe green / held-VWAP yesterday | our own stored data | the SIGNED feature that breaks vol-symmetry |
|
||||
| **Bot's own trailing-N-day realized expR + hit-rate** | our harness | the most on-target feature — a hot/cold-streak detector on our OWN edge |
|
||||
| VIX level + 1d change | CBOE/yfinance, **free** | the vol axis directly |
|
||||
|
||||
Keep to **≤5–8 features** — only ~260 sessions, ~a third fewer "runner days." More features = curve-fit.
|
||||
|
||||
## The load-bearing discipline (or the whole thing is a lie)
|
||||
|
||||
- **Filtered / walk-forward / purged only.** Only a regime label computed from data **≤ today** is tradeable. Smoothed HMM probs and full-sample fits **peek at the future**. Sanity check every book demanded: *if a smoothed-gated backtest crushes the filtered-gated one, your edge is look-ahead.* Use purged/embargoed walk-forward CV (drop the boundary day) — the same leakage-safe split used in `plans/008`.
|
||||
- **Label is same-day-outcome, features are strictly pre-open.** The session's expR (label) uses open-knowable exits; every feature is lagged to the prior close.
|
||||
- **Out-of-sample split is mandatory.** Fit on the older half, apply to the recent half; confirm the favorable-regime expR advantage survives. "In-sample regimes need not persist out-of-sample" (Ang-Timmermann) — if the edge evaporates OOS, the regimes were in-sample noise.
|
||||
- **Acceptance = NET expR improves across EVERY walk-forward fold**, not AUC. A 52.8% classifier can still be net-negative (Chan Hands-On's own OOS accuracy was 52.77%).
|
||||
- **Let the DATA pick the sign.** Ang-Timmermann finds low-vol = high-Sharpe for a *long-index* investor — but a long-*dispersion* small-cap momentum strategy likely inverts this (runners need volatility). Measure conditional expR per regime; never assume high-vol = bad.
|
||||
- **Small-sample humility.** 2 states (don't LR-test the count), shallow trees (`max_depth ≤ 3`), report bootstrap/posterior bands on the regime probs, deflate any OOS Sharpe by the number of tweaks tried (Deflated Sharpe Ratio).
|
||||
- **Regimes lag.** A daily gate enters the runner regime a day or two late and exits late — structural. It sits out *some* good days and takes *some* bad ones. The bar is net improvement, not perfection.
|
||||
|
||||
## Honest prior
|
||||
|
||||
This is the most promising lever of the session — it targets the bot's own edge, uses free data, reuses the harness, and directly explains the 07-08/09-vs-today swing. **But 260 sessions is a small sample**, so the real risk is curve-fitting a regime that doesn't persist. The threshold-baseline-first rule, purged walk-forward, and OOS split are the guards. If, after all that, the gated expR doesn't clearly beat trade-always out-of-sample → the honest finding is "no exploitable regime signal in free data at this sample size," and we've learned it cheaply.
|
||||
|
||||
## Sources (in `books/market_regime_state_detection/`)
|
||||
|
||||
- Ang & Timmermann (2011), *Regime Changes and Financial Markets*, NBER WP 17182 — the survey + the sizing template + every caveat.
|
||||
- Kritzman et al, *Regime Shifts: Implications for Dynamic Strategies* — Financial Turbulence + absorption ratio.
|
||||
- Zucchini, MacDonald, Langrock, *Hidden Markov Models for Time Series* — the estimation recipe (forward-filter, BIC state selection, why filtered ≠ smoothed).
|
||||
- Chan et al, *Hands-On AI Trading* — the exact `statsmodels MarkovRegression` daily-gate recipe (Example 4) + the 4-year-lookback calibration.
|
||||
- Chan, *Quantitative Trading* — Conditional Parameter Optimization + the metalabeling principle.
|
||||
- Kakkar, Sahni, Shanmugamani, *Trading Decoded* — the RandomForest regime-as-feature recipe + purged walk-forward CV + the "beat the trivial rule or ship the rule" discipline.
|
||||
@@ -0,0 +1,155 @@
|
||||
<!-- Distilled 2026-07-17 from books/supply_side/ (15 sources). PARKED BY OWNER DIRECTION:
|
||||
no new experiments from this note unless the owner asks — see memory: owner-intent-keep-it-simple.
|
||||
Filed for reference; its M1/M2 mechanism (filings FOLLOW strength; supply hits at the 424B
|
||||
pricing, not registration) explains dilution_veto_v1's backfire. -->
|
||||
|
||||
# Supply-Side Mechanics Shelf — Final Distillation Note
|
||||
|
||||
**System:** small-cap ORB momentum bot (gap ≥10%, $2–20, ≥300k day vol, long-only, minutes–hours holds, ~2k logged fires, pre-registered counterfactual harness).
|
||||
|
||||
**Framing.** Twelve conversion levers are dead, and the one supply-side gate already tried — the 90-day any-EDGAR-filing keep-or-zero veto — failed *backwards* (flagged fires outperformed, n=64). This shelf's 15 sources converge on a single explanation for that backfire, and it is the most useful thing in the entire pile: **a registration filing is a lagging marker of price strength, not a leading marker of supply pressure.** Firms file and sell *because* the stock ran (Greenwood §V; Billett/Floros/Garfinkel's sell-into-strength coefficient; WTD's baby-shelf capacity that mechanically grows with the 60-day high). So the naive dilution-trap story was not just noisy — it was sign-inverted by construction, and the ledger's n=64 result is what the mechanism predicts.
|
||||
|
||||
The squeeze literature warnings hold up under the primary sources here: nothing in this shelf supports any squeeze-long construction. Fabozzi/Mann, Lamont, Staley, both gamma papers, the meme-frenzy paper, and the SEC GME report collectively add zero computable squeeze-timing signal and several independent reasons the free proxies (SI, short-vol%, FTDs) are equilibrium quantities or noise. Those are all rejected below, explicitly.
|
||||
|
||||
What survives is narrow but real: **the timing axis inside the EDGAR feed itself** (registration vs. pricing), **an exit-shape prediction** (priced supply dampens continuation, not just price), and **two accrual features** (forward dilution propensity; halt-reopen discontinuity). None require new paid data. Two are not gates at all, which matters under the hardened random-gate null.
|
||||
|
||||
---
|
||||
|
||||
## 2. Convergent mechanisms
|
||||
|
||||
**M1 — Registration ≠ sale: supply hits the tape at the pricing/takedown event, not the filing.** An S-1/S-3 creates legal *capacity*; float actually changes at the 424B pricing (or ATM takedown), which is the US analog of Greenwood's pay-date and Hong–Scheinkman–Xiong's lock-up expiration. Six sources land here independently: Greenwood (pay-date, not announcement, carries the negative return), Harris (shelf vs. pricing are mechanically distinct moments with opposite information content), Asness/Fabozzi (lending-supply mechanics fire only at execution), HSX (price drops at the supply event even though the date is public), ATM/JFQA and WTD (program existence is EDGAR-visible; sale dates never are). The failed veto collapsed these two stages into one flag. This is the shelf's load-bearing mechanism.
|
||||
|
||||
**M2 — Issuance follows strength (the veto-backfire mechanism).** Greenwood §V (issuance/insider selling *after* the float-driven rise, even matched on prior returns), ATM/JFQA (same-quarter abnormal return is the only significant takedown predictor; prior quarter is not), WTD (baby-shelf legal capacity is computed off the trailing 60-day *high*, so capacity peaks exactly on gap days). Consequence: any filing-recency feature is partially a momentum-strength proxy, which is *positive-EV* in this system. Every hypothesis below must be checked net of gap%/rvol/quality for this reason.
|
||||
|
||||
**M3 — ATM supply is invisible at daily granularity and decays with program age.** Sale dates are never disclosed (quarterly aggregates only, 8-K usually backward-looking — ATM/JFQA Table 1); only ~1/3 of programs ever complete, and takedown pace decays monotonically with quarters elapsed. Two implications: (a) no EDGAR-derived feature can ever be a same-day supply flag, so expect attenuated signal and design for it; (b) a flat N-day cutoff is the wrong functional form — features should age/decay.
|
||||
|
||||
**M4 — Priced supply changes the *path*, not only the level: turnover and volatility compress.** HSX's model predicts price, turnover, and volatility all fall together when float is released, with a *nonlinear* effect that is largest exactly in the thin-float regime this system trades. Asness/Fabozzi adds the lending-side version: a priced secondary relieves borrow specialness, removing short-covering fuel for continuation legs. Both point at the same tradeable consequence: fires shortly after a priced takedown should show weaker *continuation* (lower MFE, shorter legs) — an exit-config claim, not an entry veto.
|
||||
|
||||
**M5 — Sign of the post-pricing effect is contested; the separation is not.** Greenwood + HSX predict 424B-recent fires are *worse* (supply pressure at the release event); Harris + Asness/Fabozzi predict they're *fine-to-better* (overhang resolved, borrow relieved) while *shelf-only* names carry the drag. All four agree the two buckets should behave *differently* — which is precisely the shape the hardened random-gate null demands. Register the split; let the harness pick the sign.
|
||||
|
||||
**M6 — Halt reopen is a regime switch, not a pause.** Harris: a halt swaps continuous discriminatory pricing for a single-price call auction; a stop resting through a halt fills at the discontinuous clearing price, not its level. This is a *different event class* from ordinary recrossing (the refuted trigger-erasure lever fired on continuous-market recrosses, which are normal here). Thin gappers are the maximally halt-prone population.
|
||||
|
||||
---
|
||||
|
||||
## 3. Ranked hypotheses (top 4)
|
||||
|
||||
### H1 — Filing-stage split of the dilution flag: 424B-recent vs. shelf-only vs. clean
|
||||
|
||||
*The highest-convergence item on the shelf (M1+M2+M5; proposed independently by 5 of 15 sources). Re-cuts the failed veto along the mechanism's own joint, and predicts which fires get BETTER, not just which to skip.*
|
||||
|
||||
**Statement.** The failed 90-day veto averaged two mechanically opposite populations. Split every fire three ways at fire time: **shelf-only** (S-1 or S-3 accession in trailing 90d, no 424B in trailing 15 trading days), **424B-recent** (any 424B accession in trailing 15 trading days), **clean** (neither). Prediction: the buckets separate. Primary direction (Greenwood pay-date + HSX): 424B-recent underperforms clean by ≥0.05R; shelf-only performs at/above clean (consistent with the observed n=64 outperformance, since shelf-only dominates the flagged population by base rate). Registered alternative (Harris/Asness lending-relief): the drag sits on shelf-only instead. Either outcome is a usable gate; *no separation* kills it.
|
||||
|
||||
**Data needed:** existing (EDGAR dilution_events: form type + accession date; verify the feed actually preserves 424B subtypes — filter out debt-only/boilerplate 424B3s where identifiable).
|
||||
|
||||
**Computable features:** `days_since_last_424B`, `days_since_last_S1S3`, categorical `filing_stage ∈ {424b_recent, shelf_only, clean}`; secondary graded feature per Greenwood magnitude-scaling: offering size / trailing 20d dollar volume where parseable.
|
||||
|
||||
**DRAFT PRE-REGISTRATION**
|
||||
- population: `{min_entry_gap_pct: 0.10, min_day_volume: 300000, price_min: 2.0, price_max: 20.0}`
|
||||
- intervention: keep-or-zero gate on the bucket the split identifies as the drag bucket (primary registration: zero `424b_recent`, keep `shelf_only` and `clean`); both non-gated buckets reported so the adjudicator sees separation, not exposure-cut
|
||||
- bars: `{expR_delta_min: 0.05, min_n: 300}` — random-gate null applies: gated bucket must be worse than `flagged_share × |mean baseline| + 0.05R`, AND the kept flagged bucket (shelf_only) must be ≥ baseline (that's the separation claim)
|
||||
- fidelity: `{atr_frac: 0.25, spread_mult: 3, adverse_ticks: 1}`
|
||||
|
||||
**Expected direction:** 424B-recent expR ≤ baseline − 0.05R; shelf-only ≥ baseline; clean = baseline.
|
||||
|
||||
**Kill criteria:** (a) 424B-recent and shelf-only statistically indistinguishable → mechanism dead, do not re-cut a third time; (b) separation exists but vanishes after conditioning on gap%/rvol/quality → M2 confound, it was momentum all along; (c) 424B-recent bucket still < n=300 after 3 months of accrual → park as accrual feature, do not adjudicate thin.
|
||||
|
||||
**Confound guards:** condition on gap%/rvol/quality (M2: filing stage proxies strength); exclude/segment fires already tagged `is_dilutive` by the LLM classifier so this isolates the EDGAR-timing signal from news content; EDGAR nightly refresh lags one session — use accession dates strictly < fire date.
|
||||
|
||||
**Coverage-design note (dilution-v1 lesson):** a fire whose symbol has *no EDGAR coverage at all* (no CIK match, feed gap) must be **skipped from the test population, not defaulted to "clean/keep"** — the prior veto's cousin failure (regime lever) shipped inert because missing data silently read as pass. Log coverage % per bucket in the probe.
|
||||
|
||||
---
|
||||
|
||||
### H2 — Recent-priced-supply continuation dampener (exit-config variant, not a gate)
|
||||
|
||||
*M4 (HSX volatility/turnover compression + Asness/Fabozzi covering-fuel removal + WTD ATM-absorbs-the-pop). Structurally immune to the random-gate null because it gates nothing — it's a paired counterfactual on the same fires.*
|
||||
|
||||
**Statement.** Fires in names with a 424B priced within the trailing 10 trading days have structurally weaker continuation (lower MFE, faster fade after trigger) regardless of entry EV — fresh float dampens the path. A tighter exit (earlier partial profit-take / tighter trail) beats the standard exit config *on this subset only*, while the standard config stays default everywhere else.
|
||||
|
||||
**Data needed:** existing (EDGAR 424B dates + 1-min OHLCV for MFE/MAE path replay in the harness).
|
||||
|
||||
**Computable features:** `recent_424B_flag` (≤10 trading days), per-fire MFE/MAE/realized-range under current vs. tightened exit config; optional interaction with `is_60d_high` (WTD baby-shelf capacity peaks there — flag it, don't gate it).
|
||||
|
||||
**DRAFT PRE-REGISTRATION**
|
||||
- population: `{min_entry_gap_pct: 0.10, min_day_volume: 300000, price_min: 2.0, price_max: 20.0}`, restricted to `recent_424B_flag = 1` fires
|
||||
- intervention: exit-config variant (tighter trail + earlier partial) replayed against standard exit on the flagged subset; standard exit unchanged on all other fires
|
||||
- bars: `{expR_delta_min: 0.05, min_n: 300}` on the subset (random-gate null formally applies but is trivially satisfied — no fires are zeroed; the comparison is config-vs-config on identical fires)
|
||||
- fidelity: `{atr_frac: 0.25, spread_mult: 3, adverse_ticks: 1}`
|
||||
|
||||
**Expected direction:** tighter exit wins by ≥0.05R on 424B-recent fires; loses or ties on clean fires (predicted asymmetry is the mechanism check — if the tight exit wins everywhere, that's a generic exit finding, not a supply finding, and it collides with the refuted fast-exit ledger).
|
||||
|
||||
**Kill criteria:** (a) tight exit wins on clean fires too → not supply-specific, discard as trigger-erasure-adjacent; (b) MFE distribution of 424B-recent fires indistinguishable from clean after gap/rvol conditioning → M4 doesn't bite at this horizon; (c) subset accrues < n=300 in 4 months → keep logging, don't test.
|
||||
|
||||
**Confound guards:** 424B pricing clusters with news-heavy days — compare against a news-matched clean control, not raw baseline; reverse causality (firms price after weakness) — check the flagged subset's prior-5d return distribution.
|
||||
|
||||
**Coverage-design note:** same skip-not-keep rule as H1 — no EDGAR coverage → excluded from both arms.
|
||||
|
||||
---
|
||||
|
||||
### H3 — Post-pump dilution propensity → same-name re-entry derate (accrual-first)
|
||||
|
||||
*M2 turned from a bug into a feature (Greenwood §V + ATM sell-into-strength). The only hypothesis here that uses the lagging direction of causality correctly.*
|
||||
|
||||
**Statement.** Big fires *cause* filings, not vice versa. Squeeze-tier fires (gap ≥150% & rvol ≥5) should show materially elevated (>2×) incidence of new dilution accessions in T+1..T+10 vs. ordinary-tier fires. Once accrued, the tradeable form is a **second-fire** rule: a re-fire in the same name within 10 sessions of a squeeze-tier fire, *with an intervening dilution accession*, is a genuinely distinct population predicted to underperform — the pump happened, the supply response is now filed, and the M2 strength-proxy value of the filing is already spent on fire #1.
|
||||
|
||||
**Data needed:** existing (fire tier features + EDGAR feed).
|
||||
|
||||
**Computable features:** `any_dilution_accession_T1_T10` per fire; tier-bucketed incidence table; `refire_after_squeeze_with_intervening_filing` flag on subsequent fires.
|
||||
|
||||
**DRAFT PRE-REGISTRATION** (accrual phase now; gate registration deferred)
|
||||
- population: `{min_entry_gap_pct: 0.10, min_day_volume: 300000, price_min: 2.0, price_max: 20.0}`
|
||||
- intervention (deferred): keep-or-zero on `refire_after_squeeze_with_intervening_filing = 1`; all first fires untouched
|
||||
- bars: `{expR_delta_min: 0.05, min_n: 300}`, random-gate null applies — the kept re-fires (no intervening filing) must match baseline for separation to count
|
||||
- fidelity: `{atr_frac: 0.25, spread_mult: 3, adverse_ticks: 1}`
|
||||
|
||||
**Expected direction:** incidence >2× after squeeze-tier fires; flagged re-fires expR ≤ baseline − 0.05R; unflagged re-fires ≈ baseline.
|
||||
|
||||
**Kill criteria:** incidence ratio <1.5× after 2 months of accrual → mechanism absent in this universe, drop before ever gating; flagged-refire n hopelessly thin (<50 in 3 months) → park.
|
||||
|
||||
**Confound guards:** must never be applied to the *triggering* fire (that is exactly the §V mistake the failed veto made); EDGAR nightly lag means T+1 incidence is undercounted by up to one session — measure T+2..T+10 as the robust window.
|
||||
|
||||
---
|
||||
|
||||
### H4 — Halt-discontinuity forced exit at reopen (accrual-first, tail-risk)
|
||||
|
||||
*M6 (Harris). Explicitly distinct from the refuted trigger-erasure lever: fires on a regime-switch event (halt→auction), not on continuous-market recrossing.*
|
||||
|
||||
**Statement.** A stop resting through a mid-hold halt fills at the reopen's single clearing price, not its level. Forcing exit at/immediately after the reopening print on currently-held names reduces tail losses vs. letting the stop ride the gap. This is a variance/tail claim on the halted subset, not a mean-edge claim.
|
||||
|
||||
**Data needed:** existing (1-min bar timestamp gaps for halt detection + 15s quote snapshots for size-collapse confirmation). Validate the halt flag against a real halt source before trusting (a feed hiccup also makes a bar gap) — the existing bar-gap halt guard's detection can be reused (rung 2: this codebase already detects bar gaps for the entry-side halt guard; extend, don't rebuild).
|
||||
|
||||
**Computable features:** `mid_hold_halt_flag` (intraday bar-gap >1min on a held name, quote-size collapse corroboration), `reopen_gap_pct`.
|
||||
|
||||
**DRAFT PRE-REGISTRATION** (accrual phase: log halt incidence + reopen-gap distribution on held fires)
|
||||
- population: `{min_entry_gap_pct: 0.10, min_day_volume: 300000, price_min: 2.0, price_max: 20.0}`, restricted to fires with `mid_hold_halt_flag = 1`
|
||||
- intervention: exit-config variant — forced exit at first reopen print vs. current stop-through-halt behavior, paired replay
|
||||
- bars: `{expR_delta_min: 0.05, min_n: 300}` — honestly unlikely to reach n=300 soon; adjudicate only when it does, report tail metrics (P5 of fire R) alongside expR
|
||||
- fidelity: `{atr_frac: 0.25, spread_mult: 3, adverse_ticks: 1}` — reopen fills should additionally assume the adverse side of the reopen print
|
||||
|
||||
**Expected direction:** forced reopen-exit improves expR and truncates the left tail on halted fires; no effect elsewhere by construction.
|
||||
|
||||
**Kill criteria:** halt incidence on held fires <1/month → not worth harness slots, keep as passive logging; reopen-gap distribution symmetric around zero → discontinuity isn't adverse-biased, no edge in forcing.
|
||||
|
||||
**Confound guards:** halt-up vs. halt-down must be separated (halt-up reopens may favor holding — Harris's gravitational effect); data-gap false positives excluded via quote-snapshot corroboration.
|
||||
|
||||
---
|
||||
|
||||
## 4. Rejected during synthesis
|
||||
|
||||
- **Any squeeze-long timing feature** (Staley float-scarcity, Barbon–Buraschi gamma, Baig meme-gamma, 0DTE paper) — requires options/borrow data we don't have; SEC GME report found no gamma squeeze even in the canonical case; literature-refuted on base rates.
|
||||
- **Prior-day short-sale-volume% gate** (Harris-derived) — Lamont + Asness/Fabozzi show short-flow/SI is an equilibrium quantity, sign-ambiguous at exactly the extremes; ATS-estimate noise on micro-caps; shape is exposure-shrink with no credible "these fires get better" leg. Fails the hardened bar by design.
|
||||
- **FTD-based flags** — SEC report: non-persistent, conflates long/short fails; predicts negative returns at monthly staleness; nothing to time with.
|
||||
- **Bi-monthly SI / days-to-cover anything** — doubly dead: stale (brief's own warning) and structurally confounded (Lamont's zero-SI-at-max-constraint case).
|
||||
- **Flat filing-recency re-test with a different window** — already failed at 90d; M3 says the functional form is wrong, not the window length. Age-decay is folded into H1's features instead of standing alone.
|
||||
- **S-1 vs. S-3 issuer-quality split as a standalone** (WTD) — plausible but n=64-scale before splitting; folded into H1 as a secondary cut if H1's buckets separate.
|
||||
- **60d-high × ATM-capacity entry veto** (WTD) — collides head-on with new-high momentum being positive in this system; demoted to a logged interaction flag inside H2.
|
||||
- **Sub-$5 margin-lending borrow-pool constraint** (Asness/Fabozzi) — re-litigates refuted price/float/liquidity gating.
|
||||
- **Borrow-scarcity spread/liquidity gates** (Fabozzi/Mann) — refuted liquidity-floor lever wearing a lending-market hat.
|
||||
- **Opening-auction imbalance fade** (Harris) — already captured by the static open-gap population variable; nothing incremental.
|
||||
- **Anti-shorting-firm / loan-recall event features** (Lamont) — event class not identifiable from any available feed; monthly-year horizon mismatch.
|
||||
- **A-/B-share float-speculation transfer** (Mei et al.) — its generalizable claim is the already-refuted low-float lever; market institutions non-analogous.
|
||||
|
||||
## 5. Data acquisition order
|
||||
|
||||
1. **Nothing new for H1/H2/H3** — first job is verifying the existing dilution_events feed preserves 424B form subtypes and accession dates cleanly, plus per-symbol CIK coverage stats (the skip-not-keep prerequisite).
|
||||
2. **OpenBB/yfinance float shares** — second, and only to magnitude-grade H1 (offering $/float per Greenwood scaling); never as a gate input (float gating is refuted). Accept staleness on actively-diluting names.
|
||||
3. **Do not wire** stockgrid short-vol%, FINRA SI, or SEC FTD files — every source that examined them here disqualified them; wiring them invites exactly the re-litigation this ledger exists to prevent.
|
||||
+96
-24
@@ -23,6 +23,12 @@ from dotenv import load_dotenv
|
||||
|
||||
sys.path.insert(0, ".")
|
||||
load_dotenv()
|
||||
|
||||
# config parity with the live loop: DB-backed knobs (config_overrides) beat
|
||||
# compose env — a bare env would replay under DEFAULT exits (the 4.5x-baseline
|
||||
# landmine, orderflow-shelf-verdicts 2026-07-17)
|
||||
from runtime.env_overrides import load as _load_cfg_overrides # noqa: E402
|
||||
_load_cfg_overrides()
|
||||
os.environ.setdefault("STRATEGY_SETUPS", "opening_range_break")
|
||||
|
||||
from alpaca_paper.client import AlpacaPaperClient
|
||||
@@ -32,6 +38,7 @@ from research.ingestion.market_data import (
|
||||
classify_session,
|
||||
ingest_daily_history,
|
||||
ingest_live_minute_bars,
|
||||
upsert_minute_bars,
|
||||
)
|
||||
from research.ingestion.signals import scan_gappers
|
||||
from research.multi_schema import open_research_db
|
||||
@@ -39,19 +46,22 @@ from research.query import query_avg_daily_volume, query_minute_bars, query_prev
|
||||
from strategy.backtest.engine import BacktestEngine
|
||||
from strategy.exits import ExitConfig, TRAIL_PRIOR_LOW, parse_profit_tiers
|
||||
from strategy.models import SetupCriteria
|
||||
from strategy.population import PopulationConfig, population_reject
|
||||
|
||||
# the live verified config
|
||||
PRICE_MIN = float(os.environ.get("WATCHER_PRICE_MIN", "1"))
|
||||
PRICE_MAX = float(os.environ.get("WATCHER_PRICE_MAX", "5"))
|
||||
GAP_MIN = float(os.environ.get("TRIGGER_GAP_MIN", "3"))
|
||||
GAP_MAX = float(os.environ.get("TRIGGER_GAP_MAX", "60"))
|
||||
RVOL_MIN = float(os.environ.get("TRIGGER_RVOL_MIN", "2"))
|
||||
# Population is defined ONCE, in strategy/population.py — the SAME gate the live
|
||||
# loop trades ($2-20 band + gap floor + 35-150% dead band + >=150% squeeze). This
|
||||
# replaces the stale $1-5 / 3-60% look-ahead screen. Prod thresholds are passed
|
||||
# explicitly (a bare from_env() shell would disable the gap gate).
|
||||
_POP = PopulationConfig(min_entry_gap_pct=0.10)
|
||||
RISK_PCT = float(os.environ.get("TRADING_RISK_PER_TRADE_PCT", "0.01"))
|
||||
|
||||
|
||||
def _daily_gappers(con, sess, universe):
|
||||
"""Identify today's qualifying gappers from DAILY bars (cheap) so we only
|
||||
minute-backfill the handful that matter, not all 100 names."""
|
||||
minute-backfill the handful that matter, not all 100 names. Screens through
|
||||
the SHARED live population gate (strategy/population.py) on the OPEN price and
|
||||
STATIC open gap — both open-knowable; the old screen used t_close (look-ahead)
|
||||
and a hard 60% cap that dropped the >=150% squeezes production re-admits."""
|
||||
out = []
|
||||
for sym in universe:
|
||||
# Bars ON OR BEFORE the session date, newest first — so rows[0] is the
|
||||
@@ -65,17 +75,53 @@ def _daily_gappers(con, sess, universe):
|
||||
"AND trade_date <= ? ORDER BY trade_date DESC LIMIT 25", [sym, sess]).fetchall()
|
||||
if len(rows) < 2 or str(rows[0][0]) != str(sess):
|
||||
continue
|
||||
t_open, t_close, t_vol = float(rows[0][1]), float(rows[0][2]), float(rows[0][3])
|
||||
t_open, t_vol = float(rows[0][1]), float(rows[0][3])
|
||||
prev_close = float(rows[1][2])
|
||||
prior = [float(r[3]) for r in rows[1:21]]
|
||||
adv = sum(prior) / len(prior) if prior else 0.0
|
||||
gap = (t_open / prev_close - 1) * 100 if prev_close else 0.0
|
||||
gap = (t_open / prev_close - 1) if prev_close else 0.0 # FRACTION (population contract)
|
||||
rvol = (t_vol / adv) if adv else 0.0
|
||||
if PRICE_MIN <= t_close <= PRICE_MAX and GAP_MIN <= gap <= GAP_MAX and rvol >= RVOL_MIN:
|
||||
# OPEN price + open gap are open-knowable; rvol re-admits real >=150% squeezes.
|
||||
if population_reject(_POP, gap_pct=gap, price=t_open, volume=None, rvol=rvol) is None:
|
||||
out.append(sym)
|
||||
return out
|
||||
|
||||
|
||||
def _session_universe(con, client, sess) -> list[str]:
|
||||
"""Candidate names for a session, from three sources union-ed:
|
||||
|
||||
(a) what the LIVE scanner discovered that day (events) — the top-100
|
||||
absolute-volume list misses small-cap gappers entirely: 2026-07-28 the
|
||||
replay found 1 gapper on a day the live loop signaled 8 names, because
|
||||
MUZ/DAMD/BIYA-class names never crack the most-actives list;
|
||||
(b) historical_gappers — the year-scale gapper record, covers dates before
|
||||
the live loop existed (backfill path);
|
||||
(c) today's most-actives — ONLY for a same-day replay (the API returns the
|
||||
CURRENT list; unioning it into a past date would be look-ahead).
|
||||
The shared population screen still runs after this — the union only widens
|
||||
candidates, it never bypasses the gate."""
|
||||
syms: set[str] = set()
|
||||
try:
|
||||
rows = con.execute(
|
||||
"SELECT DISTINCT payload_json::json->>'symbol' FROM events "
|
||||
"WHERE event_type = 'symbol_discovered' AND timestamp::date = ?",
|
||||
[str(sess)]).fetchall()
|
||||
syms |= {r[0] for r in rows if r and r[0]}
|
||||
except Exception as exc: # noqa: BLE001 — events table may not exist in research-only DBs
|
||||
print(f"[universe] discovered-symbols lookup skipped: {exc}")
|
||||
try:
|
||||
rows = con.execute(
|
||||
"SELECT DISTINCT symbol FROM historical_gappers WHERE session_date = ?",
|
||||
[str(sess)]).fetchall()
|
||||
syms |= {r[0] for r in rows if r and r[0]}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"[universe] historical_gappers lookup skipped: {exc}")
|
||||
if sess == datetime.now(timezone.utc).date():
|
||||
actives = client.get_most_actives(top=100, by="volume")
|
||||
syms |= {a["symbol"] for a in actives if a.get("symbol")}
|
||||
return sorted(syms)
|
||||
|
||||
|
||||
def _exit_cfg() -> ExitConfig:
|
||||
return ExitConfig(
|
||||
target_r=float(os.environ.get("TRADING_REWARD_MULTIPLE", "10")),
|
||||
@@ -94,31 +140,56 @@ def main(argv=None):
|
||||
con = open_research_db("market")
|
||||
client = AlpacaPaperClient(AlpacaPaperSettings.from_env())
|
||||
|
||||
# 1) full active universe (vs the ~20 the live loop watched)
|
||||
actives = client.get_most_actives(top=100, by="volume")
|
||||
universe = sorted({a["symbol"] for a in actives if a.get("symbol")})
|
||||
# 1) session universe: scanner-discovered + historical gapper record +
|
||||
# (same-day only) most-actives — see _session_universe
|
||||
universe = _session_universe(con, client, sess)
|
||||
print(f"=== EOD REPLAY {sess} ===")
|
||||
print(f"universe: {len(universe)} most-active names (live loop watches ~20)")
|
||||
print(f"universe: {len(universe)} candidates (scanner-discovered + "
|
||||
f"historical_gappers + same-day most-actives)")
|
||||
|
||||
# 2) cheap DAILY backfill -> screen gappers from daily bars (no need to pull
|
||||
# minute bars for all 100; only the gappers need them)
|
||||
ingest_daily_history(con, client, universe, days=30)
|
||||
gapper_syms = _daily_gappers(con, sess, universe)
|
||||
print(f"daily-screen gappers (${PRICE_MIN:g}-{PRICE_MAX:g}, gap {GAP_MIN:g}-{GAP_MAX:g}%, "
|
||||
f"rvol>={RVOL_MIN:g}): {len(gapper_syms)}")
|
||||
print(f"daily-screen gappers (shared population gate: ${_POP.price_min:g}-{_POP.price_max:g}, "
|
||||
f"gap >={_POP.min_entry_gap_pct:.0%} ex {_POP.gap_dead_band_lo:.0%}-{_POP.gap_dead_band_hi:.0%} "
|
||||
f"dead-band, squeeze >={_POP.gap_dead_band_hi:.0%}+rvol{_POP.squeeze_rvol_min:g}): "
|
||||
f"{len(gapper_syms)}")
|
||||
|
||||
# 3) minute-backfill ONLY the gappers (fast), then rank them, and grow the
|
||||
# dataset. Use the SIP (full-volume) feed so rvol/$-volume in the replay
|
||||
# reflect REAL volume — the live IEX feed sees only ~2-5% of it, which was
|
||||
# silently corrupting every volume-based backtest conclusion.
|
||||
ing = ingest_live_minute_bars(con, client, gapper_syms, lookback_minutes=440,
|
||||
feed=client.settings.backfill_feed)
|
||||
print(f"backfilled {ing.minute_rows} minute rows ({client.settings.backfill_feed}) "
|
||||
f"across {len(ing.symbols)} gappers\n")
|
||||
if sess == datetime.now(timezone.utc).date():
|
||||
ing = ingest_live_minute_bars(con, client, gapper_syms, lookback_minutes=440,
|
||||
feed=client.settings.backfill_feed)
|
||||
n_rows, n_syms = ing.minute_rows, len(ing.symbols)
|
||||
else:
|
||||
# historical session: a from-now lookback window would fetch the WRONG
|
||||
# day — fetch that session's 04:00-20:00 ET explicitly (premarket incl.)
|
||||
from zoneinfo import ZoneInfo
|
||||
et = ZoneInfo("America/New_York")
|
||||
start = datetime(sess.year, sess.month, sess.day, 4, 0, tzinfo=et)
|
||||
end = datetime(sess.year, sess.month, sess.day, 20, 0, tzinfo=et)
|
||||
by_sym = client.get_minute_bars(
|
||||
gapper_syms, start_iso=start.isoformat(), end_iso=end.isoformat(),
|
||||
feed=client.settings.backfill_feed) if gapper_syms else {}
|
||||
n_rows, n_syms = 0, 0
|
||||
for sym, bars in by_sym.items():
|
||||
if bars:
|
||||
n_rows += upsert_minute_bars(con, sym, bars,
|
||||
source_provider="alpaca_sip_backfill")
|
||||
n_syms += 1
|
||||
print(f"backfilled {n_rows} minute rows ({client.settings.backfill_feed}) "
|
||||
f"across {n_syms} gappers\n")
|
||||
keep = set(gapper_syms)
|
||||
gappers = [g for g in scan_gappers(con, sess, min_gap_pct=GAP_MIN, min_relative_volume=RVOL_MIN,
|
||||
price_min=PRICE_MIN, price_max=PRICE_MAX, limit=200)
|
||||
if g.gap_pct <= GAP_MAX and g.symbol in keep]
|
||||
# Population membership was already decided by the shared gate in
|
||||
# _daily_gappers; scan_gappers here only computes gap/rvol/vwap for ranking, so
|
||||
# pass permissive bounds and let `keep` be the sole authority (this also keeps
|
||||
# the >=150% squeeze names the old `gap_pct <= GAP_MAX` cap wrongly dropped).
|
||||
gappers = [g for g in scan_gappers(con, sess, min_gap_pct=0.0, min_relative_volume=0.0,
|
||||
price_min=0.0, price_max=1e9, limit=200)
|
||||
if g.symbol in keep]
|
||||
if not gappers:
|
||||
print("no qualifying gappers today.")
|
||||
return
|
||||
@@ -127,7 +198,8 @@ def main(argv=None):
|
||||
cfg = _exit_cfg()
|
||||
eng = BacktestEngine(Config(), target_r=cfg.target_r, warmup_bars=3, eval_every=1,
|
||||
ready_score_pct=55, min_bars=3, exit_config=cfg,
|
||||
criteria=SetupCriteria(gap_pct_min=0.05, relative_volume_min=RVOL_MIN))
|
||||
# intrabar ORB SETUP criteria (not population) — preserved
|
||||
criteria=SetupCriteria(gap_pct_min=0.05, relative_volume_min=2.0))
|
||||
equity = float(client.get_account().get("equity") or 0) or 100000.0
|
||||
risk_dollars = equity * RISK_PCT
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-of-day whole-market replay: what today's logic would have done + grow the
|
||||
# stored dataset for the tuner/verification. Schedule right after the close,
|
||||
# before nightly_tune (which then tunes on the freshly-grown data).
|
||||
# crontab -e -> 5 16 * * 1-5 /mnt/c/code/momentum/eod_replay.sh
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
VENV_PY="/home/philip/.venvs/momentum/bin/python"
|
||||
cd "$ROOT" || exit 1
|
||||
mkdir -p "$ROOT/data"
|
||||
echo "=== eod_replay $(date) ===" >> "$ROOT/data/eod_replay.log"
|
||||
PYTHONPATH="$ROOT" "$VENV_PY" eod_replay.py >> "$ROOT/data/eod_replay.log" 2>&1
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"id": "asym_exit_v1",
|
||||
"hypothesis": "An asymmetric exit (volatility-fractional stop with a spread floor, no profit target, trail to EOD) captures the published fat-right-tail edge better than the current managed exit.",
|
||||
"population": {"min_entry_gap_pct": 0.10, "min_day_volume": 300000, "price_min": 2.0, "price_max": 20.0},
|
||||
"intervention": {"kind": "asym_exit", "atr_frac": 0.25, "no_target": true, "trail": "eod"},
|
||||
"bars": {"expR_delta_min": 0.10, "min_n": 300},
|
||||
"futility": {"min_n": 500, "expR_delta_max": -0.02},
|
||||
"fidelity": {"atr_frac": 0.25, "spread_mult": 3, "adverse_ticks": 1}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"id": "depth_overhang_broad_v1",
|
||||
"hypothesis": "Statistically-powered version of depth_overhang_v1 over the BROAD $2-20 v1 ORB population (drops the gap>=10% live-band filter to reach n). Same overhang gate (keep>=0.034) under the REAL deployed asymmetric exit. This is the powered number; depth_overhang_v1 (gap>=10%) is the smaller live-population directional check.",
|
||||
"population": {"min_entry_gap_pct": 0.0, "min_day_volume": 0, "price_min": 2.0, "price_max": 20.0},
|
||||
"intervention": {"kind": "depth_overhang_gate", "min_overhang": 0.034},
|
||||
"bars": {"expR_delta_min": 0.05, "min_n": 300},
|
||||
"fidelity": {"atr_frac": 0.25, "spread_mult": 3, "adverse_ticks": 1}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"id": "depth_overhang_v1",
|
||||
"hypothesis": "L2 order-book overhang (plan 017): resting ask size within 1% ABOVE the ORB break, normalized to bid depth, from Databento depth. Sniff+reweighted analysis (n=602, 21 sessions) cleared the bar AND the random-gate null: keep fires with overhang>=0.034, zero the rest, lifts win rate 23%->28%. Counter-intuitive direction (more overhang above = MORE likely to win = absorption/fuel). This runs it under the REAL deployed asymmetric exit (simulate_exit, winners trail to +3R) which should exceed the conservative +1/-1 estimate. Coverage: only the 602 depth-backfilled fires have data; the rest SKIP (variant None, excluded from n). Random-gate null applies.",
|
||||
"population": {"min_entry_gap_pct": 0.10, "min_day_volume": 300000, "price_min": 2.0, "price_max": 20.0},
|
||||
"intervention": {"kind": "depth_overhang_gate", "min_overhang": 0.034},
|
||||
"bars": {"expR_delta_min": 0.05, "min_n": 300},
|
||||
"fidelity": {"atr_frac": 0.25, "spread_mult": 3, "adverse_ticks": 1}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"id": "dilution_veto_v1",
|
||||
"hypothesis": "Zeroing ORB fires on names with an S-1/S-3/424B filing in the trailing 90 days raises whole-population expR — the dilution-trap gaps fade as new shares hit.",
|
||||
"population": {"min_entry_gap_pct": 0.10, "min_day_volume": 300000, "price_min": 2.0, "price_max": 20.0},
|
||||
"intervention": {"kind": "dilution_gate", "lookback_days": 90},
|
||||
"bars": {"expR_delta_min": 0.05, "min_n": 300},
|
||||
"fidelity": {"atr_frac": 0.25, "spread_mult": 3, "adverse_ticks": 1}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"id": "exit_pm1_fattail",
|
||||
"hypothesis": "Removing the fixed target and step-locking profit (trail_r_step=0.5) captures the premarket fat tail (pm1 winners median 2.64R, peak 22min, tail to 10R) that a target-2R cap throws away — the biggest exit delta measured (+0.27R over the live exit).",
|
||||
"population": {"min_entry_gap_pct": 0.10, "min_day_volume": 300000, "price_min": 2.0, "price_max": 20.0},
|
||||
"intervention": {"kind": "exit_config", "exit_cfg": {"target_r": 0.0, "breakeven_at_pct": 0.05, "trail_r_step": 0.5}},
|
||||
"bars": {"expR_delta_min": 0.05, "min_n": 300},
|
||||
"futility": {"min_n": 500, "expR_delta_max": -0.03},
|
||||
"fidelity": {"atr_frac": 0.25, "spread_mult": 3, "adverse_ticks": 1}
|
||||
}
|
||||
Loaded 100 of 358 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user