diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e1ba753..e96ee7e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,20 +39,11 @@ jobs: # F errors anywhere in scope still fail; remove each pin when its owning # plan lands: # api/main.py F821 (missing os/hmac imports) -> plan 007 - # research/gapper_history.py F401, research/labeler.py F541/F841, - # storage/event_store.py F401, - # strategy/evaluation/structure.py F401/F841 (dead code) -> plan 002 - run: >- ruff check --select F run_live_paper.py trading_execution.py telegram_bot.py api/ alpaca_paper/ runtime/ storage/ strategy/ research/ --per-file-ignores "api/main.py:F821" - --per-file-ignores "research/gapper_history.py:F401" - --per-file-ignores "research/labeler.py:F541" - --per-file-ignores "research/labeler.py:F841" - --per-file-ignores "storage/event_store.py:F401" - --per-file-ignores "strategy/evaluation/structure.py:F401" - --per-file-ignores "strategy/evaluation/structure.py:F841" - run: python -m pytest -q env: DATABASE_URL: postgresql://momentum:momentum@localhost:5432/momentum diff --git a/README.md b/README.md index 8d989ee..072d32f 100644 --- a/README.md +++ b/README.md @@ -110,8 +110,8 @@ The `.sh` wrappers set the venv + cwd so cron can call them directly: ```bash DATABASE_URL=postgresql://user:pass@host:5432/momentum python -m pytest -# ~130 tests: strategy, storage, triggers, execution (cooldown/re-entry/EOD -# flatten/naked-stop guards), exits, journal P&L, schwab, api, research. +# ~365 tests: strategy, storage, triggers, execution (cooldown/re-entry/EOD +# flatten/naked-stop guards), exits, journal P&L, api, research. ``` Tests isolate via throwaway `mem_*` Postgres schemas; `tests/conftest.py` sweeps @@ -180,7 +180,6 @@ strategy/ pure strategy engine: evaluation, structure, exits, backte storage/ event schema + store (PostgreSQL), projections research/ research DBs, ingestion (bars, RSS, gappers), screeners alpaca_paper/ Alpaca paper client, executor, account/order sync -schwab/ Schwab OAuth, market data, positions/orders, health api/ + dashboard_api.py dashboard JSON API + static UI (:8765) admin_api.py admin panel (:8767): health, config overrides, profiles, restarts youtube_claims/ trader-video transcript -> claim extraction pipeline @@ -193,7 +192,7 @@ tests/ pytest suite (unit + integration) ## Security -`.env` and all of `data/` (DBs, `data/schwab_tokens.json`) are git-ignored and +`.env` and all of `data/` are git-ignored and never committed; token files are written `0600`. The `.env` holds **live broker / database / webhook credentials** — keep it local. If any secret was ever committed to a repo's history, **rotate it** (re-issue Alpaca keys / re-run the diff --git a/plans/002-deletions.log b/plans/002-deletions.log new file mode 100644 index 0000000..d2b0241 --- /dev/null +++ b/plans/002-deletions.log @@ -0,0 +1,47 @@ +# Plan 002 deletions log — 2026-07-13, branch milestone-2026-07 + +Every deletion was grep-gated: `grep -rn "" --include="*.py" --include="*.yml" --include="*.sh"` excluding the target itself, plans/, docs/. "clear" = zero live references. + +## Deleted (grep receipts) +- schwab/ (23 tracked files) — clear (README prose + .env.example keys only; both cleaned). Commit message of d54a2a9 claimed this deletion; it never happened. +- tests/unit/schwab/ — tests of the deleted module. +- milestone3_verify.py — schwab-only walk-through; clear. +- comprehensive_backtest.py — crashes on import (research/connection.py absent); clear. +- strategy/universe.py + UniverseConfig — clear. +- strategy/backtest/outcomes.py (+ its exports in strategy/backtest/__init__.py) — clear. +- detect_momentum_ignition in strategy/evaluation/ignition.py — clear; **kept `_last_higher_low`** (imported by runner.py:33, premarket_setup.py:32, liquid_breakout.py:44). tests/unit/test_ignition.py trimmed to the kept helper. +- grafana/ (standalone stack) — pointed at decommissioned pre-migration DB. **Provisioning dashboards MOVED** to deploy/compose/grafana/dashboards/ and docker-compose.yml bind-mount re-pointed (the in-compose grafana service keeps working). SECURITY NOTE: the deleted datasource YAML carried committed dev Postgres credentials (values not reproduced here); they remain in git history — rotate that DB user's password if the old 5432 instance still runs. +- telemetry/ — contained only stale __pycache__ (untracked); removed from disk. +- .github/workflows/docker-publish.yml — fired on nonexistent `main` + stale artifact branch; pushed wrong image name (momentum-system). +- uv.lock — DuckDB-era lockfile; requirements.txt is the real manifest. +- SCHWAB_* keys from .env.example. +- pydantic-settings from pyproject.toml — zero imports repo-wide. +- Dead intra-file code (the CI ruff pins annotated "plan 002", all removed and pins dropped from docker.yml): unused imports in research/gapper_history.py, storage/event_store.py, strategy/evaluation/structure.py; empty f-string + dead `orb` query in research/labeler.py; dead `closes` assignment in structure.py. + +## Dead DDL removed from storage/pg_schema.sql (10 tables, zero code references) +backtest_runs, halt_events, market_context, paper_missed_trades, paper_sessions, +short_interest, simulated_fills, simulated_orders, strategy_runs, telemetry_events + +## Dead projections removed from storage/projections.py (zero consumers) +query_event_timeline, query_positions_view, query_settings_snapshot, query_symbol_status +(private helpers _evt_window/_hist_window/_latest_event_payload/_state_until/_sum_position_unrealized all have internal callers — kept) + +## Kept (grep hit live code, or disposition elsewhere) +- strategy/evaluation/ignition.py `_last_higher_low` — live (3 importers). +- quote_snapshots, float_cache — created by CODE (run_live_paper.py, research/ingestion/float_cache.py), no pg_schema DDL to remove; write-only today; disposition in plan 006. +- dilution_events — pg_schema DDL kept; disposition in plan 006. + +## FOR THE OPERATOR — run only after plan 001's backup exists + one session soak +```sql +DROP TABLE IF EXISTS backtest_runs; +DROP TABLE IF EXISTS halt_events; +DROP TABLE IF EXISTS market_context; +DROP TABLE IF EXISTS paper_missed_trades; +DROP TABLE IF EXISTS paper_sessions; +DROP TABLE IF EXISTS short_interest; +DROP TABLE IF EXISTS simulated_fills; +DROP TABLE IF EXISTS simulated_orders; +DROP TABLE IF EXISTS strategy_runs; +DROP TABLE IF EXISTS telemetry_events; +``` +These were removed from pg_schema.sql only; the production database still holds the (empty/stale) tables until you run the DROPs. diff --git a/plans/README.md b/plans/README.md index 19dac86..7a2931c 100644 --- a/plans/README.md +++ b/plans/README.md @@ -17,7 +17,7 @@ your row when done. | Plan | Title | Priority | Effort | Depends on | Status | |------|-------|----------|--------|------------|--------| | 001 | Production safety rails — CI test gate, dead-man's switch, pg_dump backup | P1 | S | — | DONE | -| 002 | Dead-weight deletion — schwab/, grafana/, dead modules/tables/projections | P2 | S | 001 | TODO | +| 002 | Dead-weight deletion — schwab/, grafana/, dead modules/tables/projections | P2 | S | 001 | DONE | | 003 | Money-path hardening — daily self-restart, positive-evidence contract, reconcile, cage count | P1 | M | 001 | TODO | | 004 | Learning flywheel — one population module, scheduled eod_replay + nightly_tune, atomic writes | P1 | M | 001 (002 rec.) | TODO | | 005 | Entry-path unification + decision receipts | P2 | M | 001, 003, 004 | TODO | diff --git a/pyproject.toml b/pyproject.toml index ddf85b1..41611a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,6 @@ dependencies = [ "psycopg2-binary>=2.9.0", "pandas>=2.0.0", "pydantic>=2.0.0", - "pydantic-settings>=2.0.0", "pyyaml>=6.0", "typer>=0.9.0", "rich>=13.0.0", diff --git a/research/gapper_history.py b/research/gapper_history.py index fefdd7d..c9aa874 100644 --- a/research/gapper_history.py +++ b/research/gapper_history.py @@ -24,7 +24,7 @@ market at minute resolution: import argparse import time from collections import defaultdict -from datetime import date, datetime, timedelta +from datetime import datetime, timedelta import os import pandas as pd diff --git a/research/labeler.py b/research/labeler.py index 702b06e..227deab 100644 --- a/research/labeler.py +++ b/research/labeler.py @@ -774,12 +774,6 @@ def validate_runner(con): lb_1r = _wilson_lower(sum(r1), len(r1)) fresh = [r[2] for r in rows if r[3] is None or r[3] <= RUNNER_PM_EXHAUSTION_FRAC] rate_fresh = (sum(fresh) / len(fresh)) if fresh else 0.0 - orb = con.execute( - "WITH rn AS (SELECT DISTINCT symbol, session_date FROM setup_events WHERE setup_version=?) " - "SELECT avg(CASE WHEN l.reached_1r_before_minus_1r THEN 1.0 ELSE 0 END), count(*) " - "FROM setup_events se JOIN outcome_labels l ON l.setup_id = se.setup_id " - "JOIN rn ON rn.symbol = se.symbol AND rn.session_date = se.session_date " - "WHERE se.setup_version=?", [RUNNER_SETUP_VERSION, SETUP_VERSION]).fetchone() vr = con.execute( "WITH rn AS (SELECT DISTINCT symbol, session_date FROM setup_events WHERE setup_version=?) " "SELECT avg(CASE WHEN l.reached_1r_before_minus_1r THEN 1.0 ELSE 0 END), count(*) " @@ -1391,7 +1385,7 @@ def lift(con): ct = df[df["cat_type"].notna()].groupby("cat_type", observed=True)[label].agg(["mean", "count"]) ct = ct[ct["count"] >= 15].sort_values("mean", ascending=False) if len(ct): - print(f" catalyst_type (n>=15): " + " | ".join( + print(" catalyst_type (n>=15): " + " | ".join( f"{t}:{r['mean']*100:.0f}%/{int(r['count'])}({r['mean']/base:.1f}x)" for t, r in ct.iterrows())) print("\nCAVEAT: ~17-19 autocorrelated days, +20% runners are sparse — a lift is a HYPOTHESIS to") print("shadow-confirm forward, not a validated win-rate. Anything 'thin n' is noise until more data.") diff --git a/storage/event_store.py b/storage/event_store.py index 5ed7e47..844e1c5 100644 --- a/storage/event_store.py +++ b/storage/event_store.py @@ -6,7 +6,6 @@ Append-only event storage with canonical event types. from .event_schema import ( BaseEvent, EventType, - EventMode, ) from .db import get_connection from uuid import uuid4 diff --git a/storage/pg_schema.sql b/storage/pg_schema.sql index 10b2d8e..8439014 100644 --- a/storage/pg_schema.sql +++ b/storage/pg_schema.sql @@ -126,36 +126,7 @@ CREATE TABLE IF NOT EXISTS news_events ( PRIMARY KEY (id) ); -CREATE TABLE IF NOT EXISTS halt_events ( - id VARCHAR, - symbol VARCHAR, - halt_start TIMESTAMP, - halt_end TIMESTAMP, - halt_type VARCHAR, - price_at_halt DOUBLE PRECISION, - resumed_price DOUBLE PRECISION, - gap_on_resume_pct DOUBLE PRECISION, - PRIMARY KEY (id) -); -CREATE TABLE IF NOT EXISTS market_context ( - timestamp TIMESTAMP, - trade_date DATE, - spy_open DOUBLE PRECISION, - spy_high DOUBLE PRECISION, - spy_low DOUBLE PRECISION, - spy_close DOUBLE PRECISION, - spy_volume BIGINT, - qqq_open DOUBLE PRECISION, - qqq_close DOUBLE PRECISION, - spy_intraday_return_pct DOUBLE PRECISION, - vix_open DOUBLE PRECISION, - vix_level DOUBLE PRECISION, - advance_decline_ratio DOUBLE PRECISION, - up_volume_down_volume_ratio DOUBLE PRECISION, - tape_regime_flag VARCHAR, - PRIMARY KEY (timestamp) -); CREATE TABLE IF NOT EXISTS dilution_events ( id VARCHAR PRIMARY KEY, @@ -168,14 +139,6 @@ CREATE TABLE IF NOT EXISTS dilution_events ( gross_proceeds DOUBLE PRECISION ); -CREATE TABLE IF NOT EXISTS short_interest ( - symbol VARCHAR, - report_date DATE, - short_interest_shares BIGINT, - short_interest_pct_float DOUBLE PRECISION, - days_to_cover DOUBLE PRECISION, - PRIMARY KEY (symbol, report_date) -); -- ===================================================================== -- Features / setups / outcomes / backtests @@ -252,102 +215,14 @@ CREATE TABLE IF NOT EXISTS outcome_labels ( time_to_max_drawdown_minutes DOUBLE PRECISION ); -CREATE TABLE IF NOT EXISTS backtest_runs ( - run_id VARCHAR PRIMARY KEY, - run_timestamp TIMESTAMP, - strategy_name VARCHAR, - strategy_version VARCHAR, - feature_version VARCHAR, - date_range_start DATE, - date_range_end DATE, - parameter_config_json VARCHAR, - summary_metrics_json VARCHAR, - total_trades INTEGER, - win_rate DOUBLE PRECISION, - expectancy DOUBLE PRECISION, - max_drawdown DOUBLE PRECISION, - profit_factor DOUBLE PRECISION -); -- ===================================================================== -- Paper trading -- ===================================================================== -CREATE TABLE IF NOT EXISTS simulated_orders ( - order_id VARCHAR PRIMARY KEY, - session_id VARCHAR, - symbol VARCHAR, - created_at TIMESTAMP, - side VARCHAR, - order_type VARCHAR, - qty INTEGER, - intended_entry DOUBLE PRECISION, - stop_price DOUBLE PRECISION, - target_price DOUBLE PRECISION, - status VARCHAR, - strategy_name VARCHAR, - setup_id VARCHAR -); -CREATE TABLE IF NOT EXISTS simulated_fills ( - fill_id VARCHAR PRIMARY KEY, - order_id VARCHAR, - symbol VARCHAR, - fill_time TIMESTAMP, - fill_price DOUBLE PRECISION, - qty INTEGER, - slippage DOUBLE PRECISION, - commission DOUBLE PRECISION, - venue VARCHAR -); -CREATE TABLE IF NOT EXISTS strategy_runs ( - run_id VARCHAR PRIMARY KEY, - started_at TIMESTAMP, - ended_at TIMESTAMP, - mode VARCHAR, - strategy_name VARCHAR, - strategy_version VARCHAR, - config_json VARCHAR, - summary_json VARCHAR -); -CREATE TABLE IF NOT EXISTS paper_sessions ( - session_id VARCHAR PRIMARY KEY, - trade_date DATE, - started_at TIMESTAMP, - ended_at TIMESTAMP, - starting_equity DOUBLE PRECISION, - ending_equity DOUBLE PRECISION, - realized_pnl DOUBLE PRECISION, - unrealized_pnl DOUBLE PRECISION, - total_orders INTEGER, - total_fills INTEGER, - total_setups INTEGER, - notes VARCHAR -); -CREATE TABLE IF NOT EXISTS paper_missed_trades ( - audit_id VARCHAR PRIMARY KEY, - session_id VARCHAR, - symbol VARCHAR, - setup_id VARCHAR, - reason VARCHAR, - setup_time TIMESTAMP, - evaluated_at TIMESTAMP, - attempted_entry_time TIMESTAMP, - entry_cutoff VARCHAR, - minutes_past_cutoff DOUBLE PRECISION, - success_score_pct DOUBLE PRECISION, - criteria_passed INTEGER, - criteria_total INTEGER, - provider_status VARCHAR, - provider_degraded_count INTEGER, - source_status VARCHAR, - source_degraded_count INTEGER, - source_backoff_count INTEGER, - created_at TIMESTAMP, - details_json VARCHAR -); -- ===================================================================== -- Raw landing (append-only) + telemetry + scanner snapshots @@ -412,18 +287,6 @@ CREATE INDEX IF NOT EXISTS idx_news_catalyst_cache_symbol CREATE INDEX IF NOT EXISTS idx_news_catalyst_cache_enriched ON news_catalyst_cache (enriched_at); -CREATE TABLE IF NOT EXISTS telemetry_events ( - id VARCHAR PRIMARY KEY, - event_type VARCHAR, - session_id VARCHAR, - symbol VARCHAR, - source VARCHAR, - from_state VARCHAR, - to_state VARCHAR, - reason VARCHAR, - metadata_json VARCHAR, - created_at TIMESTAMP DEFAULT current_timestamp -); CREATE TABLE IF NOT EXISTS scanner_snapshots ( id VARCHAR PRIMARY KEY, diff --git a/storage/projections.py b/storage/projections.py index b0bff03..d6ec88f 100644 --- a/storage/projections.py +++ b/storage/projections.py @@ -103,27 +103,6 @@ def query_session_summary(store) -> list[dict]: return summaries -def query_symbol_status(store, symbol: str | None = None) -> list[dict]: - """Query current status for symbols.""" - events = store.query_events( - event_type="symbol_state_changed", - symbol=symbol, - ) - - return [ - { - "event_id": event["id"], - "symbol": json.loads(event.get("payload_json", "{}")).get("symbol"), - "previous_state": json.loads(event.get("payload_json", "{}")).get( - "previous_state" - ), - "new_state": json.loads(event.get("payload_json", "{}")).get("new_state"), - "timestamp": event["timestamp"], - } - for event in events - ] - - def query_approval_queue(store) -> list[dict]: requested = store.query_events(event_type="order_approval_requested", limit=None) approved = store.query_events(event_type="order_approved", limit=None) @@ -158,56 +137,6 @@ def query_approval_queue(store) -> list[dict]: return queue -def query_positions_view(store) -> list[dict]: - """Query all positions (opened/closed events).""" - opened = store.query_events(event_type="position_opened") - closed = store.query_events(event_type="position_closed") - position_events = opened + closed - - positions = [] - for event in position_events: - payload = json.loads(event.get("payload_json", "{}")) - positions.append( - { - "event_id": event["id"], - "position_id": payload.get("position_id"), - "symbol": payload.get("symbol"), - "event_type": event["event_type"], - "entry_price": payload.get("entry_price"), - "quantity": payload.get("quantity"), - "stop_loss_price": payload.get("stop_loss_price"), - "exit_price": payload.get("exit_price") - if event["event_type"] == "position_closed" - else None, - "exit_reason": payload.get("exit_reason") - if event["event_type"] == "position_closed" - else None, - "realized_pnl": payload.get("realized_pnl") - if event["event_type"] == "position_closed" - else None, - "timestamp": event["timestamp"], - } - ) - - return positions - - -def query_event_timeline(store, session_id: str | None = None) -> list[dict]: - """Query full event timeline for a session.""" - return store.query_events(session_id=session_id, limit=None) - - -def query_settings_snapshot(store) -> dict: - """Query current settings snapshot.""" - events = store.query_events(event_type="settings_snapshot") - - if not events: - return {} - - latest_event = events[-1] - return json.loads(latest_event.get("payload_json", "{}")) - - def query_ready_signals_snapshot(store, session_id: str | None = None) -> list[dict]: """Latest signal_ready per symbol, newest first. Feeds the dashboard.""" events = store.query_events( diff --git a/strategy/evaluation/structure.py b/strategy/evaluation/structure.py index 68c8307..6077260 100644 --- a/strategy/evaluation/structure.py +++ b/strategy/evaluation/structure.py @@ -7,7 +7,6 @@ based on Ross Cameron momentum strategy specifications. import os from dataclasses import dataclass from enum import Enum -from typing import Literal import numpy as np import pandas as pd @@ -192,7 +191,6 @@ def detect_first_pullback( highs = bars["high"].values lows = bars["low"].values - closes = bars["close"].values # Find impulse high in early session window (avoid late-day highs) search_end = min(len(highs), impulse_search_bars)