diff --git a/backfill_history.py b/backfill_history.py new file mode 100644 index 0000000..17ddee3 --- /dev/null +++ b/backfill_history.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Backfill historical minute + daily bars for a universe of sub-$20 names, so +learn_params.py has enough real sessions to optimize over. + +Pulls the current sub-$20 most-actives (liquid names that gap often) plus any +symbols already in the DB, then fetches ~3 weeks of 1-minute bars (one paginated, +batched range request) and 60 days of daily bars for the gap/RVOL baseline. + + PYTHONPATH=. /home/philip/.venvs/momentum/bin/python backfill_history.py +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +from dotenv import load_dotenv + +from alpaca_paper.client import AlpacaPaperClient +from alpaca_paper.settings import AlpacaPaperSettings +from research.ingestion.discovery import screen_universe +from research.ingestion.market_data import upsert_daily_bars, upsert_minute_bars +from research.multi_schema import open_research_db + +load_dotenv() + +LOOKBACK_CALENDAR_DAYS = 22 # ~15 trading days + + +def main(): + client = AlpacaPaperClient(AlpacaPaperSettings.from_env()) + con = open_research_db("market") + + universe = screen_universe(client, 1.0, 20.0, 40) + existing = [r[0] for r in con.execute("SELECT DISTINCT symbol FROM minute_bars").fetchall()] + universe = sorted(set(universe) | set(existing)) + print(f"universe ({len(universe)}): {universe}", flush=True) + if not universe: + print("no universe (screener empty?) — aborting") + return + + now = datetime.now(timezone.utc) + + # daily bars (baseline for gap% / RVOL) + daily = client.get_daily_bars( + universe, start_iso=(now - timedelta(days=60)).date().isoformat() + "T00:00:00Z" + ) + drows = sum(upsert_daily_bars(con, s, b) for s, b in daily.items()) + print(f"daily bars: {drows} rows", flush=True) + + # minute bars: one paginated range request (the client batches symbols <=100) + start = now - timedelta(days=LOOKBACK_CALENDAR_DAYS) + end = now - timedelta(minutes=20) + print(f"fetching minute bars {start.date()} .. {end.date()} (this can take a few min)...", flush=True) + bars = client.get_minute_bars( + universe, + start_iso=start.isoformat().replace("+00:00", "Z"), + end_iso=end.isoformat().replace("+00:00", "Z"), + feed="iex", + ) + total = 0 + for sym, sym_bars in bars.items(): + if sym_bars: + total += upsert_minute_bars(con, sym, sym_bars) + have = len([s for s in bars if bars[s]]) + print(f"minute bars: {total} rows across {have} symbols", flush=True) + + sessions = con.execute("SELECT COUNT(DISTINCT session_date) FROM minute_bars").fetchone()[0] + print(f"distinct sessions in DB now: {sessions}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/strategy/evaluation/setup_evaluator.py b/strategy/evaluation/setup_evaluator.py index a0da84f..0ff9d00 100644 --- a/strategy/evaluation/setup_evaluator.py +++ b/strategy/evaluation/setup_evaluator.py @@ -102,7 +102,8 @@ def evaluate_setup( "pullback", structure.setup_type in (SetupType.FIRST_PULLBACK, SetupType.BULL_FLAG, SetupType.GAP_AND_GO, - SetupType.HOD_BREAK, SetupType.CONTINUATION_FALLBACK) + SetupType.HOD_BREAK, SetupType.CONTINUATION_FALLBACK, + SetupType.OPENING_RANGE_BREAK) and structure.is_valid, "pullback/consolidation structure", ), diff --git a/strategy/evaluation/structure.py b/strategy/evaluation/structure.py index 4911386..20b0000 100644 --- a/strategy/evaluation/structure.py +++ b/strategy/evaluation/structure.py @@ -16,6 +16,7 @@ class SetupType(Enum): """Types of momentum setups.""" GAP_AND_GO = "gap_and_go" + OPENING_RANGE_BREAK = "opening_range_break" FIRST_PULLBACK = "first_pullback" BULL_FLAG = "bull_flag" HOD_BREAK = "hod_break" @@ -529,6 +530,66 @@ def detect_continuation_fallback( ) +def detect_opening_range_breakout( + bars: pd.DataFrame, + orb_bars: int = 5, + min_range_pct: float = 0.004, + min_break_pct: float = 0.0, +) -> StructureDetectionResult: + """Opening-range breakout — the EARLY trigger. + + Fires minutes after the open instead of waiting for a pullback to form: the + opening range is the high/low of the first ``orb_bars`` regular-hours bars + (09:30-09:3x); a later bar closing above that high, on at least opening-range + average volume, is the breakout. Entry = OR high, stop = OR low. + """ + if "is_regular_hours" in bars.columns: + rth = bars[bars["is_regular_hours"] == True] # noqa: E712 + if rth.empty: + return StructureDetectionResult( + SetupType.OPENING_RANGE_BREAK, False, reason="no regular-hours bars") + rth = rth.reset_index(drop=True) + else: + rth = bars.reset_index(drop=True) + + if len(rth) < orb_bars + 1: + return StructureDetectionResult( + SetupType.OPENING_RANGE_BREAK, False, reason="opening range not complete") + + orb = rth.iloc[:orb_bars] + orb_high = float(orb["high"].max()) + orb_low = float(orb["low"].min()) + if orb_high <= 0 or (orb_high - orb_low) / orb_high < min_range_pct: + return StructureDetectionResult( + SetupType.OPENING_RANGE_BREAK, False, reason="opening range too tight") + + after = rth.iloc[orb_bars:] + last_close = float(after["close"].iloc[-1]) + if last_close < orb_high * (1.0 + min_break_pct): + return StructureDetectionResult( + SetupType.OPENING_RANGE_BREAK, False, reason="no opening-range break yet") + + orb_vol = float(orb["volume"].mean()) if "volume" in orb.columns else 0.0 + break_vol = float(after["volume"].iloc[-1]) if "volume" in after.columns else 0.0 + vol_ok = orb_vol <= 0 or break_vol >= 0.8 * orb_vol + + stop = orb_low * 0.995 + if stop >= orb_high: + return StructureDetectionResult( + SetupType.OPENING_RANGE_BREAK, False, reason="invalid ORB risk geometry") + + extension = (last_close - orb_high) / orb_high + quality = 0.4 + min(0.4, extension / 0.02 * 0.4) + (0.2 if vol_ok else 0.0) + return StructureDetectionResult( + setup_type=SetupType.OPENING_RANGE_BREAK, + is_valid=True, + breakout_level=orb_high, + stop_level=float(stop), + quality_score=float(max(0.0, min(1.0, quality))), + reason=None, + ) + + def classify_setup( bars: pd.DataFrame, premarket_high: float | None = None, @@ -549,6 +610,7 @@ def classify_setup( StructureDetectionResult for best valid setup """ detectors = [ + detect_opening_range_breakout(bars), detect_bull_flag(bars, min_impulse_pct=0.015), detect_first_pullback(bars, max_pullback_depth_pct=0.6), detect_gap_and_go(bars, premarket_high, min_gap_pct=0.02),