"""Ticker verification against the tape + tape stats (ported from the pilot's join.py/ourside.py). A transcript ticker is only trusted when that symbol's SIP minute tape that day can produce the story he told: his entry price traded, then (if he named one) a high near his stated high, then (if named) a low near his stated low. Otherwise we search the day's scanner names for the one unique symbol whose tape fits. The scoring/resolution is pure -- bars come from an injected `bars_for(symbol)` callable, so it is unit-tested with fake bars.""" from __future__ import annotations import datetime as dt import json import re from pathlib import Path from typing import Callable, Sequence # a bar is (ts_et naive datetime, open, high, low, close, volume) Bar = tuple _SYM = re.compile(r"[A-Z]{1,5}") def path_score(bars: Sequence[Bar], entry: float | None, sh: float | None, sl: float | None) -> float | None: """None if the tape can't produce his story; else a closeness score (lower = better). Story: entry trades (+-2% of the day's range), then a high within 2.5% of stated_high, then a low within 3% of stated_low.""" if not bars: return None lo = min(b[3] for b in bars) hi = max(b[2] for b in bars) if entry is not None and not (lo * 0.98 <= entry <= hi * 1.02): return None if sh is None: return 0.5 if entry is not None else None t_entry = next((i for i, b in enumerate(bars) if entry is None or (b[3] <= entry * 1.01 and b[2] >= entry * 0.99)), None) if t_entry is None: return None best = None for i in range(t_entry, len(bars)): err = abs(bars[i][2] - sh) / sh if err > 0.025: continue e2 = err if sl is not None: later = [abs(b[3] - sl) / sl for b in bars[i:]] if not later or min(later) > 0.03: continue e2 += min(later) best = e2 if best is None else min(best, e2) return best def loose_fit(bars: Sequence[Bar], entry: float | None, sh: float | None, sl: float | None) -> bool: """Order-free check: every level he named traded at SOME point that day (entry inside the day's range, a bar high within 2.5% of stated_high, a bar low within 3% of stated_low). Used only for the ticker he NAMED -- the name is a strong prior, and he often tells the flush-then-squeeze out of order ("squeezed to 5 ... it had come from 2.35").""" if not bars or (entry is None and sh is None): return False lo, hi = min(b[3] for b in bars), max(b[2] for b in bars) if entry is not None and not (lo * 0.98 <= entry <= hi * 1.02): return False if sh is not None and not any(abs(b[2] - sh) / sh <= 0.025 for b in bars): return False if sl is not None and not any(abs(b[3] - sl) / sl <= 0.03 for b in bars): return False return True def resolve(item: dict, bars_for: Callable[[str], Sequence[Bar]], scanner_syms: Sequence[str]) -> tuple[str | None, str]: """(symbol, how). how: price_verified (his name fits the tape in order) | price_verified_loose (his name has all his levels, out of order) | price_matched_scanner (a unique scanner name fits in order) | ambiguous:A,B | no_price_match | unverified (no price given).""" heard = (item.get("ticker_heard") or "").upper().replace(" ", "") guess = (item.get("ticker_guess") or "").upper().replace(" ", "") entry, sh, sl = item.get("entry_price"), item.get("stated_high"), item.get("stated_low") cands = [c for c in dict.fromkeys([guess, heard]) if c and _SYM.fullmatch(c)] if entry is None and sh is None: return (cands[0] if cands else None), "unverified" for c in cands: if path_score(bars_for(c), entry, sh, sl) is not None: return c, "price_verified" for c in cands: if loose_fit(bars_for(c), entry, sh, sl): return c, "price_verified_loose" scored = [] for s in dict.fromkeys(scanner_syms): if s in cands: continue sc = path_score(bars_for(s), entry, sh, sl) if sc is not None: scored.append((sc, s)) scored.sort() if len(scored) == 1 or (len(scored) > 1 and scored[0][0] + 0.01 < scored[1][0]): return scored[0][1], "price_matched_scanner" if scored: return None, "ambiguous:" + ",".join(s for _, s in scored[:4]) return (cands[0] if cands else None), "no_price_match" def entry_time(bars: Sequence[Bar], entry: float | None, sh: float | None) -> dt.datetime | None: """First minute the tape traded his entry price, before his stated high (if any).""" if entry is None or not bars: return None stop = len(bars) if sh: hi_i = next((i for i, b in enumerate(bars) if abs(b[2] - sh) / sh <= 0.025), None) if hi_i is not None: stop = hi_i + 1 for b in bars[:stop]: if b[3] <= entry * 1.005 and b[2] >= entry * 0.995: return b[0] return None def entry_bucket(t: dt.datetime | None) -> str | None: if t is None: return None tt = t.time() return ("pre-7:00" if tt < dt.time(7) else "7:00-9:30" if tt < dt.time(9, 30) else "9:30-9:45" if tt < dt.time(9, 45) else "9:45+") def tape_stats(bars: Sequence[Bar], prev_close: float | None) -> dict: """Premarket/RTH summary of one symbol-day (times are ET).""" out: dict = {"prev_close": prev_close} if not bars: return out pre = [b for b in bars if b[0].time() < dt.time(9, 30)] rth = [b for b in bars if dt.time(9, 30) <= b[0].time() < dt.time(16, 0)] if pre: out.update(pm_high=max(b[2] for b in pre), pm_low=min(b[3] for b in pre), pm_volume=sum(b[5] for b in pre)) if rth: o = rth[0][1] hod = max(rth, key=lambda b: b[2]) out.update(open=o, hod=hod[2], hod_time=hod[0].strftime("%H:%M"), lod=min(b[3] for b in rth), close=rth[-1][4], rth_volume=sum(b[5] for b in rth), gap_pct=(o / prev_close - 1) if prev_close else None, open_to_hod_pct=hod[2] / o - 1 if o else None) top = max(bars, key=lambda b: b[2]) out["day_high"] = top[2] out["day_high_time"] = top[0].strftime("%H:%M") first = next((b for b in bars if prev_close and b[2] >= prev_close * 1.3), None) out["first_30pct_time"] = first[0].strftime("%H:%M") if first else None return out class BarSource: """Complete SIP minute tape (04:00-20:00 ET) from Alpaca, cached on disk as JSON per symbol-day (bars of a finished day never change). Falls back to our own minute_bars when Alpaca has nothing -- ours miss the pre-discovery minutes of late-found names.""" def __init__(self, cache_dir: Path, paper_con=None): self.cache_dir = Path(cache_dir) self.paper_con = paper_con self._mem: dict = {} self._client = None def _alpaca(self): if self._client is None: from alpaca_paper.client import AlpacaPaperClient from alpaca_paper.settings import AlpacaPaperSettings self._client = AlpacaPaperClient(AlpacaPaperSettings.from_env()) return self._client def bars(self, sym: str, d: str) -> list[Bar]: key = (sym, str(d)) if key in self._mem: return self._mem[key] f = self.cache_dir / str(d) / f"{sym}.json" if f.exists(): rows = json.loads(f.read_text()) out = [(dt.datetime.fromisoformat(r[0]), *r[1:]) for r in rows] self._mem[key] = out return out out = self._fetch(sym, str(d)) if out or dt.date.fromisoformat(str(d)) < dt.date.today(): f.parent.mkdir(parents=True, exist_ok=True) f.write_text(json.dumps([(b[0].isoformat(), *b[1:]) for b in out])) self._mem[key] = out return out def _fetch(self, sym: str, d: str) -> list[Bar]: from zoneinfo import ZoneInfo et = ZoneInfo("America/New_York") day = dt.date.fromisoformat(d) start = dt.datetime.combine(day, dt.time(4), et) end = dt.datetime.combine(day, dt.time(20), et) raw = [] try: raw = self._alpaca().get_minute_bars( [sym], start_iso=start.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), end_iso=end.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), feed="sip").get(sym) or [] except Exception: # noqa: BLE001 -- unknown/delisted symbol, network: fall back below raw = [] out = [] for b in raw: ts = dt.datetime.fromisoformat(b["t"].replace("Z", "+00:00")).astimezone(et).replace(tzinfo=None) out.append((ts, float(b["o"]), float(b["h"]), float(b["l"]), float(b["c"]), float(b["v"]))) if out or self.paper_con is None: return out with self.paper_con.cursor() as cur: # our own bars (UTC-naive) as a fallback cur.execute("select timestamp, open, high, low, close, volume from minute_bars " "where symbol=%s and session_date=%s order by timestamp", (sym, d)) rows = cur.fetchall() utc = dt.timezone.utc return [(r[0].replace(tzinfo=utc).astimezone(et).replace(tzinfo=None), float(r[1]), float(r[2]), float(r[3]), float(r[4]), float(r[5] or 0)) for r in rows]