Incident: entry-timeout cancels raced lazy sandbox fills -> untracked
positions; naked-stop guard then force-sold but every sell rejected
('more shares than your current long position') for hours.
1. translate_order: Tradier OTOCO/OTO parents carry no top-level
symbol/side/qty (legs only) — inherit from the entry leg so
symbol-scoped lookups can see brackets; get_orders symbols filter
also matches by leg symbols. Without this the stop-leg search read
a fully-protected position as naked.
2. close_position: cancel the symbol's resting sell orders/legs first —
Tradier reserves held shares against open sells, so selling without
releasing them always rejects.
3. cancel_entry (broker-agnostic): after cancelling, re-check the
broker's final word by client_order_id; journal a fill for any
executed shares instead of orphaning a real position (2026-07-06
class, same race exists on Alpaca).
49 tests pass incl. new coverage for all three.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
328 lines
13 KiB
Python
328 lines
13 KiB
Python
"""Alpaca paper execution bridge.
|
|
|
|
Turns an approved execution request into a real Alpaca paper order and
|
|
emits order lifecycle events into the event store.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
|
|
from alpaca_paper.client import AlpacaApiError, AlpacaPaperClient
|
|
from storage.event_schema import (
|
|
EventMode,
|
|
OrderCancelledEvent,
|
|
OrderFilledEvent,
|
|
OrderSubmittedEvent,
|
|
)
|
|
from storage.event_store import EventStore
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class ExecutionRequest:
|
|
"""Broker-agnostic order request derived from a signal."""
|
|
|
|
symbol: str
|
|
side: str
|
|
quantity: int
|
|
entry_price: float | None = None
|
|
stop_loss_price: float | None = None
|
|
take_profit_price: float | None = None
|
|
order_type: str = "market"
|
|
time_in_force: str = "day"
|
|
order_id: str = ""
|
|
|
|
def __post_init__(self):
|
|
if not self.order_id:
|
|
self.order_id = str(uuid.uuid4())
|
|
|
|
def to_payload(self) -> dict:
|
|
return {
|
|
"symbol": self.symbol,
|
|
"side": self.side,
|
|
"quantity": self.quantity,
|
|
"entry_price": self.entry_price,
|
|
"stop_loss_price": self.stop_loss_price,
|
|
"take_profit_price": self.take_profit_price,
|
|
"order_type": self.order_type,
|
|
"time_in_force": self.time_in_force,
|
|
"order_id": self.order_id,
|
|
}
|
|
|
|
@classmethod
|
|
def from_payload(cls, payload: dict) -> "ExecutionRequest":
|
|
return cls(
|
|
symbol=payload["symbol"],
|
|
side=payload.get("side", "buy"),
|
|
quantity=int(payload.get("quantity", 0)),
|
|
entry_price=payload.get("entry_price"),
|
|
stop_loss_price=payload.get("stop_loss_price"),
|
|
take_profit_price=payload.get("take_profit_price"),
|
|
order_type=payload.get("order_type", "market"),
|
|
time_in_force=payload.get("time_in_force", "day"),
|
|
order_id=str(payload.get("order_id") or ""),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class ExecutionResult:
|
|
ok: bool
|
|
order_id: str
|
|
broker_order_id: str | None = None
|
|
status: str | None = None
|
|
error: str | None = None
|
|
raw: dict | None = None
|
|
|
|
|
|
class AlpacaPaperExecutor:
|
|
"""Submits execution requests to the Alpaca paper account."""
|
|
|
|
broker_name = "alpaca_paper"
|
|
|
|
def __init__(
|
|
self,
|
|
store: EventStore,
|
|
client: AlpacaPaperClient | None = None,
|
|
session_id: str | None = None,
|
|
mode: EventMode = EventMode.PAPER,
|
|
):
|
|
self.store = store
|
|
self.client = client or AlpacaPaperClient()
|
|
# honest broker label in journaled events when a mirror client
|
|
# (e.g. TradierClient) is injected
|
|
self.broker_name = getattr(self.client, "broker_name", "alpaca_paper")
|
|
self.session_id = session_id
|
|
self.mode = mode
|
|
|
|
def execute(self, request: ExecutionRequest) -> ExecutionResult:
|
|
if request.quantity <= 0:
|
|
return ExecutionResult(
|
|
ok=False, order_id=request.order_id, error="quantity must be > 0"
|
|
)
|
|
try:
|
|
raw = self.client.submit_order(
|
|
symbol=request.symbol,
|
|
qty=request.quantity,
|
|
side=request.side,
|
|
order_type=request.order_type,
|
|
time_in_force=request.time_in_force,
|
|
limit_price=request.entry_price
|
|
if request.order_type == "limit"
|
|
else None,
|
|
stop_loss_price=request.stop_loss_price,
|
|
take_profit_price=request.take_profit_price,
|
|
client_order_id=request.order_id[:48],
|
|
)
|
|
except AlpacaApiError as exc:
|
|
logger.error("alpaca order failed: %s", exc)
|
|
self._emit_cancelled(request, f"broker rejected: {exc}")
|
|
return ExecutionResult(
|
|
ok=False, order_id=request.order_id, error=str(exc)
|
|
)
|
|
except Exception as exc:
|
|
# LOST RESPONSE: the POST may have LANDED before the connection dropped,
|
|
# leaving a real order resting at the broker. Reconcile by the
|
|
# client_order_id we sent before assuming it was cancelled — journalling
|
|
# a live order as 'cancelled' is how a duplicate/naked order is born
|
|
# (the 2026-07-06 incident). Only the generic/transport error path
|
|
# reconciles; an AlpacaApiError above is a genuine broker rejection.
|
|
logger.exception("alpaca order failed")
|
|
recovered = self._reconcile_lost_submit(request)
|
|
if recovered is not None:
|
|
logger.warning("recovered lost submit for %s via client_order_id "
|
|
"(broker status=%s)", request.symbol, recovered.get("status"))
|
|
return self._finalize(request, recovered)
|
|
self._emit_cancelled(request, f"submit error (reconcile: not found): {exc}")
|
|
return ExecutionResult(
|
|
ok=False, order_id=request.order_id, error=str(exc)
|
|
)
|
|
|
|
return self._finalize(request, raw)
|
|
|
|
_LIVE_STATUSES = {"new", "accepted", "pending_new", "partially_filled",
|
|
"filled", "held", "accepted_for_bidding"}
|
|
|
|
def _reconcile_lost_submit(self, request: "ExecutionRequest") -> dict | None:
|
|
"""Look up the order by the client_order_id we sent; return it only if it
|
|
is actually LIVE at the broker (so a lost response isn't journaled as a
|
|
cancel). Returns None on 404/lookup error — then the caller emits cancelled."""
|
|
getter = getattr(self.client, "get_order_by_client_order_id", None)
|
|
if getter is None:
|
|
return None
|
|
try:
|
|
found = getter(request.order_id[:48])
|
|
except Exception: # noqa: BLE001 — 404 (never landed) or transport error
|
|
return None
|
|
if found and str(found.get("status")) in self._LIVE_STATUSES:
|
|
return found
|
|
return None
|
|
|
|
def _finalize(self, request: "ExecutionRequest", raw: dict) -> ExecutionResult:
|
|
"""Record a submission's outcome from the broker response (or a recovered
|
|
order) and return the ExecutionResult — one event sequence for the normal
|
|
and the reconcile-recovered paths alike."""
|
|
broker_order_id = raw.get("id")
|
|
status = raw.get("status")
|
|
|
|
# An empty/odd 200 body with no order id is not a real submission — don't
|
|
# return ok=True and let it be armed as an untrackable resting order.
|
|
if not broker_order_id:
|
|
self._emit_cancelled(request, f"broker returned no order id (status={status})")
|
|
return ExecutionResult(
|
|
ok=False, order_id=request.order_id, status="error",
|
|
error="no broker order id", raw=raw,
|
|
)
|
|
|
|
self.store.emit(
|
|
OrderSubmittedEvent(
|
|
timestamp=datetime.now(),
|
|
mode=self.mode,
|
|
correlation_id=self.session_id,
|
|
message=(
|
|
f"Submitted {request.side} {request.quantity} {request.symbol} "
|
|
f"to {self.broker_name} ({status})"
|
|
),
|
|
symbol=request.symbol,
|
|
order_id=request.order_id,
|
|
side=request.side,
|
|
quantity=request.quantity,
|
|
price=request.entry_price or 0.0,
|
|
payload={
|
|
"symbol": request.symbol,
|
|
"order_id": request.order_id,
|
|
"side": request.side,
|
|
"quantity": request.quantity,
|
|
"price": request.entry_price,
|
|
"broker_order_id": broker_order_id,
|
|
"broker_status": status,
|
|
},
|
|
)
|
|
)
|
|
|
|
# Record ANY fill (partial or full), not only an exact 'filled' status,
|
|
# so positions / P&L never under-report real shares.
|
|
filled_qty = int(float(raw.get("filled_qty") or 0))
|
|
if filled_qty > 0:
|
|
fill_price = float(raw.get("filled_avg_price") or 0) or None
|
|
self.store.emit(
|
|
OrderFilledEvent(
|
|
timestamp=datetime.now(),
|
|
mode=self.mode,
|
|
correlation_id=self.session_id,
|
|
message=f"Filled {request.symbol} {filled_qty} @ {fill_price}",
|
|
symbol=request.symbol,
|
|
order_id=request.order_id,
|
|
fill_price=fill_price or 0.0,
|
|
fill_quantity=filled_qty,
|
|
side=request.side,
|
|
payload={
|
|
"symbol": request.symbol,
|
|
"order_id": request.order_id,
|
|
"fill_price": fill_price,
|
|
"fill_quantity": filled_qty,
|
|
"side": request.side,
|
|
},
|
|
)
|
|
)
|
|
|
|
# Terminal-failure statuses are NOT live orders: report ok=False so the
|
|
# caller never arms a rejected/expired order as a resting entry.
|
|
if status in {"rejected", "canceled", "cancelled", "expired",
|
|
"done_for_day", "suspended", "stopped"}:
|
|
self._emit_cancelled(request, f"broker {status}")
|
|
return ExecutionResult(
|
|
ok=False, order_id=request.order_id,
|
|
broker_order_id=broker_order_id, status=status, raw=raw,
|
|
)
|
|
|
|
return ExecutionResult(
|
|
ok=True,
|
|
order_id=request.order_id,
|
|
broker_order_id=broker_order_id,
|
|
status=status,
|
|
raw=raw,
|
|
)
|
|
|
|
def cancel_entry(
|
|
self, order_id: str, broker_order_id: str | None, symbol: str, reason: str
|
|
) -> ExecutionResult:
|
|
"""Cancel a resting (unfilled) entry order at the broker and record it.
|
|
|
|
CANCEL-vs-FILL RACE: the entry can fill at the broker before (or as)
|
|
the cancel lands — the cancel then no-ops but journaling only a cancel
|
|
orphans a REAL position (untracked shares, the 2026-07-06 class;
|
|
happened live on Tradier sandbox 2026-07-28: lazy fills + entry
|
|
timeouts). After cancelling, re-check the broker's final word and emit
|
|
the fill for any executed shares so the position is tracked."""
|
|
try:
|
|
if broker_order_id:
|
|
self.client.cancel_order(broker_order_id)
|
|
except AlpacaApiError as exc:
|
|
# already filled/cancelled at the broker, or transient — log and
|
|
# continue to the fill-race check below
|
|
logger.warning("cancel of %s returned: %s", order_id, exc)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("cancel of %s errored: %s", order_id, exc)
|
|
|
|
filled_qty, fill_price, side = 0, None, "buy"
|
|
getter = getattr(self.client, "get_order_by_client_order_id", None)
|
|
if getter is not None:
|
|
try:
|
|
found = getter(order_id[:48]) or {}
|
|
filled_qty = int(float(found.get("filled_qty") or 0))
|
|
fill_price = float(found.get("filled_avg_price") or 0) or None
|
|
side = found.get("side") or "buy"
|
|
except Exception: # noqa: BLE001 — lookup failure: keep cancel-only path
|
|
pass
|
|
|
|
if filled_qty > 0:
|
|
logger.warning("cancel of %s raced a fill: %d %s executed @ %s — "
|
|
"journaling the fill", order_id, filled_qty, symbol, fill_price)
|
|
self.store.emit(
|
|
OrderFilledEvent(
|
|
timestamp=datetime.now(),
|
|
mode=self.mode,
|
|
correlation_id=self.session_id,
|
|
message=f"Filled {symbol} {filled_qty} @ {fill_price} "
|
|
f"(cancel raced fill)",
|
|
symbol=symbol,
|
|
order_id=order_id,
|
|
fill_price=fill_price or 0.0,
|
|
fill_quantity=filled_qty,
|
|
side=side,
|
|
payload={"symbol": symbol, "order_id": order_id,
|
|
"fill_price": fill_price, "fill_quantity": filled_qty,
|
|
"side": side, "cancel_raced_fill": True},
|
|
)
|
|
)
|
|
return ExecutionResult(ok=True, order_id=order_id, status="filled")
|
|
|
|
self._emit_cancelled(
|
|
ExecutionRequest(symbol=symbol, side="buy", quantity=0, order_id=order_id),
|
|
reason,
|
|
)
|
|
return ExecutionResult(ok=True, order_id=order_id, status="cancelled")
|
|
|
|
def _emit_cancelled(self, request: ExecutionRequest, reason: str) -> None:
|
|
self.store.emit(
|
|
OrderCancelledEvent(
|
|
timestamp=datetime.now(),
|
|
mode=self.mode,
|
|
correlation_id=self.session_id,
|
|
message=f"Order for {request.symbol} cancelled: {reason}",
|
|
symbol=request.symbol,
|
|
order_id=request.order_id,
|
|
cancel_reason=reason,
|
|
payload={
|
|
"symbol": request.symbol,
|
|
"order_id": request.order_id,
|
|
"cancel_reason": reason,
|
|
},
|
|
)
|
|
)
|