Breaker: recoverable data-halt + network retries (DNS blip ended the day)

A transient WSL DNS outage made 3 consecutive equity reads fail, and the
fail-CLOSED breaker then PERMANENTLY halted new entries for the whole session
even after DNS recovered seconds later. Fixes:
- split the breaker into a permanent _halted (real daily-loss breach) and a
  recoverable _data_halt (equity unreadable) that clears on the next good read;
  raise the pause threshold to 5 (TRADING_EQUITY_FAIL_LIMIT).
- AlpacaPaperClient._request retries transient URLError/gaierror (not HTTP
  4xx/5xx) a couple times, so brief blips don't even register as failures.
- sync logs network failures as one-line warnings, not full tracebacks (the
  log was spamming stacks).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

alpaca_paper/client.py

@@ -25,9 +25,13 @@ class AlpacaApiError(RuntimeError):
 
 
 class AlpacaPaperClient:
-    def __init__(self, settings: AlpacaPaperSettings | None = None, timeout: int = 15):
+    def __init__(self, settings: AlpacaPaperSettings | None = None, timeout: int = 15,
+                 max_retries: int = 2):
         self.settings = settings or AlpacaPaperSettings.from_env()
         self.timeout = timeout
+        # retry transient network/DNS errors (not HTTP 4xx/5xx) a couple times;
+        # DNS failures fail fast, so this smooths brief blips without much latency
+        self.max_retries = max_retries
 
     # -- low level -------------------------------------------------------
 
@@ -50,12 +54,21 @@ class AlpacaPaperClient:
         req = urllib.request.Request(
             url, data=data, headers=self._headers(), method=method
         )
-        try:
-            with urllib.request.urlopen(req, timeout=self.timeout) as resp:
-                raw = resp.read().decode()
-                return json.loads(raw) if raw else {}
-        except urllib.error.HTTPError as exc:
-            raise AlpacaApiError(exc.code, exc.read().decode(errors="replace")) from exc
+        import time as _time
+        for attempt in range(self.max_retries + 1):
+            try:
+                with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                    raw = resp.read().decode()
+                    return json.loads(raw) if raw else {}
+            except urllib.error.HTTPError as exc:
+                # a real API response (4xx/5xx) — do not retry, surface it
+                raise AlpacaApiError(exc.code, exc.read().decode(errors="replace")) from exc
+            except urllib.error.URLError:
+                # transient network/DNS (e.g. gaierror) — retry a couple times
+                if attempt >= self.max_retries:
+                    raise
+                _time.sleep(0.4 * (attempt + 1))
+        raise RuntimeError("request retries exhausted")  # unreachable
 
     def _trading(self, method: str, path: str, **kw):
         return self._request(

alpaca_paper/sync.py

@@ -56,7 +56,7 @@ class AlpacaPaperSync:
         try:
             account = self.client.get_account()
         except Exception as exc:
-            logger.exception("alpaca account sync failed")
+            logger.warning("alpaca account sync failed: %s", exc)
             self._emit_health(f"account: {exc}")
             return None
         account_id = str(account.get("account_number") or account.get("id") or "paper")
@@ -81,7 +81,7 @@ class AlpacaPaperSync:
         try:
             raw = self.client.get_positions()
         except Exception as exc:
-            logger.exception("alpaca positions sync failed")
+            logger.warning("alpaca positions sync failed: %s", exc)
             self._emit_health(f"positions: {exc}")
             return None
         positions = [
@@ -114,7 +114,7 @@ class AlpacaPaperSync:
         try:
             raw = self.client.get_orders(status=status, limit=limit)
         except Exception as exc:
-            logger.exception("alpaca orders sync failed")
+            logger.warning("alpaca orders sync failed: %s", exc)
             self._emit_health(f"orders: {exc}")
             return None
         orders = [