From f3c44559d3681c0f70ecd8f201d1fc10dedfa824 Mon Sep 17 00:00:00 2001 From: KoalaTrapLord Date: Fri, 13 Mar 2026 15:25:30 +0000 Subject: [PATCH] feat: Scryfall response caching with SQLite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _ScryfallCache class: thread-safe SQLite cache for raw Scryfall JSON responses, keyed by name|set_code|collector_number, 7-day TTL - _fetch_scryfall_api() wrapper: checks cache first, live HTTP on miss, caches successful responses + resolves spelling normalization - Refactored _enrich_from_scryfall() to use cache — all 3 callers (collection import, deck card add, re-enrich) benefit automatically - Rate limiting only on actual HTTP calls, not cache hits - REST endpoints: GET/DELETE /api/cache/scryfall, POST .../evict - Fixed VACUUM-inside-transaction bug in clear() - Configurable TTL via SCRYFALL_CACHE_TTL env var --- lab_api.py | 203 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 198 insertions(+), 5 deletions(-) diff --git a/lab_api.py b/lab_api.py index 349d890..9bff116 100644 --- a/lab_api.py +++ b/lab_api.py @@ -2488,6 +2488,147 @@ async def update_collection_card(cardId: int, body: dict): # Collection Import Helpers # ══════════════════════════════════════════════════════════════ +# ── Scryfall Response Cache ───────────────────────────────── +# SQLite-backed cache for Scryfall API responses. +# Keyed by (name, set_code, collector_number). Default TTL: 7 days. +# Scryfall data changes infrequently (set releases ~every 3 months). +# Eliminates redundant HTTP calls across imports and re-enrichments. + +SCRYFALL_CACHE_DB_PATH = Path(__file__).parent / "scryfall_cache.db" +SCRYFALL_CACHE_TTL_SECONDS = int(os.environ.get("SCRYFALL_CACHE_TTL", 7 * 24 * 3600)) # 7 days + + +class _ScryfallCache: + """ + Thread-safe SQLite cache for raw Scryfall JSON responses. + + Schema: + cache_key TEXT PRIMARY KEY — "name|set|cn" normalised lowercase + json_blob TEXT — raw Scryfall API JSON response + fetched_at TEXT — ISO-8601 UTC timestamp + """ + + def __init__(self, db_path: Path = None, ttl_seconds: int = None): + self._db_path = db_path or SCRYFALL_CACHE_DB_PATH + self._ttl = ttl_seconds or SCRYFALL_CACHE_TTL_SECONDS + self._lock = threading.Lock() + self._conn: Optional[sqlite3.Connection] = None + self._hits = 0 + self._misses = 0 + self._init_db() + + # ── internal ──────────────────────────────────────────── + + def _init_db(self): + self._conn = sqlite3.connect(str(self._db_path), check_same_thread=False) + self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute(""" + CREATE TABLE IF NOT EXISTS scryfall_cache ( + cache_key TEXT PRIMARY KEY, + json_blob TEXT NOT NULL, + fetched_at TEXT NOT NULL DEFAULT (datetime('now')) + ) + """) + self._conn.commit() + + @staticmethod + def _make_key(name: str, set_code: str, collector_number: str) -> str: + return f"{name.strip().lower()}|{(set_code or '').strip().lower()}|{(collector_number or '').strip()}" + + # ── public API ────────────────────────────────────────── + + def get(self, name: str, set_code: str = "", collector_number: str = "") -> Optional[dict]: + """Return cached Scryfall JSON dict, or None if missing / expired.""" + key = self._make_key(name, set_code, collector_number) + with self._lock: + row = self._conn.execute( + "SELECT json_blob, fetched_at FROM scryfall_cache WHERE cache_key = ?", + (key,), + ).fetchone() + if not row: + self._misses += 1 + return None + # TTL check + try: + fetched = datetime.fromisoformat(row[1]) + age = (datetime.utcnow() - fetched).total_seconds() + if age > self._ttl: + self._misses += 1 + return None + except (ValueError, TypeError): + self._misses += 1 + return None + self._hits += 1 + try: + return json.loads(row[0]) + except json.JSONDecodeError: + self._misses += 1 + return None + + def put(self, name: str, set_code: str, collector_number: str, card_data: dict): + """Store a Scryfall response in the cache.""" + key = self._make_key(name, set_code, collector_number) + blob = json.dumps(card_data, separators=(",", ":")) + with self._lock: + self._conn.execute( + "INSERT OR REPLACE INTO scryfall_cache (cache_key, json_blob, fetched_at) " + "VALUES (?, ?, datetime('now'))", + (key, blob), + ) + self._conn.commit() + + def stats(self) -> dict: + """Return cache statistics.""" + with self._lock: + total = self._conn.execute("SELECT COUNT(*) FROM scryfall_cache").fetchone()[0] + oldest_row = self._conn.execute( + "SELECT MIN(fetched_at) FROM scryfall_cache" + ).fetchone() + newest_row = self._conn.execute( + "SELECT MAX(fetched_at) FROM scryfall_cache" + ).fetchone() + db_size_bytes = os.path.getsize(str(self._db_path)) if self._db_path.exists() else 0 + return { + "total_entries": total, + "hits": self._hits, + "misses": self._misses, + "hit_rate": round(self._hits / max(self._hits + self._misses, 1) * 100, 1), + "ttl_seconds": self._ttl, + "ttl_days": round(self._ttl / 86400, 1), + "oldest_entry": oldest_row[0] if oldest_row else None, + "newest_entry": newest_row[0] if newest_row else None, + "db_size_kb": round(db_size_bytes / 1024, 1), + "db_path": str(self._db_path), + } + + def clear(self) -> int: + """Delete all cached entries. Returns count of deleted rows.""" + with self._lock: + count = self._conn.execute("SELECT COUNT(*) FROM scryfall_cache").fetchone()[0] + self._conn.execute("DELETE FROM scryfall_cache") + self._conn.commit() + self._conn.execute("VACUUM") # must run outside transaction + self._hits = 0 + self._misses = 0 + return count + + def evict_expired(self) -> int: + """Remove entries older than TTL. Returns count of evicted rows.""" + cutoff = datetime.utcnow().isoformat() + with self._lock: + self._conn.execute( + "DELETE FROM scryfall_cache WHERE " + "(julianday(?) - julianday(fetched_at)) * 86400 > ?", + (cutoff, self._ttl), + ) + deleted = self._conn.total_changes + self._conn.commit() + return deleted + + +_scryfall_cache = _ScryfallCache() + + # Scryfall rate-limit semaphore (max 10 req/sec) _scryfall_lock = threading.Lock() _scryfall_last_call = 0.0 @@ -2671,20 +2812,26 @@ def _parse_csv_content(content: str, source: str, mapping: Optional[dict]) -> li return rows -def _enrich_from_scryfall(name: str, set_code: str = "", collector_number: str = "") -> dict: +def _fetch_scryfall_api(name: str, set_code: str = "", collector_number: str = "") -> dict: """ - Fetch card data from Scryfall and return enriched fields dict. - Tries set+collectorNumber first, then exact name lookup. - Returns empty dict on failure. + Fetch raw card data from Scryfall API (with caching). + Returns the raw Scryfall JSON dict, or a dict with '_error' on failure. + Cache is checked first; live HTTP call only on miss. """ from urllib.parse import quote - _scryfall_rate_limit() + # ── Cache check ────────────────────────────────────── + cached = _scryfall_cache.get(name, set_code, collector_number) + if cached is not None: + return cached + + # ── Live fetch ─────────────────────────────────────── card_data = None last_error = "" # Try set + collector number first if set_code and collector_number: + _scryfall_rate_limit() try: url = f"https://api.scryfall.com/cards/{set_code.lower()}/{collector_number}" req = Request(url, headers=_API_HEADERS) @@ -2711,6 +2858,27 @@ def _enrich_from_scryfall(name: str, set_code: str = "", collector_number: str = err_detail = card_data.get("details", "unknown error") if card_data else last_error return {"_error": f"Scryfall returned error for '{name}': {err_detail}"} + # ── Cache the successful response ──────────────────── + _scryfall_cache.put(name, set_code, collector_number, card_data) + # Also cache under the Scryfall-resolved name (handles spelling normalisation) + resolved_name = card_data.get("name", "") + if resolved_name and resolved_name.lower() != name.strip().lower(): + _scryfall_cache.put(resolved_name, set_code, collector_number, card_data) + + return card_data + + +def _enrich_from_scryfall(name: str, set_code: str = "", collector_number: str = "") -> dict: + """ + Fetch card data from Scryfall (cached) and return enriched fields dict. + Tries set+collectorNumber first, then exact name lookup. + Returns empty dict on failure. + """ + card_data = _fetch_scryfall_api(name, set_code, collector_number) + + if not card_data or "_error" in card_data: + return card_data or {} + # Extract fields type_line = card_data.get("type_line", "") color_identity = card_data.get("color_identity", []) @@ -2927,6 +3095,31 @@ async def import_collection(body: dict): } +# ══════════════════════════════════════════════════════════════ +# Scryfall Cache Management Endpoints +# ══════════════════════════════════════════════════════════════ + + +@app.get("/api/cache/scryfall") +async def scryfall_cache_stats(): + """Return Scryfall response cache statistics.""" + return _scryfall_cache.stats() + + +@app.delete("/api/cache/scryfall") +async def scryfall_cache_clear(): + """Clear all cached Scryfall responses.""" + deleted = _scryfall_cache.clear() + return {"cleared": deleted, "message": f"Deleted {deleted} cached entries"} + + +@app.post("/api/cache/scryfall/evict") +async def scryfall_cache_evict_expired(): + """Remove only expired entries (older than TTL) from the cache.""" + evicted = _scryfall_cache.evict_expired() + return {"evicted": evicted, "message": f"Evicted {evicted} expired entries"} + + # ══════════════════════════════════════════════════════════════ # Deck Builder Helpers # ══════════════════════════════════════════════════════════════