diff --git a/lab_api.py b/lab_api.py index 8c3bbef..fa4a6bf 100644 --- a/lab_api.py +++ b/lab_api.py @@ -3,9 +3,13 @@ Commander AI Lab — FastAPI Backend Thin entry point: registers routers and starts uvicorn. """ +from __future__ import annotations + import argparse +import asyncio import logging import os +from contextlib import asynccontextmanager from pathlib import Path try: @@ -41,12 +45,38 @@ from routes.ws_game import router as ws_game_router log = logging.getLogger("commander_ai_lab.api") _DEFAULT_ORIGINS = "http://localhost:5173,http://localhost:3000,http://localhost:8080" -# ── CORS origins (env-configurable) ──────────────────────────── _allowed = os.environ.get("ALLOWED_ORIGINS", _DEFAULT_ORIGINS) allowed_origins = [o.strip() for o in _allowed.split(",") if o.strip()] -# ── App ──────────────────────────────────────────────────────── -app = FastAPI(title="Commander AI Lab API", version="3.0.0") + +# ── Lifespan (replaces deprecated @app.on_event) ─────────────────────────────── +@asynccontextmanager +async def _lifespan(application: FastAPI): + # ──── STARTUP ──── + if not CFG.precon_dir: + CFG.precon_dir = _resolve_precon_dir() + init_collection_db() + if not PRECON_INDEX: + download_precon_database() + if not COMMANDER_META: + load_commander_meta() + + # RAG Phase 1: kick off Scryfall bulk download in background thread. + # run_in_executor returns a Future; we do NOT await it so startup + # completes immediately and the download runs concurrently. + from services.scryfall_bulk import ensure_bulk_db + loop = asyncio.get_event_loop() + loop.run_in_executor(None, ensure_bulk_db) + + yield # ← server is running + + # ──── SHUTDOWN ──── + from services.async_db import async_close_db + await async_close_db() + + +# ── App ──────────────────────────────────────────────────────────────── +app = FastAPI(title="Commander AI Lab API", version="3.0.0", lifespan=_lifespan) app.add_middleware( CORSMiddleware, allow_origins=allowed_origins, @@ -68,32 +98,31 @@ app.include_router(ws_game_router) app.include_router(ml_router) +# ── API endpoints ──────────────────────────────────────────────────────────── +# NOTE: all @app.get routes MUST be registered before app.mount() calls. +# Once StaticFiles is mounted at "/", it registers a wildcard catch-all that +# shadows any routes added after it. + @app.get("/api/health") async def health_check(): """Health check endpoint for load balancers and Unity client polling.""" return {"status": "ok"} -@app.on_event("startup") -async def _on_startup(): - """Ensure DB and precon index are ready (supports uvicorn --reload).""" - # Ensure precon_dir is resolved even when started via uvicorn directly - if not CFG.precon_dir: - CFG.precon_dir = _resolve_precon_dir() - init_collection_db() - if not PRECON_INDEX: - download_precon_database() - if not COMMANDER_META: - load_commander_meta() +@app.get("/api/rag/status") +async def rag_status(): + """ + Returns the current state of the Scryfall bulk card database. + Useful for confirming Phase 1 is live before building the ChromaDB index. + """ + try: + from services.scryfall_bulk import get_stats + return {"bulk_db": get_stats(), "status": "ok"} + except Exception as exc: + return {"status": "error", "error": str(exc)} -@app.on_event("shutdown") -async def _on_shutdown(): - """Close the async SQLite connection pool.""" - from services.async_db import async_close_db - await async_close_db() - -# ── Static UI ────────────────────────────────────────────────── +# ── Static UI (MUST come after all @app.get routes) ──────────────────────────── _legacy_ui_dir = Path(__file__).parent / "ui" _spa_dir = Path(__file__).parent / "frontend" / "commander-ai-lab-ui" / "dist" @@ -103,6 +132,7 @@ elif _spa_dir.exists(): _spa_assets = _spa_dir / "assets" if _spa_assets.exists(): app.mount("/assets", StaticFiles(directory=str(_spa_assets)), name="spa-assets") + @app.get("/{full_path:path}") async def _spa_catchall(full_path: str): from fastapi.responses import FileResponse @@ -111,7 +141,8 @@ elif _spa_dir.exists(): return FileResponse(str(requested_file)) return FileResponse(str(_spa_dir / "index.html")) -# ── Startup ──────────────────────────────────────────────────── + +# ── Argument parsing + auto-detection helpers ───────────────────────────────── def _parse_args(): p = argparse.ArgumentParser(description="Commander AI Lab API Server") p.add_argument("--forge-jar", default=os.environ.get("FORGE_JAR", "")) @@ -125,6 +156,7 @@ def _parse_args(): p.add_argument("--verbose", "-v", action="store_true") return p.parse_args() + def _resolve_forge_decks_dir() -> str: import sys if sys.platform == "win32": @@ -142,52 +174,35 @@ def _resolve_forge_decks_dir() -> str: return str(candidate) return "" + def _resolve_forge_dir() -> str: - """Auto-detect the Forge installation directory. - - The Forge dir is the root of a Forge install — it typically contains - a 'res' subdirectory with card data. We derive it from the JAR - location (parent of 'target/') or look in common install paths. - """ + """Auto-detect the Forge installation directory.""" import sys - - # 1. If forge_jar is already resolved, derive forge_dir from it. - # JAR lives in /forge-gui-desktop/target/ - # so forge_dir = JAR's grandparent's parent. if CFG.forge_jar and os.path.isfile(CFG.forge_jar): jar_path = Path(CFG.forge_jar) - # target/ -> forge-gui-desktop/ -> candidate = jar_path.parent.parent.parent if candidate.is_dir() and (candidate / "res").is_dir(): return str(candidate) - # Also try just one level up from target/ candidate = jar_path.parent.parent - # Check forge-gui sibling of forge-gui-desktop (res/ lives there) forge_gui_sibling = jar_path.parent.parent.parent / "forge-gui" if forge_gui_sibling.is_dir() and (forge_gui_sibling / "res").is_dir(): return str(forge_gui_sibling) if candidate.is_dir() and (candidate / "res").is_dir(): return str(candidate) - # Fall back to the directory containing the JAR itself candidate = jar_path.parent if candidate.is_dir(): return str(candidate) - - # 2. Check common sibling/child directories relative to project root project_root = Path(__file__).parent for candidate in [ project_root / "forge", project_root.parent / "forge-repo", - project_root.parent / "forge-repo" / "forge-gui", + project_root.parent / "forge-repo" / "forge-gui", project_root.parent / "forge", project_root / "forge-gui-desktop", ]: if candidate.is_dir(): - # Prefer directory that has 'res' subfolder if (candidate / "res").is_dir(): return str(candidate) - - # 3. Platform-specific user data directories if sys.platform == "win32": appdata = os.environ.get("APPDATA", "") if appdata: @@ -195,32 +210,25 @@ def _resolve_forge_dir() -> str: if candidate.is_dir(): return str(candidate) home = Path.home() - for candidate in [ - home / ".forge", - home / "Forge", - ]: + for candidate in [home / ".forge", home / "Forge"]: if candidate.is_dir(): return str(candidate) return "" + def _resolve_forge_jar() -> str: """Auto-detect the Forge GUI desktop JAR in common locations.""" project_root = Path(__file__).parent - - # Candidate directories where the Forge JAR might live candidates = [ project_root / "forge-gui-desktop" / "target", project_root.parent / "forge-repo" / "forge-gui-desktop" / "target", project_root.parent / "forge" / "forge-gui-desktop" / "target", ] - - # Also check FORGE_DIR/../forge-gui-desktop/target if FORGE_DIR is set forge_dir_env = os.environ.get("FORGE_DIR", "") if forge_dir_env: fd = Path(forge_dir_env) candidates.append(fd.parent / "forge-gui-desktop" / "target") candidates.append(fd / "target") - for target_dir in candidates: if target_dir.is_dir(): for pattern in [ @@ -234,6 +242,7 @@ def _resolve_forge_jar() -> str: return str(jars[0]) return "" + def _resolve_lab_jar() -> str: target_dir = Path(__file__).parent / "target" if target_dir.exists(): @@ -248,36 +257,25 @@ def _resolve_lab_jar() -> str: return str(jars[0]) return "" + def _resolve_precon_dir() -> str: - """Auto-detect the precon-decks directory. - - The precon-decks dir lives in the project root and contains downloaded - .dck files plus precon-index.json. We check relative to this file - (lab_api.py) first, then common sibling paths. - """ + """Auto-detect the precon-decks directory.""" project_root = Path(__file__).parent - - # 1. Standard location: /precon-decks candidate = project_root / "precon-decks" if candidate.is_dir(): return str(candidate) - - # 2. Check parent directory (in case running from a subdirectory) candidate = project_root.parent / "precon-decks" if candidate.is_dir(): return str(candidate) - - # 3. Derive from forge_dir if available if CFG.forge_dir: candidate = Path(CFG.forge_dir) / "precon-decks" if candidate.is_dir(): return str(candidate) - - # 4. Fall back: create the directory in the project root so downloads work fallback = project_root / "precon-decks" fallback.mkdir(parents=True, exist_ok=True) return str(fallback) + def main(): args = _parse_args() setup_logging(logging.DEBUG if args.verbose else logging.INFO) @@ -311,8 +309,15 @@ def main(): init_collection_db() init_coach_service() + # RAG Phase 1: synchronous boot-time check so logs are visible. + # The async lifespan handler also fires it in a thread on uvicorn startup, + # but calling here ensures it runs (and logs) before the first request. + from services.scryfall_bulk import ensure_bulk_db + ensure_bulk_db() + import uvicorn uvicorn.run(app, host="0.0.0.0", port=CFG.port, log_level="info") + if __name__ == "__main__": main() diff --git a/services/scryfall_bulk.py b/services/scryfall_bulk.py new file mode 100644 index 0000000..a915036 --- /dev/null +++ b/services/scryfall_bulk.py @@ -0,0 +1,372 @@ +""" +services/scryfall_bulk.py +========================= +Downloads and indexes the full Scryfall `default_cards` bulk export into a +local SQLite database with FTS5 full-text search over card names and oracle +text. This replaces per-card live Scryfall API calls for the RAG pipeline. + +Usage (one-time or nightly): + from services.scryfall_bulk import ensure_bulk_db + ensure_bulk_db() # call at startup; no-ops if data is fresh + +Public API: + ensure_bulk_db() -> None (download if stale) + lookup_card(name) -> dict | None + search_oracle(query, colors, limit) -> list[dict] + get_bulk_db() -> sqlite3.Connection + +Data path: ./data/scryfall_bulk.db +Re-download: every SCRYFALL_BULK_STALENESS_DAYS days (default 7) +""" +from __future__ import annotations + +import json +import logging +import os +import sqlite3 +import threading +import time +from pathlib import Path +from urllib.request import Request, urlopen + +logger = logging.getLogger("commander_ai_lab.scryfall_bulk") + +# ── Config ────────────────────────────────────────────────────────────────── +BULK_DB_PATH = Path(__file__).parent.parent / "data" / "scryfall_bulk.db" +BULK_META_URL = "https://api.scryfall.com/bulk-data" +SCRYFALL_BULK_STALENESS_DAYS = int( + os.environ.get("SCRYFALL_BULK_STALENESS_DAYS", "7") +) +_API_HEADERS = {"User-Agent": "CommanderAILab/3.0", "Accept": "application/json"} + +# ── Module-level singletons ────────────────────────────────────────────────── +_bulk_conn: sqlite3.Connection | None = None +_bulk_lock = threading.Lock() + + +# ── DB init ────────────────────────────────────────────────────────────────── + +def _init_db(conn: sqlite3.Connection) -> None: + """Create tables and FTS5 virtual table on first run.""" + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + conn.executescript(""" + CREATE TABLE IF NOT EXISTS cards ( + scryfall_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + oracle_text TEXT DEFAULT '', + type_line TEXT DEFAULT '', + mana_cost TEXT DEFAULT '', + cmc REAL DEFAULT 0, + color_identity TEXT DEFAULT '[]', + keywords TEXT DEFAULT '[]', + power TEXT DEFAULT '', + toughness TEXT DEFAULT '', + rarity TEXT DEFAULT '', + edhrec_rank INTEGER DEFAULT 0, + legalities TEXT DEFAULT '{}', + rulings_uri TEXT DEFAULT '', + set_code TEXT DEFAULT '', + updated_at TEXT DEFAULT (datetime('now')) + ); + + CREATE VIRTUAL TABLE IF NOT EXISTS cards_fts USING fts5( + name, + oracle_text, + content='cards', + content_rowid='rowid' + ); + + CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + """) + conn.commit() + + +# ── Staleness check ─────────────────────────────────────────────────────────── + +def _is_stale(conn: sqlite3.Connection) -> bool: + row = conn.execute( + "SELECT value FROM meta WHERE key = 'last_download'" + ).fetchone() + if not row: + return True + try: + age_days = (time.time() - float(row[0])) / 86400 + return age_days > SCRYFALL_BULK_STALENESS_DAYS + except (ValueError, TypeError): + return True + + +# ── Download + index ────────────────────────────────────────────────────────── + +def _get_bulk_download_url() -> str: + """Fetch the current default_cards download URL from the Scryfall manifest.""" + req = Request(BULK_META_URL, headers=_API_HEADERS) + with urlopen(req, timeout=15) as resp: + meta = json.loads(resp.read()) + for entry in meta.get("data", []): + if entry.get("type") == "default_cards": + return entry["download_uri"] + raise RuntimeError("Could not find default_cards entry in Scryfall bulk-data manifest") + + +def _download_and_index(conn: sqlite3.Connection) -> int: + """ + Download the full default_cards JSON from Scryfall and index into SQLite. + Replaces all existing rows. Returns number of cards indexed. + """ + logger.info("Fetching Scryfall bulk-data manifest...") + url = _get_bulk_download_url() + logger.info("Downloading Scryfall default_cards bulk export (~100 MB)...") + + req = Request(url, headers=_API_HEADERS) + with urlopen(req, timeout=300) as resp: + raw_bytes = resp.read() + + cards_json: list[dict] = json.loads(raw_bytes) + logger.info(f"Received {len(cards_json)} card objects from Scryfall.") + + # Wipe existing data for a clean rebuild + conn.execute("DELETE FROM cards") + conn.execute("DELETE FROM cards_fts") + conn.commit() + + BATCH_SIZE = 1000 + inserted = 0 + batch: list[tuple] = [] + + _SKIP_LAYOUTS = {"token", "emblem", "art_series", "double_faced_token"} + + for card in cards_json: + if card.get("layout") in _SKIP_LAYOUTS: + continue + + # Handle double-faced / meld cards: concatenate face oracle texts + oracle = card.get("oracle_text", "") + if not oracle and card.get("card_faces"): + oracle = "\n//\n".join( + f.get("oracle_text", "") + for f in card["card_faces"] + if f.get("oracle_text") + ) + + mana_cost = card.get("mana_cost", "") + if not mana_cost and card.get("card_faces"): + mana_cost = card["card_faces"][0].get("mana_cost", "") + + batch.append(( + card.get("id", ""), + card.get("name", ""), + oracle, + card.get("type_line", ""), + mana_cost, + float(card.get("cmc") or 0), + json.dumps(card.get("color_identity", [])), + json.dumps(card.get("keywords", [])), + card.get("power", "") or "", + card.get("toughness", "") or "", + card.get("rarity", ""), + int(card.get("edhrec_rank") or 0), + json.dumps(card.get("legalities", {})), + card.get("rulings_uri", "") or "", + card.get("set", ""), + )) + + if len(batch) >= BATCH_SIZE: + _flush_batch(conn, batch) + inserted += len(batch) + batch.clear() + logger.debug(f" Indexed {inserted} cards...") + + if batch: + _flush_batch(conn, batch) + inserted += len(batch) + + # Rebuild FTS5 index from cards table + conn.execute("INSERT INTO cards_fts(cards_fts) VALUES('rebuild')") + + # Record download timestamp + conn.execute( + "INSERT OR REPLACE INTO meta (key, value) VALUES ('last_download', ?)", + (str(time.time()),) + ) + conn.execute( + "INSERT OR REPLACE INTO meta (key, value) VALUES ('card_count', ?)", + (str(inserted),) + ) + conn.commit() + + logger.info(f"Scryfall bulk index complete: {inserted} cards stored.") + return inserted + + +def _flush_batch(conn: sqlite3.Connection, batch: list[tuple]) -> None: + conn.executemany( + """ + INSERT OR REPLACE INTO cards + (scryfall_id, name, oracle_text, type_line, mana_cost, cmc, + color_identity, keywords, power, toughness, rarity, + edhrec_rank, legalities, rulings_uri, set_code) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + """, + batch, + ) + conn.commit() + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def get_bulk_db() -> sqlite3.Connection: + """ + Return the module-level SQLite connection, creating it on first call. + Thread-safe via _bulk_lock. + """ + global _bulk_conn + with _bulk_lock: + if _bulk_conn is None: + BULK_DB_PATH.parent.mkdir(parents=True, exist_ok=True) + _bulk_conn = sqlite3.connect( + str(BULK_DB_PATH), check_same_thread=False + ) + _bulk_conn.row_factory = sqlite3.Row + _init_db(_bulk_conn) + return _bulk_conn + + +def ensure_bulk_db() -> None: + """ + Call at startup. Downloads and indexes the full Scryfall card set if the + local DB is missing or older than SCRYFALL_BULK_STALENESS_DAYS. + No-ops if data is already fresh. + """ + conn = get_bulk_db() + if _is_stale(conn): + logger.info( + "Scryfall bulk DB is stale or missing — downloading now " + "(~100 MB, one-time per week)..." + ) + try: + _download_and_index(conn) + except Exception as exc: + logger.error( + f"Scryfall bulk download failed: {exc}. " + "Continuing without bulk data — per-card Scryfall cache will be used." + ) + else: + row = conn.execute( + "SELECT value FROM meta WHERE key = 'card_count'" + ).fetchone() + count = row[0] if row else "unknown" + logger.info(f"Scryfall bulk DB is fresh ({count} cards cached).") + + +def lookup_card(name: str) -> dict | None: + """ + Exact case-insensitive card name lookup. + + Returns a dict with all columns from the `cards` table, or None if not + found. Prefers the most-recently-printed printing (lowest edhrec_rank). + """ + conn = get_bulk_db() + row = conn.execute( + """ + SELECT * FROM cards + WHERE LOWER(name) = LOWER(?) + ORDER BY edhrec_rank ASC + LIMIT 1 + """, + (name,), + ).fetchone() + return dict(row) if row else None + + +def search_oracle( + query: str, + color_identity: list[str] | None = None, + type_filter: str | None = None, + limit: int = 10, +) -> list[dict]: + """ + Full-text search over card names and oracle text using SQLite FTS5. + + Args: + query: Natural language or keyword query string. + color_identity: If provided, restrict results to cards whose color + identity is a subset of this list (e.g. ['W', 'U']). + type_filter: Optional type-line substring filter (e.g. 'Creature'). + limit: Maximum number of results to return. + + Returns: + List of card dicts ordered by FTS5 relevance rank. + """ + conn = get_bulk_db() + + # Build FTS5 query: wrap each token in quotes to allow phrase matching + tokens = [t.strip() for t in query.split() if t.strip()] + if not tokens: + return [] + fts_query = " OR ".join(f'"{t}"' for t in tokens) + + # Over-fetch to allow post-filtering by color/type + fetch_limit = max(limit * 5, 50) + rows = conn.execute( + """ + SELECT c.* + FROM cards c + JOIN cards_fts fts ON fts.rowid = c.rowid + WHERE cards_fts MATCH ? + ORDER BY rank + LIMIT ? + """, + (fts_query, fetch_limit), + ).fetchall() + + results: list[dict] = [] + for row in rows: + card = dict(row) + + # Color identity filter: card's CI must be a subset of deck's CI + if color_identity: + try: + card_ci = set(json.loads(card.get("color_identity") or "[]")) + except (json.JSONDecodeError, TypeError): + card_ci = set() + if not card_ci.issubset(set(color_identity)): + continue + + # Optional type-line filter + if type_filter: + if type_filter.lower() not in (card.get("type_line") or "").lower(): + continue + + results.append(card) + if len(results) >= limit: + break + + return results + + +def get_stats() -> dict: + """Return metadata about the current bulk DB state.""" + conn = get_bulk_db() + rows = conn.execute("SELECT key, value FROM meta").fetchall() + meta = {r["key"]: r["value"] for r in rows} + card_count = conn.execute("SELECT COUNT(*) FROM cards").fetchone()[0] + stale = _is_stale(conn) + last_dl = meta.get("last_download") + age_days: float | None = None + if last_dl: + try: + age_days = round((time.time() - float(last_dl)) / 86400, 1) + except (ValueError, TypeError): + pass + return { + "card_count": card_count, + "stale": stale, + "age_days": age_days, + "db_path": str(BULK_DB_PATH), + "staleness_threshold_days": SCRYFALL_BULK_STALENESS_DAYS, + }