diff --git a/coach/embeddings.py b/coach/embeddings.py index e218054..ff06e48 100644 --- a/coach/embeddings.py +++ b/coach/embeddings.py @@ -28,6 +28,7 @@ from urllib.parse import quote import numpy as np +from services.card_text import build_card_text from .config import EMBEDDINGS_DIR, EMBEDDINGS_NPZ, EMBEDDINGS_PARQUET logger = logging.getLogger("coach.embeddings") @@ -114,15 +115,8 @@ def _fetch_scryfall_card(card_name: str) -> Optional[dict]: def _build_card_text(card: dict) -> str: - """Build the text string to embed, matching minimaxir/mtg-embeddings format.""" - parts = [card["name"]] - if card.get("mana_cost"): - parts.append(card["mana_cost"]) - if card.get("type_line"): - parts.append(card["type_line"]) - if card.get("oracle_text"): - parts.append(card["oracle_text"]) - return " | ".join(parts) + """Delegate to shared build_card_text() -- see services/card_text.py.""" + return build_card_text(card) class CardMatch: diff --git a/lab_api.py b/lab_api.py index f581a0f..82a3c3d 100644 --- a/lab_api.py +++ b/lab_api.py @@ -77,6 +77,24 @@ async def _lifespan(application: FastAPI): from services.rag_store import ensure_rag_store loop.run_in_executor(None, ensure_rag_store) + # RAG Phase 3: background staleness monitor (rebuilds if >14 days old) + async def _rag_staleness_monitor(): + """Periodically check if the RAG index needs rebuilding.""" + import time as _time + while True: + await asyncio.sleep(6 * 3600) # check every 6 hours + try: + from services.rag_store import check_staleness, build_index + info = check_staleness() + if info.get("stale"): + log.info("RAG staleness monitor: %s — triggering rebuild.", info["reason"]) + loop.run_in_executor(None, lambda: build_index(force=True)) + else: + log.debug("RAG staleness monitor: index is current (%.1f days).", info.get("age_days", 0)) + except Exception as exc: + log.warning("RAG staleness monitor failed: %s", exc) + asyncio.ensure_future(_rag_staleness_monitor()) + yield # ← server is running # ──── SHUTDOWN ──── diff --git a/services/card_text.py b/services/card_text.py new file mode 100644 index 0000000..c9b918a --- /dev/null +++ b/services/card_text.py @@ -0,0 +1,37 @@ +""" +services/card_text.py +===================== +Shared utility for building card text representations. + +Both services/rag_store.py and coach/embeddings.py need to produce +identical text strings for embedding so that vector similarity is +consistent across both systems. This module provides a single +canonical implementation. + +Format: Name | Mana Cost | Type | Oracle Text +(fields that are empty/missing are omitted) +""" +from __future__ import annotations + + +def build_card_text(card: dict) -> str: + """Build the canonical text representation of a card for embedding. + + Args: + card: Dict with keys 'name', 'mana_cost', 'type_line', 'oracle_text'. + Missing keys are treated as empty strings. + + Returns: + Pipe-separated string like 'Sol Ring | {1} | Artifact | {T}: Add {C}{C}.' + """ + parts = [card.get("name", "") or ""] + mana_cost = card.get("mana_cost", "") or "" + if mana_cost: + parts.append(mana_cost) + type_line = card.get("type_line", "") or "" + if type_line: + parts.append(type_line) + oracle = card.get("oracle_text", "") or "" + if oracle: + parts.append(oracle) + return " | ".join(parts) diff --git a/services/rag_store.py b/services/rag_store.py index d839df2..2cff406 100644 --- a/services/rag_store.py +++ b/services/rag_store.py @@ -26,6 +26,7 @@ import logging import os import time import threading +from services.card_text import build_card_text from pathlib import Path from typing import Optional @@ -49,23 +50,8 @@ _chroma_collection = None # ── Helpers ──────────────────────────────────────────────────────────────── def _build_document(card: dict) -> str: - """Build the text document to embed for a single card. - - Format: Name | Mana Cost | Type | Oracle Text - Matches the structure used by coach/embeddings._build_card_text() so - vector similarity is consistent across both embedding systems. - """ - parts = [card.get("name", "")] - mana_cost = card.get("mana_cost", "") - if mana_cost: - parts.append(mana_cost) - type_line = card.get("type_line", "") - if type_line: - parts.append(type_line) - oracle = card.get("oracle_text", "") - if oracle: - parts.append(oracle) - return " | ".join(parts) + """Delegate to shared build_card_text() -- see services/card_text.py.""" + return build_card_text(card) def _build_metadata(card: dict) -> dict: @@ -516,3 +502,57 @@ def get_rag_stats() -> dict: pass return result + + +def is_chroma_available() -> bool: + """Check whether ChromaDB + Ollama are reachable without side effects.""" + try: + _get_chromadb_client() + except Exception: + return False + try: + import urllib.request + req = urllib.request.Request( + f"{OLLAMA_BASE_URL}/api/tags", + headers={"User-Agent": "commander-ai-lab/1.0"}, + ) + with urllib.request.urlopen(req, timeout=3): + pass + except Exception: + return False + return True + + +def check_staleness() -> dict: + """Return staleness info so a background scheduler can trigger rebuilds. + + Returns dict with keys: + stale (bool): True if the index should be rebuilt. + reason (str): Human-readable explanation. + age_days (float | None): Age of the current build in days. + """ + meta = _read_build_meta() + built_at = meta.get("built_at") + if not built_at: + return {"stale": True, "reason": "no build metadata found", "age_days": None} + try: + age_days = (time.time() - float(built_at)) / 86400 + except (ValueError, TypeError): + return {"stale": True, "reason": "corrupt build timestamp", "age_days": None} + if age_days > 14: + return {"stale": True, "reason": f"index is {age_days:.1f} days old (>14)", "age_days": round(age_days, 1)} + try: + from services.scryfall_bulk import get_stats as bulk_stats + bulk_count = bulk_stats().get("card_count", 0) + collection = _get_collection() + chroma_count = collection.count() + ratio = chroma_count / max(bulk_count, 1) + if ratio < 0.95 or ratio > 1.05: + return { + "stale": True, + "reason": f"card count drift: chroma={chroma_count}, bulk={bulk_count}", + "age_days": round(age_days, 1), + } + except Exception as exc: + logger.warning("check_staleness card count check failed: %s", exc) + return {"stale": False, "reason": "index is current", "age_days": round(age_days, 1)}