refactor: RAG improvements — shared card_text, Chroma fallback, staleness monitor (fixes #133, #134, #135) (#136)

* refactor: add shared build_card_text() utility

* refactor: unify _build_document via shared card_text + add is_chroma_available/check_staleness (fixes #133, #134, #135)

* refactor: delegate embeddings._build_card_text to shared utility

* feat: add background RAG staleness monitor to lab_api lifespan (fixes #135)

coach/embeddings.py

@@ -28,6 +28,7 @@ from urllib.parse import quote
 
 import numpy as np
 
+from services.card_text import build_card_text
 from .config import EMBEDDINGS_DIR, EMBEDDINGS_NPZ, EMBEDDINGS_PARQUET
 
 logger = logging.getLogger("coach.embeddings")
@@ -114,15 +115,8 @@ def _fetch_scryfall_card(card_name: str) -> Optional[dict]:
 
 
 def _build_card_text(card: dict) -> str:
-    """Build the text string to embed, matching minimaxir/mtg-embeddings format."""
-    parts = [card["name"]]
-    if card.get("mana_cost"):
-        parts.append(card["mana_cost"])
-    if card.get("type_line"):
-        parts.append(card["type_line"])
-    if card.get("oracle_text"):
-        parts.append(card["oracle_text"])
-    return " | ".join(parts)
+    """Delegate to shared build_card_text() -- see services/card_text.py."""
+    return build_card_text(card)
 
 
 class CardMatch:

lab_api.py

@@ -77,6 +77,24 @@ async def _lifespan(application: FastAPI):
     from services.rag_store import ensure_rag_store
     loop.run_in_executor(None, ensure_rag_store)
 
+        # RAG Phase 3: background staleness monitor (rebuilds if >14 days old)
+        async def _rag_staleness_monitor():
+            """Periodically check if the RAG index needs rebuilding."""
+            import time as _time
+            while True:
+                await asyncio.sleep(6 * 3600)  # check every 6 hours
+                try:
+                    from services.rag_store import check_staleness, build_index
+                    info = check_staleness()
+                    if info.get("stale"):
+                        log.info("RAG staleness monitor: %s — triggering rebuild.", info["reason"])
+                        loop.run_in_executor(None, lambda: build_index(force=True))
+                    else:
+                        log.debug("RAG staleness monitor: index is current (%.1f days).", info.get("age_days", 0))
+                except Exception as exc:
+                    log.warning("RAG staleness monitor failed: %s", exc)
+        asyncio.ensure_future(_rag_staleness_monitor())
+
     yield  # ← server is running
 
     # ──── SHUTDOWN ────

services/card_text.py

@@ -0,0 +1,37 @@
+"""
+services/card_text.py
+=====================
+Shared utility for building card text representations.
+
+Both services/rag_store.py and coach/embeddings.py need to produce
+identical text strings for embedding so that vector similarity is
+consistent across both systems.  This module provides a single
+canonical implementation.
+
+Format:  Name | Mana Cost | Type | Oracle Text
+(fields that are empty/missing are omitted)
+"""
+from __future__ import annotations
+
+
+def build_card_text(card: dict) -> str:
+    """Build the canonical text representation of a card for embedding.
+
+    Args:
+        card: Dict with keys 'name', 'mana_cost', 'type_line', 'oracle_text'.
+              Missing keys are treated as empty strings.
+
+    Returns:
+        Pipe-separated string like 'Sol Ring | {1} | Artifact | {T}: Add {C}{C}.'
+    """
+    parts = [card.get("name", "") or ""]
+    mana_cost = card.get("mana_cost", "") or ""
+    if mana_cost:
+        parts.append(mana_cost)
+    type_line = card.get("type_line", "") or ""
+    if type_line:
+        parts.append(type_line)
+    oracle = card.get("oracle_text", "") or ""
+    if oracle:
+        parts.append(oracle)
+    return " | ".join(parts)

services/rag_store.py

@@ -26,6 +26,7 @@ import logging
 import os
 import time
 import threading
+from services.card_text import build_card_text
 from pathlib import Path
 from typing import Optional
 
@@ -49,23 +50,8 @@ _chroma_collection = None
 # ── Helpers ────────────────────────────────────────────────────────────────
 
 def _build_document(card: dict) -> str:
-    """Build the text document to embed for a single card.
-
-    Format: Name | Mana Cost | Type | Oracle Text
-    Matches the structure used by coach/embeddings._build_card_text() so
-    vector similarity is consistent across both embedding systems.
-    """
-    parts = [card.get("name", "")]
-    mana_cost = card.get("mana_cost", "")
-    if mana_cost:
-        parts.append(mana_cost)
-    type_line = card.get("type_line", "")
-    if type_line:
-        parts.append(type_line)
-    oracle = card.get("oracle_text", "")
-    if oracle:
-        parts.append(oracle)
-    return " | ".join(parts)
+    """Delegate to shared build_card_text() -- see services/card_text.py."""
+    return build_card_text(card)
 
 
 def _build_metadata(card: dict) -> dict:
@@ -516,3 +502,57 @@ def get_rag_stats() -> dict:
             pass
 
     return result
+
+
+def is_chroma_available() -> bool:
+    """Check whether ChromaDB + Ollama are reachable without side effects."""
+    try:
+        _get_chromadb_client()
+    except Exception:
+        return False
+    try:
+        import urllib.request
+        req = urllib.request.Request(
+            f"{OLLAMA_BASE_URL}/api/tags",
+            headers={"User-Agent": "commander-ai-lab/1.0"},
+        )
+        with urllib.request.urlopen(req, timeout=3):
+            pass
+    except Exception:
+        return False
+    return True
+
+
+def check_staleness() -> dict:
+    """Return staleness info so a background scheduler can trigger rebuilds.
+
+    Returns dict with keys:
+        stale (bool): True if the index should be rebuilt.
+        reason (str): Human-readable explanation.
+        age_days (float | None): Age of the current build in days.
+    """
+    meta = _read_build_meta()
+    built_at = meta.get("built_at")
+    if not built_at:
+        return {"stale": True, "reason": "no build metadata found", "age_days": None}
+    try:
+        age_days = (time.time() - float(built_at)) / 86400
+    except (ValueError, TypeError):
+        return {"stale": True, "reason": "corrupt build timestamp", "age_days": None}
+    if age_days > 14:
+        return {"stale": True, "reason": f"index is {age_days:.1f} days old (>14)", "age_days": round(age_days, 1)}
+    try:
+        from services.scryfall_bulk import get_stats as bulk_stats
+        bulk_count = bulk_stats().get("card_count", 0)
+        collection = _get_collection()
+        chroma_count = collection.count()
+        ratio = chroma_count / max(bulk_count, 1)
+        if ratio < 0.95 or ratio > 1.05:
+            return {
+                "stale": True,
+                "reason": f"card count drift: chroma={chroma_count}, bulk={bulk_count}",
+                "age_days": round(age_days, 1),
+            }
+    except Exception as exc:
+        logger.warning("check_staleness card count check failed: %s", exc)
+    return {"stale": False, "reason": "index is current", "age_days": round(age_days, 1)}