fix: correct indentation of commander_keywords and oracle overlap scoring in deckgen.py

GameSession.java references these APIs which were missing from
PolicyClient, causing 3 compilation errors. Added:
- getBaseUrl() getter
- PolicyDecision record (action, confidence)
- decide(String, int) overload that delegates to decide(JsonObject)

.env.example

@@ -9,6 +9,10 @@ XIMILAR_API_KEY=your-ximilar-api-key-here
 # Perplexity API Key (AI deck research & generation)
 # Get yours at: https://www.perplexity.ai/settings/api
 PPLX_API_KEY=your-perplexity-api-key-here
+# Perplexity model for deck research & generation (sonar / sonar-pro / sonar-deep-research)
+PPLX_MODEL=sonar-pro
+# Coach chat provider: "local" (Ollama) or "perplexity" (Perplexity API)
+COACH_PROVIDER=local
 
 # CORS allowed origins (comma-separated)
 # Default: http://localhost:5173,http://localhost:8080

coach/coach_service.py

@@ -9,6 +9,8 @@ calls the LLM, parses responses, and persists coaching sessions.
 import json
 import logging
 import uuid
+import os
+import re
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Optional, List, Dict
@@ -16,6 +18,7 @@ from typing import Optional, List, Dict
 from .config import (
     DECK_REPORTS_DIR, COACH_SESSIONS_DIR,
     MAX_UNDERPERFORMERS, MAX_CANDIDATES_PER_UNDERPERFORMER,
+    COACH_PROVIDER, PPLX_MODEL,
     UNDERPERFORMER_IMPACT_THRESHOLD, ensure_dirs,
 )
 from .models import (
@@ -28,6 +31,18 @@ from .prompt_template import build_system_prompt, build_user_prompt
 
 logger = logging.getLogger("coach.service")
 
+# SQLite persistence for coach sessions
+try:
+    from services.database import (
+        save_coach_session as _db_save_session,
+        load_coach_session as _db_load_session,
+        list_coach_sessions as _db_list_sessions,
+        delete_coach_session as _db_delete_session,
+    )
+    _DB_AVAILABLE = True
+except ImportError:
+    _DB_AVAILABLE = False
+
 
 class CoachService:
     """
@@ -55,7 +70,7 @@ class CoachService:
         if not report_path.exists():
             # Try case-insensitive match
             for f in DECK_REPORTS_DIR.glob("*.json"):
-                if f.stem.lower() == deck_id.lower():
+                if re.sub(r'[^a-z0-9]', '', f.stem.lower()) == re.sub(r'[^a-z0-9]', '', deck_id.lower()):
                     report_path = f
                     break
 
@@ -80,15 +95,62 @@ class CoachService:
     # ── Coach Session I/O ──────────────────────────────────────
 
     def save_session(self, session: CoachSession) -> Path:
-        """Persist a coaching session to disk."""
+        """Persist a coaching session to disk and SQLite."""
+        # 1. Write JSON file (backward compat)
         COACH_SESSIONS_DIR.mkdir(parents=True, exist_ok=True)
         path = COACH_SESSIONS_DIR / f"{session.sessionId}.json"
         with open(path, "w") as f:
             f.write(session.model_dump_json(indent=2))
+        # 2. Write to SQLite
+        if _DB_AVAILABLE:
+            try:
+                _db_save_session({
+                    "session_id": session.sessionId,
+                    "deck_id": session.deckId,
+                    "timestamp": session.timestamp,
+                    "model_used": session.modelUsed or "",
+                    "prompt_tokens": session.promptTokens or 0,
+                    "completion_tokens": session.completionTokens or 0,
+                    "summary": session.summary or "",
+                    "goals_json": json.dumps(session.goals.model_dump() if session.goals else {}),
+                    "cuts_json": json.dumps([c.model_dump() for c in session.suggestedCuts]),
+                    "adds_json": json.dumps([a.model_dump() for a in session.suggestedAdds]),
+                    "heuristic_hints_json": json.dumps(session.heuristicHints or []),
+                    "mana_base_advice": session.manaBaseAdvice or "",
+                    "raw_text": session.rawTextExplanation or "",
+                })
+                logger.info("Coach session persisted to SQLite: %s", session.sessionId)
+            except Exception as e:
+                logger.warning("Failed to persist session to SQLite: %s", e)
         return path
 
+
     def load_session(self, session_id: str) -> Optional[CoachSession]:
-        """Load a coaching session from disk."""
+        """Load a coaching session from SQLite, falling back to disk."""
+        # Try SQLite first
+        if _DB_AVAILABLE:
+            try:
+                row = _db_load_session(session_id)
+                if row:
+                    data = dict(row)
+                    return CoachSession(
+                        sessionId=data["session_id"],
+                        deckId=data["deck_id"],
+                        timestamp=data["timestamp"],
+                        modelUsed=data.get("model_used", ""),
+                        promptTokens=data.get("prompt_tokens", 0),
+                        completionTokens=data.get("completion_tokens", 0),
+                        summary=data.get("summary", ""),
+                        goals=json.loads(data["goals_json"]) if data.get("goals_json") and data["goals_json"] != "{}" else None,
+                        suggestedCuts=[SuggestedCut(**c) for c in json.loads(data.get("cuts_json", "[]"))],
+                        suggestedAdds=[SuggestedAdd(**a) for a in json.loads(data.get("adds_json", "[]"))],
+                        heuristicHints=json.loads(data.get("heuristic_hints_json", "[]")),
+                        manaBaseAdvice=data.get("mana_base_advice", ""),
+                        rawTextExplanation=data.get("raw_text", ""),
+                    )
+            except Exception as e:
+                logger.warning("SQLite load failed for %s: %s", session_id, e)
+        # Fallback to disk
         path = COACH_SESSIONS_DIR / f"{session_id}.json"
         if not path.exists():
             return None
@@ -100,8 +162,18 @@ class CoachService:
             logger.error("Failed to load session %s: %s", session_id, e)
             return None
 
+
     def list_sessions(self, deck_id: str = None) -> List[dict]:
-        """List all coaching sessions, optionally filtered by deck."""
+        """List coaching sessions from SQLite, falling back to disk."""
+        # Try SQLite first
+        if _DB_AVAILABLE:
+            try:
+                rows = _db_list_sessions(deck_id)
+                if rows is not None:
+                    return rows
+            except Exception as e:
+                logger.warning("SQLite list_sessions failed: %s", e)
+        # Fallback to disk
         if not COACH_SESSIONS_DIR.exists():
             return []
         sessions = []
@@ -123,6 +195,7 @@ class CoachService:
                 continue
         return sessions
 
+
     # ── Main Coaching Pipeline ─────────────────────────────────
 
     async def run_coaching_session(self, deck_id: str,
@@ -265,13 +338,25 @@ class CoachService:
         """Check the health of all coach subsystems."""
         status = CoachStatus()
 
-        # Check LLM
-        llm_status = self.llm.check_connection()
-        status.llmConnected = llm_status.get("connected", False)
-        status.llmModel = llm_status.get("active_model")
-        status.llmModels = llm_status.get("models", [])
-        if not status.llmConnected:
-            status.error = llm_status.get("error", "LM Studio not reachable")
+        # Check LLM / Perplexity provider
+        if COACH_PROVIDER == "perplexity":
+            pplx_key = os.environ.get("PPLX_API_KEY", "")
+            if pplx_key:
+                status.llmConnected = True
+                status.llmModel = f"Perplexity ({PPLX_MODEL})"
+                status.llmModels = [PPLX_MODEL]
+            else:
+                status.llmConnected = False
+                status.llmModel = None
+                status.llmModels = []
+                status.error = "Perplexity API key not configured. Set PPLX_API_KEY env var."
+        else:
+            llm_status = self.llm.check_connection()
+            status.llmConnected = llm_status.get("connected", False)
+            status.llmModel = llm_status.get("active_model")
+            status.llmModels = llm_status.get("models", [])
+            if not status.llmConnected:
+                status.error = llm_status.get("error", "Local LLM not reachable")
 
         # Check embeddings
         status.embeddingsLoaded = self.embeddings.loaded

coach/config.py

@@ -13,7 +13,7 @@ from pathlib import Path
 LAB_ROOT = Path(os.environ.get("COMMANDER_LAB_ROOT", Path(__file__).parent.parent))
 
 # ── Ollama (OpenAI-compatible API) ──────────────────────
-LLM_URL = os.environ.get("LLM_URL", "http://192.168.0.240:11434/v1")
+LLM_URL = os.environ.get("LLM_URL", "http://localhost:11434/v1")
 LLM_MODEL = os.environ.get("LLM_MODEL", "gpt-oss:20b")
 LLM_TIMEOUT = int(os.environ.get("LLM_TIMEOUT", "360"))  # seconds
 LLM_MAX_RETRIES = 3
@@ -44,7 +44,11 @@ DEFAULT_MAX_TOKENS = 8192
 DECK_GEN_PROVIDER = os.environ.get("DECK_GEN_PROVIDER", "local")
 # Model for deck generation (sonar = fast/$0.004, sonar-pro = deep/$0.04)
 DECK_GEN_MODEL = os.environ.get("DECK_GEN_MODEL", "gpt-oss:20b")
-DECK_GEN_BASE_URL = os.environ.get("DECK_GEN_BASE_URL", "http://192.168.0.240:11434/v1")
+# Perplexity cloud model (sonar / sonar-pro / sonar-deep-research)
+PPLX_MODEL = os.environ.get("PPLX_MODEL", "sonar-pro")
+# Coach chat provider: "local" (Ollama) or "perplexity" (Perplexity API)
+COACH_PROVIDER = os.environ.get("COACH_PROVIDER", "local")
+DECK_GEN_BASE_URL = os.environ.get("DECK_GEN_BASE_URL", "http://localhost:11434/v1")
 DECK_GEN_TEMPERATURE = float(os.environ.get("DECK_GEN_TEMPERATURE", "0.2"))
 DECK_GEN_MAX_TOKENS = int(os.environ.get("DECK_GEN_MAX_TOKENS", "16384"))
 

coach/embeddings.py

@@ -9,6 +9,11 @@ Data source: https://huggingface.co/datasets/minimaxir/mtg-embeddings
 Format: Parquet with 768-dim float32 embeddings per card (~32K cards)
 Model: Alibaba-NLP/gte-modernbert-base
 
+Unknown cards (e.g. newly printed sets) are fetched from Scryfall,
+embedded on-the-fly using the same model, and persisted to a local
+custom cache (embeddings/mtg_embeddings_custom.npz) so they survive
+restarts without re-hitting Scryfall.
+
 Uses NumPy brute-force cosine similarity — fast enough for ~32K cards (<100ms).
 """
 
@@ -16,9 +21,11 @@ import json
 import logging
 import re
 import os
+import time
 from pathlib import Path
 from typing import List, Optional, Tuple
 from urllib.request import urlopen, Request
+from urllib.parse import quote
 
 import numpy as np
 
@@ -32,8 +39,16 @@ HF_PARQUET_URL = (
     "resolve/main/mtg_embeddings.parquet"
 )
 
+# ── Scryfall API ───────────────────────────────────────────
+SCRYFALL_NAMED_URL = "https://api.scryfall.com/cards/named?fuzzy={}"
+
+# ── Custom cache for on-the-fly embedded cards ─────────────
+CUSTOM_CACHE_PATH = EMBEDDINGS_DIR / "mtg_embeddings_custom.npz"
+
+# ── Embedding model (same as minimaxir/mtg-embeddings) ─────
+EMBEDDING_MODEL = "Alibaba-NLP/gte-modernbert-base"
+
 # ── Color identity parsing ─────────────────────────────────
-# Maps mana symbols like {W}, {U}, {B}, {R}, {G} to color letters
 MANA_SYMBOL_COLORS = {
     "W": "W", "U": "U", "B": "B", "R": "R", "G": "G"
 }
@@ -46,7 +61,6 @@ def _parse_color_identity(mana_cost: str) -> str:
     colors = set()
     for match in re.finditer(r'\{([WUBRG])\}', mana_cost or ""):
         colors.add(match.group(1))
-    # Sort in WUBRG order
     order = "WUBRG"
     return "".join(c for c in order if c in colors)
 
@@ -61,12 +75,11 @@ def _parse_mana_value(mana_cost: str) -> float:
         if symbol in MANA_SYMBOL_COLORS:
             total += 1.0
         elif symbol == "X":
-            pass  # X = 0 for CMC purposes
+            pass
         else:
             try:
                 total += float(symbol)
             except ValueError:
-                # Hybrid/phyrexian symbols count as 1
                 total += 1.0
     return total
 
@@ -75,7 +88,42 @@ def _is_color_subset(card_colors: str, deck_colors: List[str]) -> bool:
     """Check if card's color identity is within deck's color identity."""
     deck_set = set(deck_colors)
     card_set = set(card_colors)
-    return card_set.issubset(deck_set) or len(card_set) == 0  # colorless always allowed
+    return card_set.issubset(deck_set) or len(card_set) == 0
+
+
+def _fetch_scryfall_card(card_name: str) -> Optional[dict]:
+    """Fetch card data from Scryfall by fuzzy name match.
+
+    Returns a dict with keys: name, mana_cost, type_line, oracle_text,
+    color_identity. Returns None on any error.
+    """
+    url = SCRYFALL_NAMED_URL.format(quote(card_name))
+    try:
+        req = Request(url, headers={"User-Agent": "commander-ai-lab/1.0"})
+        with urlopen(req, timeout=10) as resp:
+            data = json.loads(resp.read().decode("utf-8"))
+        return {
+            "name": data.get("name", card_name),
+            "mana_cost": data.get("mana_cost", ""),
+            "type_line": data.get("type_line", ""),
+            "oracle_text": data.get("oracle_text", ""),
+            "color_identity": "".join(data.get("color_identity", [])),
+        }
+    except Exception as e:
+        logger.warning("Scryfall lookup failed for '%s': %s", card_name, e)
+        return None
+
+
+def _build_card_text(card: dict) -> str:
+    """Build the text string to embed, matching minimaxir/mtg-embeddings format."""
+    parts = [card["name"]]
+    if card.get("mana_cost"):
+        parts.append(card["mana_cost"])
+    if card.get("type_line"):
+        parts.append(card["type_line"])
+    if card.get("oracle_text"):
+        parts.append(card["oracle_text"])
+    return " | ".join(parts)
 
 
 class CardMatch:
@@ -114,15 +162,17 @@ class MTGEmbeddingIndex:
     """
 
     def __init__(self):
-        self.names: np.ndarray = None       # (N,) string array
-        self.colors: np.ndarray = None      # (N,) string array (e.g., "WU", "BRG")
-        self.mana_values: np.ndarray = None # (N,) float array
-        self.types: np.ndarray = None       # (N,) string array
-        self.mana_costs: np.ndarray = None  # (N,) string array
-        self.texts: np.ndarray = None       # (N,) string array
-        self.vectors: np.ndarray = None     # (N, 768) float32, L2-normalized
+        self.names: np.ndarray = None
+        self.colors: np.ndarray = None
+        self.mana_values: np.ndarray = None
+        self.types: np.ndarray = None
+        self.mana_costs: np.ndarray = None
+        self.texts: np.ndarray = None
+        self.vectors: np.ndarray = None
         self._name_to_idx: dict = {}
         self._loaded = False
+        self._sentence_model = None   # lazy-loaded only when needed
+        self._custom_dirty = False    # tracks whether custom cache needs saving
 
     @property
     def loaded(self) -> bool:
@@ -152,10 +202,7 @@ class MTGEmbeddingIndex:
         return EMBEDDINGS_PARQUET
 
     def convert_parquet_to_npz(self) -> Path:
-        """
-        Convert the Parquet file to a compact NPZ for fast loading.
-        Requires pandas and pyarrow.
-        """
+        """Convert the Parquet file to a compact NPZ for fast loading."""
         try:
             import pandas as pd
         except ImportError:
@@ -169,14 +216,11 @@ class MTGEmbeddingIndex:
         types = df["type"].fillna("").values.astype(str)
         texts = df["text"].fillna("").values.astype(str)
 
-        # Parse color identities from mana costs
         colors = np.array([_parse_color_identity(mc) for mc in mana_costs])
         mana_values = np.array([_parse_mana_value(mc) for mc in mana_costs], dtype=np.float32)
 
-        # Extract embedding vectors
         embeddings = np.vstack(df["embedding"].to_numpy()).astype(np.float32)
 
-        # L2 normalize for cosine similarity via dot product
         norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
         norms[norms == 0] = 1.0
         embeddings = embeddings / norms
@@ -198,25 +242,27 @@ class MTGEmbeddingIndex:
         return EMBEDDINGS_NPZ
 
     def load(self, force_download: bool = False) -> bool:
-        """
-        Load embeddings from NPZ (fast) or fall back to downloading
+        """Load embeddings from NPZ (fast) or fall back to downloading
         and converting from Parquet.
         """
         if self._loaded and not force_download:
             return True
 
-        # Try NPZ first (fast path)
         if EMBEDDINGS_NPZ.exists() and not force_download:
-            return self._load_npz()
+            ok = self._load_npz()
+        else:
+            try:
+                self.download_parquet(force=force_download)
+                self.convert_parquet_to_npz()
+                ok = self._load_npz()
+            except Exception as e:
+                logger.error("Failed to load embeddings: %s", e)
+                return False
 
-        # Download and convert
-        try:
-            self.download_parquet(force=force_download)
-            self.convert_parquet_to_npz()
-            return self._load_npz()
-        except Exception as e:
-            logger.error("Failed to load embeddings: %s", e)
-            return False
+        if ok:
+            self._load_custom_cache()
+
+        return ok
 
     def _load_npz(self) -> bool:
         """Load from pre-built NPZ file."""
@@ -230,7 +276,6 @@ class MTGEmbeddingIndex:
             self.texts = data.get("texts", np.array([""] * len(self.names)))
             self.vectors = data["vectors"]
 
-            # Build name lookup
             self._name_to_idx = {
                 name.lower(): i for i, name in enumerate(self.names)
             }
@@ -241,12 +286,169 @@ class MTGEmbeddingIndex:
             logger.error("Failed to load NPZ: %s", e)
             return False
 
+    # ── Custom cache ──────────────────────────────────────────
+
+    def _load_custom_cache(self) -> None:
+        """Merge on-the-fly embedded cards from the custom cache into the index."""
+        if not CUSTOM_CACHE_PATH.exists():
+            return
+        try:
+            data = np.load(str(CUSTOM_CACHE_PATH), allow_pickle=True)
+            c_names = data["names"]
+            # Only add cards that aren't already in the main index
+            new_mask = np.array(
+                [n.lower() not in self._name_to_idx for n in c_names]
+            )
+            if not new_mask.any():
+                return
+            self._append_cards(
+                c_names[new_mask],
+                data["colors"][new_mask],
+                data["mana_values"][new_mask],
+                data["types"][new_mask],
+                data["mana_costs"][new_mask],
+                data["texts"][new_mask],
+                data["vectors"][new_mask],
+            )
+            logger.info("Loaded %d custom-cached card embeddings", new_mask.sum())
+        except Exception as e:
+            logger.warning("Failed to load custom embedding cache: %s", e)
+
+    def _save_custom_cache(self) -> None:
+        """Persist the custom cache NPZ (only cards not in the base dataset)."""
+        if not self._custom_dirty:
+            return
+        try:
+            EMBEDDINGS_DIR.mkdir(parents=True, exist_ok=True)
+            # Collect indices that correspond to custom-added cards
+            # We track these by loading the base count at load time — simpler:
+            # just re-save the full custom cache from CUSTOM_CACHE_PATH if it
+            # exists, merging with any new cards we've added this session.
+            existing_names: set = set()
+            existing: dict = {}
+            if CUSTOM_CACHE_PATH.exists():
+                try:
+                    d = np.load(str(CUSTOM_CACHE_PATH), allow_pickle=True)
+                    for n in d["names"]:
+                        existing_names.add(n.lower())
+                    existing = dict(d)
+                except Exception:
+                    pass
+
+            # Find names in our live index that aren't in the base NPZ
+            base_data = np.load(str(EMBEDDINGS_NPZ), allow_pickle=True)
+            base_names = set(n.lower() for n in base_data["names"])
+            custom_mask = np.array(
+                [n.lower() not in base_names for n in self.names]
+            )
+            if not custom_mask.any():
+                return
+
+            np.savez_compressed(
+                str(CUSTOM_CACHE_PATH),
+                names=self.names[custom_mask],
+                colors=self.colors[custom_mask],
+                mana_values=self.mana_values[custom_mask],
+                types=self.types[custom_mask],
+                mana_costs=self.mana_costs[custom_mask],
+                texts=self.texts[custom_mask],
+                vectors=self.vectors[custom_mask],
+            )
+            self._custom_dirty = False
+            logger.info("Saved custom embedding cache: %d cards", custom_mask.sum())
+        except Exception as e:
+            logger.warning("Failed to save custom embedding cache: %s", e)
+
+    def _append_cards(
+        self,
+        names: np.ndarray,
+        colors: np.ndarray,
+        mana_values: np.ndarray,
+        types: np.ndarray,
+        mana_costs: np.ndarray,
+        texts: np.ndarray,
+        vectors: np.ndarray,
+    ) -> None:
+        """Append new card arrays to the live in-memory index."""
+        start_idx = len(self.names)
+        self.names = np.concatenate([self.names, names])
+        self.colors = np.concatenate([self.colors, colors])
+        self.mana_values = np.concatenate([self.mana_values, mana_values])
+        self.types = np.concatenate([self.types, types])
+        self.mana_costs = np.concatenate([self.mana_costs, mana_costs])
+        self.texts = np.concatenate([self.texts, texts])
+        self.vectors = np.concatenate([self.vectors, vectors], axis=0)
+        for i, name in enumerate(names):
+            self._name_to_idx[name.lower()] = start_idx + i
+
+    # ── Sentence model (lazy) ─────────────────────────────────
+
+    def _get_sentence_model(self):
+        """Lazy-load the sentence-transformers model on first cache miss."""
+        if self._sentence_model is None:
+            try:
+                from sentence_transformers import SentenceTransformer
+                logger.info("Loading embedding model %s (first cache miss)...", EMBEDDING_MODEL)
+                self._sentence_model = SentenceTransformer(EMBEDDING_MODEL)
+                logger.info("Embedding model loaded.")
+            except Exception as e:
+                logger.error("Failed to load SentenceTransformer: %s", e)
+                return None
+        return self._sentence_model
+
+    # ── On-the-fly Scryfall fallback ──────────────────────────
+
+    def _fetch_and_embed_card(self, card_name: str) -> Optional[np.ndarray]:
+        """Fetch card from Scryfall, embed it, append to index, persist to cache.
+
+        Returns the L2-normalized embedding vector on success, None on failure.
+        """
+        logger.info("Fetching unknown card from Scryfall: '%s'", card_name)
+        card_data = _fetch_scryfall_card(card_name)
+        if card_data is None:
+            return None
+
+        model = self._get_sentence_model()
+        if model is None:
+            return None
+
+        text = _build_card_text(card_data)
+        vec = model.encode(text, normalize_embeddings=True).astype(np.float32)
+        # Ensure shape (1, 768)
+        vec = vec.reshape(1, -1)
+
+        resolved_name = card_data["name"]
+        mana_cost = card_data.get("mana_cost", "")
+        color_identity = card_data.get("color_identity", "") or _parse_color_identity(mana_cost)
+
+        self._append_cards(
+            names=np.array([resolved_name]),
+            colors=np.array([color_identity]),
+            mana_values=np.array([_parse_mana_value(mana_cost)], dtype=np.float32),
+            types=np.array([card_data.get("type_line", "")]),
+            mana_costs=np.array([mana_cost]),
+            texts=np.array([card_data.get("oracle_text", "")]),
+            vectors=vec,
+        )
+
+        self._custom_dirty = True
+        self._save_custom_cache()
+        logger.info("Embedded and cached new card: '%s'", resolved_name)
+        return self.vectors[self._name_to_idx[resolved_name.lower()]]
+
+    # ── Public API ────────────────────────────────────────────
+
     def get_card_vector(self, card_name: str) -> Optional[np.ndarray]:
-        """Get the embedding vector for a card by name (case-insensitive)."""
+        """Get the embedding vector for a card by name (case-insensitive).
+
+        On a cache miss, fetches from Scryfall and embeds on the fly.
+        """
         idx = self._name_to_idx.get(card_name.lower())
         if idx is not None:
             return self.vectors[idx]
-        return None
+
+        # Cache miss — try Scryfall fallback
+        return self._fetch_and_embed_card(card_name)
 
     def search_similar(self, query_card: str,
                        color_filter: List[str] = None,
@@ -274,38 +476,32 @@ class MTGEmbeddingIndex:
 
         query_vec = self.get_card_vector(query_card)
         if query_vec is None:
-            logger.warning("Card not found in embeddings: %s", query_card)
+            logger.warning("Card not found in embeddings and Scryfall fallback failed: %s", query_card)
             return []
 
-        # Compute similarities (dot product on normalized vectors = cosine similarity)
         similarities = self.vectors @ query_vec
 
-        # Build exclusion set
         exclude_set = set()
         exclude_set.add(query_card.lower())
         if exclude_cards:
             exclude_set.update(name.lower() for name in exclude_cards)
 
-        # Filter and collect results
         results = []
         for idx in np.argsort(similarities)[::-1]:
             name = str(self.names[idx])
             if name.lower() in exclude_set:
                 continue
 
-            # Color filter
             if color_filter:
                 card_colors = str(self.colors[idx])
                 if not _is_color_subset(card_colors, color_filter):
                     continue
 
-            # Type filter
             if type_filter:
                 card_type = str(self.types[idx])
                 if type_filter.lower() not in card_type.lower():
                     continue
 
-            # Mana value range filter
             if mana_range:
                 mv = float(self.mana_values[idx])
                 if mv < mana_range[0] or mv > mana_range[1]:

coach/report_generator.py

@@ -38,7 +38,7 @@ def generate_deck_reports(results_dir: str, reports_dir: str = None) -> List[str
     output_path.mkdir(parents=True, exist_ok=True)
 
     # Load all batch results
-    batch_files = sorted(results_path.glob("batch-*.json"))
+    batch_files = sorted(results_path.glob("ml-decision-*.json"))
     if not batch_files:
         logger.info("No batch result files found in %s", results_dir)
         return []
@@ -108,7 +108,7 @@ def generate_single_deck_report(batch_json_path: str, reports_dir: str = None) -
     # Also load existing batch results for comprehensive reports
     results_dir = Path(batch_json_path).parent
     all_batches = []
-    for bf in sorted(results_dir.glob("batch-*.json")):
+    for bf in sorted(results_dir.glob("ml-decision-*.json")):
         try:
             with open(bf, "r", encoding="utf-8") as f:
                 all_batches.append(json.load(f))

lab_api.py

@@ -22,15 +22,10 @@ except ImportError:
     print("ERROR: FastAPI not installed. Run: pip install fastapi uvicorn")
     import sys; sys.exit(1)
 
-from routes.shared import (
-    CFG,
-    setup_logging,
-    init_collection_db,
-    download_precon_database,
-    load_commander_meta,
-    PRECON_INDEX,
-    COMMANDER_META,
-)
+from models.state import CFG, COMMANDER_META, load_commander_meta
+from services.database import init_collection_db
+from services.logging import setup_logging
+from services.precon_service import PRECON_INDEX, download_precon_database
 from routes.collection import router as collection_router
 from routes.deckbuilder import router as deckbuilder_router
 from routes.precon import router as precon_router
@@ -45,7 +40,7 @@ from routes.ml import router as ml_router
 log = logging.getLogger("commander_ai_lab.api")
 
 # ── CORS origins (env-configurable) ────────────────────────────
-_DEFAULT_ORIGINS = "http://localhost:5173,http://localhost:8080,http://192.168.0.240:5173,http://192.168.0.240:8080"
+_DEFAULT_ORIGINS = "http://localhost:5173,http://localhost:8080"
 _allowed = os.environ.get("ALLOWED_ORIGINS", _DEFAULT_ORIGINS)
 allowed_origins = [o.strip() for o in _allowed.split(",") if o.strip()]
 
@@ -80,15 +75,14 @@ async def health_check():
 @app.on_event("startup")
 async def _on_startup():
     """Ensure DB and precon index are ready (supports uvicorn --reload)."""
-    import routes.shared as _shared
     # Ensure precon_dir is resolved even when started via uvicorn directly
     if not CFG.precon_dir:
         CFG.precon_dir = _resolve_precon_dir()
-    _shared.init_collection_db()
-    if not _shared.PRECON_INDEX:
-        _shared.download_precon_database()
-    if not _shared.COMMANDER_META:
-        _shared.load_commander_meta()
+    init_collection_db()
+    if not PRECON_INDEX:
+        download_precon_database()
+    if not COMMANDER_META:
+        load_commander_meta()
 
 # ── Static UI ──────────────────────────────────────────────────
 _legacy_ui_dir = Path(__file__).parent / "ui"
@@ -288,7 +282,7 @@ def main():
     CFG.ximilar_api_key = args.ximilar_key
     CFG.pplx_api_key = args.pplx_key
 
-    from routes.shared import get_java17
+    from services.forge_runner import get_java17
     j17 = get_java17()
 
     log.info("Commander AI Lab — API Server v3.0.0")

ml/training/__init__.py

@@ -0,0 +1,40 @@
+"""
+Commander AI Lab - Training Subpackage
+==========================================
+Provides RL training infrastructure including PPO, rollout buffers,
+Forge episode generation, self-play, and distillation.
+"""
+
+from ml.training.rollout_buffer import RolloutBuffer
+from ml.training.reward import compute_intermediate_reward, compute_terminal_reward, RewardConfig
+from ml.training.decision_exporter import DecisionExporter
+from ml.training.forge_episode_generator import (
+    ForgeTransition,
+    ForgeEpisode,
+    ForgeEpisodeQueue,
+    forge_producer,
+    ppo_consumer,
+    run_forge_pipeline,
+    parse_forge_jsonl,
+    snapshots_to_episode,
+)
+
+__all__ = [
+    # Rollout & Buffers
+    "RolloutBuffer",
+    # Rewards
+    "compute_intermediate_reward",
+    "compute_terminal_reward",
+    "RewardConfig",
+    # Decision export
+    "DecisionExporter",
+    # Forge episode generation (Phase 3)
+    "ForgeTransition",
+    "ForgeEpisode",
+    "ForgeEpisodeQueue",
+    "forge_producer",
+    "ppo_consumer",
+    "run_forge_pipeline",
+    "parse_forge_jsonl",
+    "snapshots_to_episode",
+]

ml/training/forge_episode_generator.py

@@ -0,0 +1,352 @@
+"""
+Commander AI Lab — Forge Episode Generator (Phase 3.2)
+
+Async producer-consumer pipeline: runs Forge batch games in parallel
+with PPO training.  Forge produces episodes → shared queue → PPO
+trainer consumes.
+
+Uses the existing watchdog (_global_watchdog_loop in forge_runner.py)
+to monitor Forge worker processes for stalls.
+"""
+
+import asyncio
+import json
+import logging
+import os
+import time
+from collections import deque
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Deque, Dict, List, Optional, Tuple
+
+import numpy as np
+
+from ml.config.scope import (
+    ACTION_TO_IDX,
+    GAME_SCOPE,
+    IDX_TO_ACTION,
+    MacroAction,
+    NUM_ACTIONS,
+    STATE_DIMS,
+)
+from ml.training.reward import (
+    RewardConfig,
+    compute_intermediate_reward,
+    compute_terminal_reward,
+)
+
+logger = logging.getLogger("ml.forge_episode_generator")
+
+
+# ══════════════════════════════════════════════════════════
+# Episode / Trajectory data structures
+# ══════════════════════════════════════════════════════════
+
+@dataclass
+class ForgeTransition:
+    """Single (s, a, r, log_prob, value, done) tuple from a Forge game."""
+    state: np.ndarray
+    action: int
+    reward: float
+    log_prob: float
+    value: float
+    done: bool
+
+
+@dataclass
+class ForgeEpisode:
+    """Complete trajectory from a single Forge game."""
+    game_id: str
+    transitions: List[ForgeTransition] = field(default_factory=list)
+    winner: Optional[int] = None
+    total_turns: int = 0
+    deck_names: List[str] = field(default_factory=list)
+
+    @property
+    def length(self) -> int:
+        return len(self.transitions)
+
+    @property
+    def total_reward(self) -> float:
+        return sum(t.reward for t in self.transitions)
+
+
+# ══════════════════════════════════════════════════════════
+# JSONL Parser — reads Forge ML decision snapshots
+# ══════════════════════════════════════════════════════════
+
+def parse_forge_jsonl(path: Path) -> List[Dict[str, Any]]:
+    """Parse a JSONL file of Forge ML decision snapshots.
+
+    Each line is a JSON object with keys like:
+        game_id, turn, phase, active_seat, state, action, outcome
+    """
+    snapshots = []
+    with open(path, "r") as f:
+        for lineno, line in enumerate(f, 1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                snapshots.append(json.loads(line))
+            except json.JSONDecodeError as e:
+                logger.warning("Skipping malformed line %d in %s: %s", lineno, path, e)
+    logger.info("Parsed %d snapshots from %s", len(snapshots), path)
+    return snapshots
+
+
+def snapshots_to_episode(
+    snapshots: List[Dict[str, Any]],
+    reward_config: Optional[RewardConfig] = None,
+) -> ForgeEpisode:
+    """Convert a sequence of Forge snapshots into a ForgeEpisode.
+
+    Maps Forge action labels → MacroAction indices and computes
+    rewards from game outcomes using ml.training.reward.
+    """
+    if not snapshots:
+        return ForgeEpisode(game_id="empty")
+
+    first = snapshots[0]
+    episode = ForgeEpisode(
+        game_id=first.get("game_id", "unknown"),
+        deck_names=first.get("deck_names", []),
+    )
+
+    rc = reward_config or RewardConfig()
+
+    for i, snap in enumerate(snapshots):
+        # State vector — zero-padded if not present
+        state_vec = np.zeros(STATE_DIMS.total_state_dim, dtype=np.float32)
+        if "state_vector" in snap:
+            raw = np.array(snap["state_vector"], dtype=np.float32)
+            state_vec[: min(len(raw), len(state_vec))] = raw[: len(state_vec)]
+
+        # Action mapping
+        action_label = snap.get("action", "pass").lower().replace(" ", "_")
+        try:
+            macro = MacroAction(action_label)
+            action_idx = ACTION_TO_IDX[macro]
+        except (ValueError, KeyError):
+            action_idx = ACTION_TO_IDX[MacroAction.PASS]
+
+        # Reward
+        is_done = i == len(snapshots) - 1
+        if is_done and "outcome" in snap:
+            reward = compute_terminal_reward(
+                won=snap["outcome"].get("won", False),
+                turns=snap.get("turn", 1),
+                life_remaining=snap["outcome"].get("life_remaining", 0),
+                config=rc,
+            )
+        else:
+            reward = compute_intermediate_reward(
+                life_delta=snap.get("life_delta", 0),
+                cards_drawn=snap.get("cards_drawn", 0),
+                creatures_played=snap.get("creatures_played", 0),
+                config=rc,
+            )
+
+        episode.transitions.append(ForgeTransition(
+            state=state_vec,
+            action=action_idx,
+            reward=reward,
+            log_prob=snap.get("log_prob", 0.0),
+            value=snap.get("value", 0.0),
+            done=is_done,
+        ))
+
+    episode.total_turns = snapshots[-1].get("turn", len(snapshots))
+    if "outcome" in snapshots[-1]:
+        episode.winner = snapshots[-1]["outcome"].get("winner_seat")
+
+    return episode
+
+
+# ══════════════════════════════════════════════════════════
+# Async Episode Queue (producer-consumer)
+# ══════════════════════════════════════════════════════════
+
+class ForgeEpisodeQueue:
+    """Thread-safe async queue bridging Forge producers and PPO consumer.
+
+    Forge batch workers produce ForgeEpisodes; the PPO trainer consumes
+    them as they become available.  Backpressure is applied when the
+    queue reaches max_size.
+    """
+
+    def __init__(self, max_size: int = 256):
+        self._queue: asyncio.Queue[ForgeEpisode] = asyncio.Queue(maxsize=max_size)
+        self._produced = 0
+        self._consumed = 0
+        self._closed = False
+
+    async def put(self, episode: ForgeEpisode) -> None:
+        await self._queue.put(episode)
+        self._produced += 1
+
+    async def get(self, timeout: float = 30.0) -> Optional[ForgeEpisode]:
+        try:
+            ep = await asyncio.wait_for(self._queue.get(), timeout=timeout)
+            self._consumed += 1
+            return ep
+        except asyncio.TimeoutError:
+            return None
+
+    def close(self) -> None:
+        self._closed = True
+
+    @property
+    def is_closed(self) -> bool:
+        return self._closed and self._queue.empty()
+
+    @property
+    def stats(self) -> Dict[str, int]:
+        return {
+            "produced": self._produced,
+            "consumed": self._consumed,
+            "pending": self._queue.qsize(),
+        }
+
+
+# ══════════════════════════════════════════════════════════
+# Forge Batch Producer
+# ══════════════════════════════════════════════════════════
+
+async def forge_producer(
+    queue: ForgeEpisodeQueue,
+    results_dir: str = "results",
+    batch_id: str = "forge-batch",
+    num_games: int = 64,
+    poll_interval: float = 2.0,
+    reward_config: Optional[RewardConfig] = None,
+) -> int:
+    """Producer coroutine: watches for Forge JSONL output files,
+    parses them into episodes, and pushes them onto the queue.
+
+    Returns total number of episodes produced.
+    """
+    results_path = Path(results_dir)
+    pattern = f"ml-decisions-forge-{batch_id}*.jsonl"
+    seen_files: set = set()
+    total_produced = 0
+
+    logger.info("[PRODUCER] Watching %s for pattern %s", results_path, pattern)
+
+    while total_produced < num_games and not queue.is_closed:
+        for jsonl_file in sorted(results_path.glob(pattern)):
+            if jsonl_file in seen_files:
+                continue
+            seen_files.add(jsonl_file)
+
+            try:
+                snapshots = parse_forge_jsonl(jsonl_file)
+                if not snapshots:
+                    continue
+
+                # Group by game_id
+                games: Dict[str, List] = {}
+                for snap in snapshots:
+                    gid = snap.get("game_id", "unknown")
+                    games.setdefault(gid, []).append(snap)
+
+                for gid, game_snaps in games.items():
+                    episode = snapshots_to_episode(game_snaps, reward_config)
+                    if episode.length > 0:
+                        await queue.put(episode)
+                        total_produced += 1
+                        logger.debug("[PRODUCER] Queued episode %s (%d transitions)",
+                                     gid, episode.length)
+
+            except Exception as e:
+                logger.error("[PRODUCER] Error processing %s: %s", jsonl_file, e)
+
+        await asyncio.sleep(poll_interval)
+
+    logger.info("[PRODUCER] Done — produced %d episodes", total_produced)
+    return total_produced
+
+
+async def ppo_consumer(
+    queue: ForgeEpisodeQueue,
+    buffer_size: int = 64,
+    on_batch_ready=None,
+) -> int:
+    """Consumer coroutine: pulls episodes from the queue and batches
+    them for PPO training updates.
+
+    Args:
+        queue: ForgeEpisodeQueue to consume from
+        buffer_size: number of episodes per training batch
+        on_batch_ready: callback(episodes) when a batch is full
+
+    Returns total episodes consumed.
+    """
+    batch: List[ForgeEpisode] = []
+    total_consumed = 0
+
+    while not queue.is_closed:
+        episode = await queue.get(timeout=10.0)
+        if episode is None:
+            if queue.is_closed:
+                break
+            continue
+
+        batch.append(episode)
+        total_consumed += 1
+
+        if len(batch) >= buffer_size:
+            if on_batch_ready:
+                on_batch_ready(batch)
+            logger.info("[CONSUMER] Batch of %d episodes ready for PPO update", len(batch))
+            batch = []
+
+    # Flush remaining
+    if batch and on_batch_ready:
+        on_batch_ready(batch)
+        logger.info("[CONSUMER] Final batch of %d episodes flushed", len(batch))
+
+    logger.info("[CONSUMER] Done — consumed %d episodes", total_consumed)
+    return total_consumed
+
+
+# ══════════════════════════════════════════════════════════
+# Convenience runner
+# ══════════════════════════════════════════════════════════
+
+async def run_forge_pipeline(
+    results_dir: str = "results",
+    batch_id: str = "forge-batch",
+    num_games: int = 64,
+    buffer_size: int = 64,
+    max_queue: int = 256,
+    on_batch_ready=None,
+    reward_config: Optional[RewardConfig] = None,
+) -> Dict[str, int]:
+    """Run the full Forge producer → PPO consumer pipeline.
+
+    The Forge Java batch must be started separately (via forge_runner.py
+    or MultiThreadBatchRunner). This function handles the Python side:
+    reading JSONL output and feeding episodes into the PPO trainer.
+    """
+    queue = ForgeEpisodeQueue(max_size=max_queue)
+
+    producer_task = asyncio.create_task(
+        forge_producer(queue, results_dir, batch_id, num_games,
+                       reward_config=reward_config)
+    )
+    consumer_task = asyncio.create_task(
+        ppo_consumer(queue, buffer_size, on_batch_ready)
+    )
+
+    produced = await producer_task
+    queue.close()
+    consumed = await consumer_task
+
+    stats = {
+        "produced": produced,
+        "consumed": consumed,
+        **queue.stats,
+    }
+    logger.info("[PIPELINE] Complete: %s", stats)
+    return stats

ml/training/self_play.py

@@ -17,9 +17,15 @@ that feed directly into the RolloutBuffer for PPO training.
 For real game integration (Phase 9+), the Java sim would call the
 policy server and feed back actual game states — this synthetic
 version allows training to begin immediately.
+
+.. deprecated:: Phase 3 (Issue #83)
+    Synthetic training mode is DEPRECATED for production training.
+    Use forge_episode_generator.py with real Forge engine games.
+    Retained ONLY for quick local testing and unit tests.
 """
 
 import logging
+import warnings
 import random
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Tuple
@@ -42,6 +48,12 @@ from ml.training.decision_exporter import DecisionExporter
 
 logger = logging.getLogger("ml.self_play")
 
+_TRAINING_DEPRECATION_MSG = (
+    "self_play.py synthetic training is deprecated for production training (Issue #83). "
+    "Use forge_episode_generator.py with real Forge engine games instead. "
+    "Synthetic self-play is retained for local testing and unit tests only."
+)
+
 
 @dataclass
 class PlayerState:
@@ -554,7 +566,8 @@ def collect_rollouts(
     total_wins = 0
     total_losses = 0
     total_draws = 0
-    total_steps = 0
+    warnings.warn(_TRAINING_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
+  total_steps = 0
     total_rewards = 0.0
 
     for ep in range(num_episodes):

requirements.txt

@@ -13,3 +13,7 @@ requests>=2.31.0
 beautifulsoup4>=4.12.0
 ollama>=0.2.0
 scrython>=2.0.0
+
+# Embeddings
+sentence-transformers>=3.0.0
+torch>=2.0.0

routes/__init__.py

@@ -2,5 +2,5 @@
 routes package — API routers for Commander AI Lab.
 
 Each module defines a FastAPI APIRouter with real endpoint implementations.
-All shared helpers, models, and state live in routes.shared.
+Shared helpers live in services/, models live in models/.
 """

routes/coach.py

@@ -33,12 +33,14 @@ from fastapi import APIRouter, HTTPException
 from fastapi.responses import StreamingResponse
 from models.requests import CoachRequestBody, CoachChatMessage, CoachChatRequest, CoachApplyRequest, CoachGoalsRequest
 
-from routes.shared import (
-    CFG,
-    log_coach, log_deckgen,
-    _get_db_conn,
-    _compute_deck_analysis,
-)
+from models.state import CFG
+from services.database import _get_db_conn
+from services.deck_service import _compute_deck_analysis
+from services.logging import log_coach, log_deckgen
+import httpx
+from coach.config import COACH_PROVIDER, PPLX_MODEL
+
+LLM_TIMEOUT = 120  # seconds
 
 router = APIRouter(tags=["coach"])
 
@@ -87,7 +89,7 @@ def init_coach_service():
         if llm_status.get("connected"):
             log_coach.info(f"  Coach LLM:    Connected ({llm_status.get('active_model', 'unknown')})")
         else:
-            log_coach.warning(f"  Coach LLM:    Not connected (start Ollama on 192.168.0.240:11434)")
+            log_coach.warning(f"  Coach LLM:    Not connected (start Ollama on localhost:11434)")
 
         # Initialize V3 Deck Generator (Ollama-based)
         _deck_gen_v3_error = None
@@ -419,13 +421,68 @@ async def coach_chat(body: CoachChatRequest):
     # Build messages array for LLM
     messages = [{"role": "system", "content": system_prompt}]
     for msg in body.messages:
-        messages.append({"role": msg.get("role", "user"), "content": msg.get("content", "")})
-
-    # Call LLM with multi-turn messages
+            messages.append({"role": msg.get("role", "user"), "content": msg.get("content", "")})
+          # ── Perplexity provider path ──────────────────────────────
+    if COACH_PROVIDER == 'perplexity':
+      if not CFG.pplx_api_key:
+        raise HTTPException(400, 'Perplexity API key not configured. Set PPLX_API_KEY env var.')
+      pplx_payload = {
+        'model': PPLX_MODEL,
+        'messages': messages,
+        'max_tokens': 4096,
+        'temperature': 0.7,
+      }
+      pplx_headers = {
+        'Content-Type': 'application/json',
+        'Authorization': f'Bearer {CFG.pplx_api_key}',
+      }
+      if body.stream:
+        async def pplx_stream():
+          pplx_payload['stream'] = True
+          async with httpx.AsyncClient(timeout=120.0) as client:
+            async with client.stream('POST', 'https://api.perplexity.ai/chat/completions',
+                                     json=pplx_payload, headers=pplx_headers) as resp:
+              async for line in resp.aiter_lines():
+                if line.startswith('data: '):
+                  data = line[6:]
+                  if data == '[DONE]':
+                    yield 'data: [DONE]\n\n'
+                    break
+                  try:
+                    chunk = json.loads(data)
+                    content = chunk.get('choices', [{}])[0].get('delta', {}).get('content', '')
+                    if content:
+                      yield f'data: {json.dumps({"content": content})}\n\n'
+                  except json.JSONDecodeError:
+                    continue
+        return StreamingResponse(pplx_stream(), media_type='text/event-stream')
+      else:
+        try:
+          async with httpx.AsyncClient(timeout=120.0) as client:
+            resp = await client.post(
+              'https://api.perplexity.ai/chat/completions',
+              json=pplx_payload, headers=pplx_headers,
+            )
+            resp.raise_for_status()
+            raw = resp.json()
+          content = raw.get('choices', [{}])[0].get('message', {}).get('content', '')
+          usage = raw.get('usage', {})
+          return {
+            'content': content,
+            'model': raw.get('model', ''),
+            'prompt_tokens': usage.get('prompt_tokens', 0),
+            'completion_tokens': usage.get('completion_tokens', 0),
+          }
+        except httpx.HTTPStatusError as e:
+          raise HTTPException(502, f'Perplexity API returned {e.response.status_code}')
+        except httpx.RequestError as e:
+          raise HTTPException(502, f'Perplexity API call failed: {e}')
+  
+          # Call LLM with multi-turn messages
     try:
         import json
         from urllib.request import urlopen, Request as UrlRequest
-        from coach.config import LLM_URL, LLM_TIMEOUT
+        
 
         model_name = _coach_llm._resolve_model()
         llm_body = {

routes/collection.py

@@ -25,35 +25,123 @@ from typing import Optional
 from fastapi import APIRouter, HTTPException
 from fastapi.responses import JSONResponse, StreamingResponse
 
-from routes.shared import (
-    CFG,
-    _get_db_conn,
-    _row_to_dict,
-    _add_image_url,
-    _build_collection_filters,
-    _detect_card_roles,
-    _classify_card_type,
-    _enrich_from_scryfall,
-    _fetch_scryfall_api,
-    _parse_finish,
-    _parse_text_line,
-    _parse_csv_content,
-    _auto_infer_mapping,
-    _scryfall_cache,
-    _scryfall_rate_limit,
-    _snake_to_camel,
-    VALID_SORT_FIELDS,
-    _JSON_FIELDS,
-    SCRYFALL_CACHE_DB_PATH,
-    SCRYFALL_CACHE_TTL_SECONDS,
-    _API_HEADERS,
-    log_collect,
-    log_cache,
+from models.state import CFG
+from services.card_analysis import _detect_card_roles
+from services.database import (
+    _get_db_conn, _row_to_dict, _add_image_url, _build_collection_filters,
+    _snake_to_camel, VALID_SORT_FIELDS, _JSON_FIELDS,
+)
+from services.deck_service import _classify_card_type
+from services.import_service import (
+    _parse_finish, _parse_text_line, _parse_csv_content, _auto_infer_mapping,
+)
+from services.logging import log_collect, log_cache
+from services.scryfall import (
+    SCRYFALL_CACHE_DB_PATH, SCRYFALL_CACHE_TTL_SECONDS, _API_HEADERS,
+    _scryfall_cache, _scryfall_rate_limit,
+    _enrich_from_scryfall, _fetch_scryfall_api,
 )
 
 router = APIRouter(tags=["collection"])
 
 
+@router.get("/api/collection/stats")
+async def collection_stats():
+    """Aggregate collection statistics for the dashboard."""
+    conn = _get_db_conn()
+    stats = {}
+
+    # Total cards and unique cards
+    row = conn.execute(
+        "SELECT COUNT(*) as total_rows, COALESCE(SUM(quantity),0) as total_cards, "
+        "COALESCE(SUM(tcg_price * quantity),0) as total_value "
+        "FROM collection_entries"
+    ).fetchone()
+    stats["totalUniqueCards"] = row["total_rows"]
+    stats["totalCards"] = row["total_cards"]
+    stats["totalValue"] = round(row["total_value"], 2)
+
+    # CMC distribution (0-7+)
+    cmc_rows = conn.execute(
+        "SELECT CASE WHEN cmc >= 7 THEN 7 ELSE CAST(cmc AS INTEGER) END as cmc_bucket, "
+        "SUM(quantity) as count FROM collection_entries GROUP BY cmc_bucket ORDER BY cmc_bucket"
+    ).fetchall()
+    stats["cmcCurve"] = {str(r["cmc_bucket"]): r["count"] for r in cmc_rows}
+
+    # Color identity distribution
+    color_rows = conn.execute(
+        "SELECT color_identity, SUM(quantity) as count FROM collection_entries GROUP BY color_identity"
+    ).fetchall()
+    color_counts = {"W": 0, "U": 0, "B": 0, "R": 0, "G": 0, "C": 0}
+    for r in color_rows:
+        ci = r["color_identity"] or "[]"
+        try:
+            colors = json.loads(ci) if isinstance(ci, str) else ci
+        except (json.JSONDecodeError, TypeError):
+            colors = []
+        qty = r["count"] or 0
+        if not colors:
+            color_counts["C"] += qty
+        else:
+            for c in colors:
+                if c in color_counts:
+                    color_counts[c] += qty
+    stats["colorDistribution"] = color_counts
+
+    # Type distribution
+    type_rows = conn.execute(
+        "SELECT type_line, SUM(quantity) as count FROM collection_entries GROUP BY type_line"
+    ).fetchall()
+    type_counts = {}
+    for r in type_rows:
+        tl = r["type_line"] or ""
+        qty = r["count"] or 0
+        for t in ["Creature", "Instant", "Sorcery", "Artifact", "Enchantment", "Planeswalker", "Land"]:
+            if t in tl:
+                type_counts[t] = type_counts.get(t, 0) + qty
+    stats["typeDistribution"] = type_counts
+
+    # Category/role distribution
+    cat_rows = conn.execute(
+        "SELECT category, SUM(quantity) as count FROM collection_entries "
+        "WHERE category != '' AND category != '[]' GROUP BY category"
+    ).fetchall()
+    cat_counts = {}
+    for r in cat_rows:
+        raw = r["category"] or "[]"
+        qty = r["count"] or 0
+        try:
+            cats = json.loads(raw) if isinstance(raw, str) else raw
+        except (json.JSONDecodeError, TypeError):
+            cats = []
+        if isinstance(cats, list):
+            for c in cats:
+                cat_counts[c] = cat_counts.get(c, 0) + qty
+    stats["categoryDistribution"] = cat_counts
+
+    # Rarity distribution
+    rarity_rows = conn.execute(
+        "SELECT LOWER(rarity) as rarity, SUM(quantity) as count FROM collection_entries "
+        "WHERE rarity != '' GROUP BY LOWER(rarity)"
+    ).fetchall()
+    stats["rarityDistribution"] = {r["rarity"]: r["count"] for r in rarity_rows}
+
+    # Top 10 most valuable cards
+    top_value = conn.execute(
+        "SELECT name, tcg_price, quantity FROM collection_entries "
+        "WHERE tcg_price > 0 ORDER BY tcg_price DESC LIMIT 10"
+    ).fetchall()
+    stats["topValueCards"] = [{"name": r["name"], "price": r["tcg_price"], "qty": r["quantity"]} for r in top_value]
+
+    # Average CMC (excluding lands)
+    avg_row = conn.execute(
+        "SELECT AVG(cmc) as avg_cmc FROM collection_entries WHERE type_line NOT LIKE '%Land%'"
+    ).fetchone()
+    stats["avgCmc"] = round(avg_row["avg_cmc"] or 0, 2)
+
+    return stats
+
+
 @router.get("/api/collection/export")
 async def export_collection(
     format: str = "INTERNAL_CSV",
@@ -70,6 +158,17 @@ async def export_collection(
     priceMin: Optional[float] = None,
     priceMax: Optional[float] = None,
     category: Optional[str] = None,
+        rarity: Optional[str] = None,
+    setCode: Optional[str] = None,
+    powerMin: Optional[str] = None,
+    powerMax: Optional[str] = None,
+    toughMin: Optional[str] = None,
+    toughMax: Optional[str] = None,
+    keyword: Optional[str] = None,
+    edhrecMin: Optional[int] = None,
+    edhrecMax: Optional[int] = None,
+    qtyMin: Optional[int] = None,
+    qtyMax: Optional[int] = None,
 ):
     """Export collection in various formats with optional filters."""
     where_str, params = _build_collection_filters(
@@ -77,6 +176,12 @@ async def export_collection(
         isBasic=isBasic, isGameChanger=isGameChanger, highSalt=highSalt,
         finish=finish, cmcMin=cmcMin, cmcMax=cmcMax,
         priceMin=priceMin, priceMax=priceMax, category=category,
+                rarity=rarity, setCode=setCode,
+        powerMin=powerMin, powerMax=powerMax,
+        toughMin=toughMin, toughMax=toughMax,
+        keyword=keyword,
+        edhrecMin=edhrecMin, edhrecMax=edhrecMax,
+        qtyMin=qtyMin, qtyMax=qtyMax,
     )
 
     sql = f"SELECT * FROM collection_entries {where_str} ORDER BY name ASC"

routes/deckbuilder.py

@@ -36,35 +36,25 @@ from urllib.parse import quote
 from fastapi import APIRouter, HTTPException, Request as FastAPIRequest
 from fastapi.responses import JSONResponse
 
-from routes.shared import (
-    CFG,
-    _get_db_conn,
-    _row_to_dict,
-    _add_image_url,
-    _get_deck_or_404,
-    _compute_deck_analysis,
-    _check_ratio_limit,
-    _classify_card_type,
-    _detect_card_roles,
-    _enrich_from_scryfall,
-    _fetch_scryfall_api,
-    _scryfall_rate_limit,
-    _to_edhrec_slug,
-    _edhrec_cache_get,
-    _edhrec_cache_set,
-    _fetch_edhrec_average,
+from models.state import CFG
+from models.requests import (
+    CreateDeckRequest, UpdateDeckRequest, AddDeckCardRequest,
+    PatchDeckCardRequest, BulkAddRequest, BulkAddRecommendedRequest,
+)
+from services.card_analysis import _detect_card_roles
+from services.database import _get_db_conn, _row_to_dict, _add_image_url
+from services.deck_service import (
+    _get_deck_or_404, _compute_deck_analysis, _check_ratio_limit,
+    _classify_card_type, _to_edhrec_slug, _TYPE_TARGETS,
     _save_profile_to_dck,
-    _parse_text_decklist,
-    _API_HEADERS,
-    _TYPE_TARGETS,
-    CreateDeckRequest,
-    UpdateDeckRequest,
-    AddDeckCardRequest,
-    PatchDeckCardRequest,
-    BulkAddRequest,
-    BulkAddRecommendedRequest,
-    log_collect,
-    log_deckgen,
+)
+from services.import_helpers import (
+    _fetch_edhrec_average, _parse_text_decklist,
+    _edhrec_cache_get, _edhrec_cache_set,
+)
+from services.logging import log_collect, log_deckgen
+from services.scryfall import (
+    _API_HEADERS, _enrich_from_scryfall, _fetch_scryfall_api, _scryfall_rate_limit,
 )
 
 router = APIRouter(tags=["deckbuilder"])

routes/deckgen.py

@@ -36,18 +36,57 @@ from fastapi.responses import JSONResponse, StreamingResponse
 from models.requests import DeckResearchRequest, DeckGenerateAIRequest
 
 import routes.coach as _coach  # for _deck_gen_v3 / _deck_gen_v3_error module-level access
-from routes.shared import (
-    CFG,
-    log_deckgen, log_pplx,
-    _get_db_conn,
-    _scryfall_rate_limit, _API_HEADERS,
+from models.state import CFG
+from models.requests import (
     DeckGenerationRequest, DeckGenV3Request, DeckGenV3SubstituteRequest,
-    GeneratedDeckCard, DeckGenerationSourceConfig,
+    DeckGenerationSourceConfig,
 )
+from models.responses import GeneratedDeckCard
+from services.database import _get_db_conn
+from services.logging import log_deckgen, log_pplx
+from services.scryfall import _scryfall_rate_limit, _API_HEADERS
 from services.deck_service import _write_dck_file, _build_dck_lines
 
 router = APIRouter(tags=["deckgen"])
 
+# ── V3 Result Cache (avoids re-generating on export) ─────────────
+import hashlib
+_v3_result_cache: dict[str, tuple[float, dict]] = {}  # key -> (timestamp, result)
+_V3_CACHE_TTL = 600  # 10 minutes
+
+def _v3_cache_key(req) -> str:
+    """Deterministic cache key from a DeckGenV3Request."""
+    blob = json.dumps({
+        'commander': req.commander_name.strip().lower(),
+        'strategy': req.strategy or '',
+        'bracket': req.target_bracket,
+        'budget': req.budget_usd,
+        'budget_mode': req.budget_mode,
+        'omit': sorted(req.omit_cards or []),
+        'collection': req.use_collection,
+        'model': req.model or '',
+    }, sort_keys=True)
+    return hashlib.sha256(blob.encode()).hexdigest()[:16]
+
+def _v3_cache_get(key: str) -> dict | None:
+    """Return cached result if within TTL, else None."""
+    entry = _v3_result_cache.get(key)
+    if entry and (time.time() - entry[0]) < _V3_CACHE_TTL:
+        return entry[1]
+    if entry:
+        del _v3_result_cache[key]
+    return None
+
+def _v3_cache_put(key: str, result: dict):
+    """Store result in cache with current timestamp."""
+    # Evict stale entries if cache grows large
+    if len(_v3_result_cache) > 50:
+        cutoff = time.time() - _V3_CACHE_TTL
+        stale = [k for k, (ts, _) in _v3_result_cache.items() if ts < cutoff]
+        for k in stale:
+            del _v3_result_cache[k]
+    _v3_result_cache[key] = (time.time(), result)
+
 
 # ══════════════════════════════════════════════════════════════
 # Auto Deck Generator
@@ -64,7 +103,7 @@ def _resolve_commander(req: DeckGenerationRequest) -> dict:
     # Try scryfall_id first
     if req.commander_scryfall_id:
         row = conn.execute(
-            "SELECT name, type_line, color_identity, mana_cost, scryfall_id FROM collection_entries WHERE scryfall_id = ? LIMIT 1",
+            "SELECT name, type_line, color_identity, mana_cost, scryfall_id, oracle_text FROM collection_entries WHERE scryfall_id = ? LIMIT 1",
             (req.commander_scryfall_id,)
         ).fetchone()
         if row:
@@ -80,6 +119,7 @@ def _resolve_commander(req: DeckGenerationRequest) -> dict:
                 "color_identity": ci,
                 "type_line": row["type_line"],
                 "mana_cost": row["mana_cost"] or "",
+                "oracle_text": row["oracle_text"] or "",
                 "image_url": f"https://api.scryfall.com/cards/{row['scryfall_id']}?format=image&version=normal",
             }
         # Fallback to Scryfall API
@@ -97,7 +137,7 @@ def _resolve_commander(req: DeckGenerationRequest) -> dict:
     if req.commander_name:
         name_lower = req.commander_name.strip().lower()
         row = conn.execute(
-            "SELECT name, type_line, color_identity, mana_cost, scryfall_id FROM collection_entries WHERE LOWER(name) = ? LIMIT 1",
+            "SELECT name, type_line, color_identity, mana_cost, scryfall_id, oracle_text FROM collection_entries WHERE LOWER(name) = ? LIMIT 1",
             (name_lower,)
         ).fetchone()
         if row:
@@ -113,6 +153,7 @@ def _resolve_commander(req: DeckGenerationRequest) -> dict:
                 "color_identity": ci,
                 "type_line": row["type_line"],
                 "mana_cost": row["mana_cost"] or "",
+                "oracle_text": row["oracle_text"] or "",
                 "image_url": f"https://api.scryfall.com/cards/{row['scryfall_id']}?format=image&version=normal",
             }
 
@@ -144,6 +185,7 @@ def _scryfall_to_commander(data: dict) -> dict:
         "color_identity": ci,
         "type_line": data.get("type_line", ""),
         "mana_cost": data.get("mana_cost", ""),
+        "oracle_text": data.get("oracle_text", ""),
         "image_url": image_uris.get("normal", ""),
     }
 
@@ -194,6 +236,31 @@ def _get_collection_for_colors(color_identity: list) -> list:
     return valid
 
 
+# ── Oracle-text keyword extraction for scoring ────────────────────
+_ORACLE_STOP_WORDS = frozenset({
+    'a', 'an', 'the', 'of', 'to', 'and', 'or', 'is', 'it', 'its',
+    'in', 'on', 'at', 'for', 'by', 'with', 'from', 'as', 'if',
+    'that', 'this', 'you', 'your', 'up', 'all', 'each', 'may',
+    'can', 'one', 'two', 'three', 'four', 'five', 'card', 'cards',
+    'target', 'player', 'creature', 'creatures', 'spell', 'spells',
+    'permanent', 'permanents', 'turn', 'end', 'step', 'phase',
+    'mana', 'color', 'pay', 'cost', 'tap', 'untap', 'put', 'get',
+    'has', 'have', 'are', 'be', 'been', 'was', 'were', 'do', 'does',
+    'any', 'no', 'not', 'only', 'also', 'then', 'when', 'whenever',
+    'where', 'which', 'who', 'them', 'they', 'their', 'other',
+    'more', 'less', 'than', 'into', 'under', 'over', 'until',
+    'would', 'could', 'instead', 'about', 'number', 'equal',
+})
+
+
+def _extract_oracle_keywords(oracle_text: str) -> set[str]:
+    """Extract meaningful keywords from oracle text, filtering stop words."""
+    if not oracle_text:
+        return set()
+    cleaned = re.sub(r'\([^)]*\)', '', oracle_text)
+    tokens = re.findall(r'[a-z]{3,}', cleaned.lower())
+    return {t for t in tokens if t not in _ORACLE_STOP_WORDS}
+
 def _generate_deck(req: DeckGenerationRequest) -> dict:
     """
     Core deck generation algorithm.
@@ -212,6 +279,8 @@ def _generate_deck(req: DeckGenerationRequest) -> dict:
 
     color_identity = req.color_identity or commander.get("color_identity", [])
     log_deckgen.info(f"  Commander: {commander['name']}, CI: {color_identity}")
+    commander_keywords = _extract_oracle_keywords(commander.get("oracle_text", ""))
+    log_deckgen.info(f"  Commander keywords ({len(commander_keywords)}): {sorted(commander_keywords)[:15]}")
 
     # 2. Load collection
     collection = _get_collection_for_colors(color_identity)
@@ -278,6 +347,7 @@ def _generate_deck(req: DeckGenerationRequest) -> dict:
             )
             candidate_map[key] = {
                 "scryfall_id": card.get("scryfall_id", ""),
+                "oracle_text": card.get("oracle_text", ""),
                 "name": name,
                 "type_line": type_line,
                 "mana_cost": card.get("mana_cost", ""),
@@ -370,6 +440,17 @@ def _generate_deck(req: DeckGenerationRequest) -> dict:
                 score += 5
             elif cmc <= 4:
                 score += 3
+            # Oracle-text keyword overlap with commander
+            card_oracle = card.get("oracle_text") or ""
+            if commander_keywords and card_oracle:
+                card_kw = _extract_oracle_keywords(card_oracle)
+                overlap = len(commander_keywords & card_kw)
+                if overlap >= 4:
+                    score += 12
+                elif overlap >= 2:
+                    score += 7
+                elif overlap >= 1:
+                    score += 3
             card["_score"] = score
         cards.sort(key=lambda x: x["_score"], reverse=True)
 
@@ -771,7 +852,7 @@ async def _call_pplx_api(messages: list[dict], max_tokens: int = 4096, temperatu
     Uses httpx.AsyncClient so the uvicorn event loop is never stalled.
     Returns the assistant message content.
     """
-    from coach.config import DECK_GEN_PROVIDER, DECK_GEN_BASE_URL, DECK_GEN_MODEL
+    from coach.config import DECK_GEN_PROVIDER, DECK_GEN_BASE_URL, DECK_GEN_MODEL, PPLX_MODEL
 
     if DECK_GEN_PROVIDER == 'local':
         base_url = DECK_GEN_BASE_URL
@@ -781,7 +862,7 @@ async def _call_pplx_api(messages: list[dict], max_tokens: int = 4096, temperatu
         if not CFG.pplx_api_key:
             raise HTTPException(400, 'Perplexity API key not configured. Set PPLX_API_KEY env var or --pplx-key.')
         base_url = 'https://api.perplexity.ai'
-        model = 'sonar'
+        model = PPLX_MODEL
         headers = {
             'Content-Type': 'application/json',
             'Authorization': f'Bearer {CFG.pplx_api_key}',
@@ -798,7 +879,7 @@ async def _call_pplx_api(messages: list[dict], max_tokens: int = 4096, temperatu
                 f'{base_url}/chat/completions',
                 json=payload,
                 headers=headers,
-                      )
+            )
             resp.raise_for_status()
             data = resp.json()
     except httpx.HTTPStatusError as e:
@@ -947,7 +1028,7 @@ def _fill_basic_lands(cards: list[dict], color_identity: list[str] | None = None
     return non_basics
 
 
-# ── Research Endpoint ─────────────────────────────────────────
+# ── Research Endpoint ───────────────────────────────────────────────────
 
 @router.post('/api/deck-research')
 async def deck_research(req: DeckResearchRequest):
@@ -1102,7 +1183,7 @@ Analyze EVERYTHING: the deck's identity, strategy, archetype, bracket level (1-4
     return analysis
 
 
-# ── Generate Endpoint ─────────────────────────────────────────
+# ── Generate Endpoint ───────────────────────────────────────────────
 
 @router.post('/api/deck-generate')
 async def deck_generate_ai(req: DeckGenerateAIRequest):
@@ -1232,7 +1313,7 @@ async def deck_gen_v3_status():
         'initialized': _coach._deck_gen_v3 is not None,
         'pplx_configured': bool(CFG.pplx_api_key),
         'model': getattr(_coach._deck_gen_v3, 'model', getattr(getattr(_coach._deck_gen_v3, 'pplx', None), 'model', 'ollama/gpt-oss:20b')) if _coach._deck_gen_v3 else None,
-                'embeddings_loaded': False,
+        'embeddings_loaded': False,
         'embedding_cards': 0,
         'error': _coach._deck_gen_v3_error,
     }
@@ -1289,6 +1370,9 @@ async def deck_gen_v3_generate(req: DeckGenV3Request):
             from coach.services.deck_generator import DeckGeneratorV3
             result['stats'] = DeckGeneratorV3._compute_stats(sub_result.cards)
 
+        # Cache result for export endpoints
+        _v3_cache_put(_v3_cache_key(req), result)
+
         return result
 
     except ValueError as e:
@@ -1406,17 +1490,20 @@ async def deck_gen_v3_export_csv(req: DeckGenV3Request):
     if _coach._deck_gen_v3 is None:
         raise HTTPException(503, 'V3 Deck Generator not initialized.')
 
-    result = _coach._deck_gen_v3.generate_deck(
-        commander_name=req.commander_name.strip(),
-        strategy=req.strategy,
-        target_bracket=req.target_bracket,
-        budget_usd=req.budget_usd,
-        budget_mode=req.budget_mode,
-        omit_cards=req.omit_cards,
-        use_collection=req.use_collection,
-        model=req.model,
-    )
-
+    _ck = _v3_cache_key(req)
+    result = _v3_cache_get(_ck)
+    if result is None:
+        result = _coach._deck_gen_v3.generate_deck(
+            commander_name=req.commander_name.strip(),
+            strategy=req.strategy,
+            target_bracket=req.target_bracket,
+            budget_usd=req.budget_usd,
+            budget_mode=req.budget_mode,
+            omit_cards=req.omit_cards,
+            use_collection=req.use_collection,
+            model=req.model,
+        )
+        _v3_cache_put(_ck, result)
     cards = result.get('cards', [])
     commander = result.get('commander', {})
 
@@ -1453,16 +1540,20 @@ async def deck_gen_v3_export_dck(req: DeckGenV3Request):
     if _coach._deck_gen_v3 is None:
         raise HTTPException(503, 'V3 Deck Generator not initialized.')
 
-    result = _coach._deck_gen_v3.generate_deck(
-        commander_name=req.commander_name.strip(),
-        strategy=req.strategy,
-        target_bracket=req.target_bracket,
-        budget_usd=req.budget_usd,
-        budget_mode=req.budget_mode,
-        omit_cards=req.omit_cards,
-        use_collection=req.use_collection,
-        model=req.model,
-    )
+    _ck = _v3_cache_key(req)
+    result = _v3_cache_get(_ck)
+    if result is None:
+        result = _coach._deck_gen_v3.generate_deck(
+            commander_name=req.commander_name.strip(),
+            strategy=req.strategy,
+            target_bracket=req.target_bracket,
+            budget_usd=req.budget_usd,
+            budget_mode=req.budget_mode,
+            omit_cards=req.omit_cards,
+            use_collection=req.use_collection,
+            model=req.model,
+        )
+        _v3_cache_put(_ck, result)
 
     cards = result.get('cards', [])
     commander = result.get('commander', {})
@@ -1488,16 +1579,20 @@ async def deck_gen_v3_export_moxfield(req: DeckGenV3Request):
     if _coach._deck_gen_v3 is None:
         raise HTTPException(503, 'V3 Deck Generator not initialized.')
 
-    result = _coach._deck_gen_v3.generate_deck(
-        commander_name=req.commander_name.strip(),
-        strategy=req.strategy,
-        target_bracket=req.target_bracket,
-        budget_usd=req.budget_usd,
-        budget_mode=req.budget_mode,
-        omit_cards=req.omit_cards,
-        use_collection=req.use_collection,
-        model=req.model,
-    )
+    _ck = _v3_cache_key(req)
+    result = _v3_cache_get(_ck)
+    if result is None:
+        result = _coach._deck_gen_v3.generate_deck(
+            commander_name=req.commander_name.strip(),
+            strategy=req.strategy,
+            target_bracket=req.target_bracket,
+            budget_usd=req.budget_usd,
+            budget_mode=req.budget_mode,
+            omit_cards=req.omit_cards,
+            use_collection=req.use_collection,
+            model=req.model,
+        )
+        _v3_cache_put(_ck, result)
 
     cards = result.get('cards', [])
     commander = result.get('commander', {})
@@ -1525,22 +1620,26 @@ async def deck_gen_v3_export_shopping(req: DeckGenV3Request):
     if _coach._deck_gen_v3 is None:
         raise HTTPException(503, 'V3 Deck Generator not initialized.')
 
-    result = _coach._deck_gen_v3.generate_deck(
-        commander_name=req.commander_name.strip(),
-        strategy=req.strategy,
-        target_bracket=req.target_bracket,
-        budget_usd=req.budget_usd,
-        budget_mode=req.budget_mode,
-        omit_cards=req.omit_cards,
-        use_collection=req.use_collection,
-        model=req.model,
-    )
+    _ck = _v3_cache_key(req)
+    result = _v3_cache_get(_ck)
+    if result is None:
+        result = _coach._deck_gen_v3.generate_deck(
+            commander_name=req.commander_name.strip(),
+            strategy=req.strategy,
+            target_bracket=req.target_bracket,
+            budget_usd=req.budget_usd,
+            budget_mode=req.budget_mode,
+            omit_cards=req.omit_cards,
+            use_collection=req.use_collection,
+            model=req.model,
+        )
+        _v3_cache_put(_ck, result)
 
     cards = result.get('cards', [])
     shopping = []
     total = 0.0
     for card in cards:
-        if not card.get('owned', True) and card.get('status') != 'substituted':
+        if not card.get('from_collection', False) and card.get('status') != 'substituted':
             price = card.get('estimated_price_usd', 0)
             shopping.append({
                 'name': card.get('name', ''),
@@ -1557,5 +1656,3 @@ async def deck_gen_v3_export_shopping(req: DeckGenV3Request):
         'total_missing': len(shopping),
         'estimated_cost_usd': round(total, 2),
     }
-
-

routes/deepseek.py

@@ -28,12 +28,10 @@ from fastapi import APIRouter, HTTPException
 from fastapi import Request as FastAPIRequest
 from fastapi.responses import JSONResponse
 
-from routes.shared import (
-    CFG, log_sim,
-    _get_db_conn,
-    _get_deepseek_brain,
-    _ml_logging_enabled,
-)
+from models.state import CFG
+from services.database import _get_db_conn
+from services.forge_runner import _get_deepseek_brain
+from services.logging import log_sim, _ml_logging_enabled
 from commander_ai_lab.sim.validator_brain import ValidatorBrain, ValidatorConfig
 logger = logging.getLogger(__name__)
 
@@ -136,7 +134,7 @@ def _run_game_loop(engine, deck_a, deck_b, deck_name, opponent_name,
                             name_b=opponent_name, **kw)
         game_data = result.to_dict()
         game_data['gameNumber'] = i + 1
-              # Phase 8: Surface player_explanation from validation in turn log
+        # Phase 8: Surface player_explanation from validation in turn log
         for turn_entry in game_data.get('log', []):
             v = turn_entry.get('validation')
             if isinstance(v, dict) and v.get('player_explanation'):
@@ -210,7 +208,7 @@ def _finish_sim(sim_id, summary, game_results, engine=None):
         _sim_runs[sim_id]['status'] = 'complete'
         _sim_runs[sim_id]['result'] = {'summary': summary, 'games': game_results}
 
-          # --- Persist batch-*.json for coach / training dashboard ---
+    # --- Persist ml-decision-*.json for coach / training dashboard ---
     try:
         deck_name = summary.get('deckName', 'Unknown')
         opponent_name = summary.get('opponentName', 'Training Deck')
@@ -237,7 +235,7 @@ def _finish_sim(sim_id, summary, game_results, engine=None):
         }
         results_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'results')
         os.makedirs(results_dir, exist_ok=True)
-        batch_path = os.path.join(results_dir, f'batch-sim-{sim_id}.json')
+        batch_path = os.path.join(results_dir, f'ml-decision-sim-{sim_id}.json')
         with open(batch_path, 'w') as f:
             json.dump(batch_data, f, indent=2, default=str)
         logger.info(f'Saved batch results to {batch_path}')
@@ -257,6 +255,8 @@ def _finish_sim(sim_id, summary, game_results, engine=None):
         logger.error(f'Failed to persist sim results: {e}')
 
     # --- Flush ML decisions JSONL for training pipeline ---
+    # NOTE: DeepSeek thread flushes manually before calling _finish_sim(engine=None)
+    # to avoid a double-flush that would produce an empty .jsonl file.
     if engine is not None and hasattr(engine, 'flush_ml_decisions'):
         ml_decisions = engine.flush_ml_decisions()
         if ml_decisions:
@@ -266,7 +266,8 @@ def _finish_sim(sim_id, summary, game_results, engine=None):
                 for dec in ml_decisions:
                     mf.write(json.dumps(dec) + '\n')
             logger.info(f'Wrote {len(ml_decisions)} ML decisions to {ml_path}')
-        
+
+
 def _fail_sim(sim_id, error):
     """Mark a sim run as errored."""
     with _sim_lock:
@@ -362,21 +363,32 @@ def _run_sim_thread_deepseek(sim_id: str, card_data: list[dict],
             game_id_prefix=f'ds-sim-{sim_id[:8]}',
             archetype='midrange')
 
-        # Write ML decision JSONL for training pipeline
+        # Flush ML decisions once here — pass engine=None to _finish_sim
+        # to prevent a second flush that would produce an empty .jsonl file.
         ml_decisions = engine.flush_ml_decisions()
-        if ml_decisions:
+        if ml_decisions and brain and brain._connected:
             ml_path = os.path.join('results',
                                    f'ml-decisions-sim-{sim_id[:8]}.jsonl')
             os.makedirs('results', exist_ok=True)
             with open(ml_path, 'w', encoding='utf-8') as mf:
                 for dec in ml_decisions:
                     mf.write(json.dumps(dec) + '\n')
+            logger.info(f'[ML Data] Wrote {len(ml_decisions)} decision snapshots to {ml_path}')
+        elif ml_decisions and (not brain or not brain._connected):
+            logger.warning(
+                f'[ML Data] Skipping {len(ml_decisions)} snapshots for sim {sim_id} — '
+                f'brain not connected, all actions are heuristic fallback '
+                f'(not useful for supervised training)'
+            )
+        else:
+            logger.warning(f'[ML Data] No decision snapshots captured for sim {sim_id} (0 decisions)')
 
         ds_stats = brain.get_stats() if brain else {}
         summary = _build_summary(
             stats, deck_name, 'DeepSeek AI', num_games, elapsed,
             opponentType='deepseek', deepseekStats=ds_stats)
-        _finish_sim(sim_id, summary, game_results, engine=engine)
+        # engine=None: ML decisions already flushed above — do not flush again
+        _finish_sim(sim_id, summary, game_results, engine=None)
     except Exception as e:
         _fail_sim(sim_id, e)
         traceback.print_exc()
@@ -783,7 +795,7 @@ async def deepseek_logs():
         'flushedPath': flushed_path,
         'logFiles': log_files[-20:],
     })
-  
+
 
 @router.get("/api/validator/status")
 async def validator_status():

routes/import_routes.py

@@ -15,17 +15,13 @@ from typing import Optional
 from fastapi import APIRouter, HTTPException, BackgroundTasks
 from fastapi.responses import JSONResponse
 
-import routes.shared as _shared
-from routes.shared import (
-    ImportUrlRequest,
-    ImportTextRequest,
-    MetaFetchRequest,
-    _import_from_url,
-    _parse_text_decklist,
-    _save_profile_to_dck,
+from models.state import COMMANDER_META
+from models.requests import ImportUrlRequest, ImportTextRequest, MetaFetchRequest
+from services.import_helpers import (
+    _import_from_url, _parse_text_decklist, _save_profile_to_dck,
     _fetch_edhrec_average,
-    log,
 )
+from services.logging import log
 
 router = APIRouter(tags=["import"])
 
@@ -74,7 +70,7 @@ async def import_from_text(req: ImportTextRequest):
 async def list_meta_commanders():
     """List available commanders in the meta mapping."""
     commanders = []
-    for name, entries in _shared.COMMANDER_META.items():
+    for name, entries in COMMANDER_META.items():
         entry = entries[0] if entries else {}
         commanders.append({
             "name": name,
@@ -90,7 +86,7 @@ async def search_meta_commanders(q: str = ""):
     """Search commanders by partial name match."""
     query = q.lower()
     matches = []
-    for name, entries in _shared.COMMANDER_META.items():
+    for name, entries in COMMANDER_META.items():
         if query in name.lower():
             entry = entries[0] if entries else {}
             matches.append({

routes/lab.py

@@ -31,23 +31,15 @@ from typing import Optional
 from fastapi import APIRouter, HTTPException, BackgroundTasks, Request as FastAPIRequest
 from fastapi.responses import JSONResponse
 
-from routes.shared import (
-    CFG,
-    active_batches,
-    BatchState,
-    StartRequest,
-    StartResponse,
-    StatusResponse,
-    AI_PROFILES,
-    build_java_command,
-    run_batch_subprocess,
-    _run_deepseek_batch_thread,
-    parse_dck_file,
-    _get_db_conn,
-    _load_deck_cards_by_name,
-    log,
-    log_batch,
+from models.state import CFG, active_batches, BatchState, AI_PROFILES
+from models.requests import StartRequest
+from models.responses import StartResponse, StatusResponse
+from services.database import _get_db_conn
+from services.deck_service import parse_dck_file, _load_deck_cards_by_name
+from services.forge_runner import (
+    build_java_command, run_batch_subprocess, _run_deepseek_batch_thread,
 )
+from services.logging import log, log_batch
 
 router = APIRouter(tags=["lab"])
 
@@ -87,7 +79,7 @@ async def start_batch(req: StartRequest, background_tasks: BackgroundTasks):
     active_batches[batch_id] = state
 
     os.makedirs(CFG.results_dir, exist_ok=True)
-    output_path = os.path.join(CFG.results_dir, f"batch-{batch_id}.json")
+    output_path = os.path.join(CFG.results_dir, f"ml-decision-{batch_id}.json")
     state.result_path = output_path
 
       # Auto-export deckbuilder decks to .dck if not already exported
@@ -250,7 +242,7 @@ async def list_history():
     if not results_dir.exists():
         return {"results": []}
     results = []
-    for f in sorted(results_dir.glob("batch-*.json"), reverse=True):
+    for f in sorted(results_dir.glob("ml-decision-*.json"), reverse=True):
         try:
             with open(f, "r", encoding="utf-8") as fh:
                 data = json.load(fh)
@@ -383,7 +375,7 @@ async def get_deck_trends(deck_name: str):
     if not results_dir.exists():
         return {"deckName": deck_name, "history": []}
     history = []
-    for f in sorted(results_dir.glob("batch-*.json")):
+    for f in sorted(results_dir.glob("ml-decision-*.json")):
         try:
             with open(f, "r", encoding="utf-8") as fh:
                 data = json.load(fh)

routes/ml.py

@@ -32,7 +32,8 @@ from pathlib import Path
 
 from fastapi import APIRouter, HTTPException, Request as FastAPIRequest
 
-from routes.shared import CFG, _ml_logging_enabled
+from models.state import CFG
+from services.logging import _ml_logging_enabled
 
 log_ml = logging.getLogger("commander_ai_lab.ml")
 
@@ -121,7 +122,7 @@ def _get_policy_service():
 @router.get("/api/ml/status")
 async def ml_status():
     """Get ML decision logging status and available training data."""
-    import routes.shared as _shared
+    import services.logging as _logging_mod
     lab_root = Path(__file__).resolve().parent.parent
     results_dir = lab_root / CFG.results_dir
     ml_files = []
@@ -140,7 +141,7 @@ async def ml_status():
             })
     total_decisions = sum(f["decisions"] for f in ml_files)
     return {
-        "ml_logging_enabled": _shared._ml_logging_enabled,
+        "ml_logging_enabled": _logging_mod._ml_logging_enabled,
         "training_files": ml_files,
         "total_decisions": total_decisions,
         "total_files": len(ml_files),
@@ -150,10 +151,10 @@ async def ml_status():
 @router.post("/api/ml/toggle")
 async def ml_toggle(enable: bool = True):
     """Enable or disable ML decision logging for future batch runs."""
-    import routes.shared as _shared
-    _shared._ml_logging_enabled = enable
+    import services.logging as _logging_mod
+    _logging_mod._ml_logging_enabled = enable
     return {
-        "ml_logging_enabled": _shared._ml_logging_enabled,
+        "ml_logging_enabled": _logging_mod._ml_logging_enabled,
         "message": f"ML decision logging {'enabled' if enable else 'disabled'} for future batches",
     }
 

routes/policy.py

@@ -0,0 +1,321 @@
+"""Commander AI Lab — Policy Decision Routes (Phase 2)
+════════════════════════════════════════════════════
+
+Live Forge ↔ policy server IPC endpoints for online training.
+These endpoints are purpose-built for the real-time decision loop
+where Forge's Java side calls the Python policy server at each
+game decision point.
+
+Endpoints:
+    POST /api/policy/decide   — Accept full Forge game state, return action + log_prob + value
+    POST /api/policy/collect   — Accept batch of (state, action, reward) tuples
+    POST /api/policy/reward    — Submit end-of-game reward
+    GET  /api/policy/health    — Readiness check for the live IPC loop
+    GET  /api/policy/stats     — Online learning session statistics
+
+Refs: Issue #83 Phase 2.2
+"""
+import logging
+import time
+from typing import Dict, List, Optional
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger("routes.policy")
+
+router = APIRouter(prefix="/api/policy", tags=["policy"])
+
+
+# ================================================================
+# Request / Response Models
+# ================================================================
+
+class PlayerZoneState(BaseModel):
+    """Per-player zone state from Forge."""
+    seat: int = 0
+    name: str = ""
+    life: int = 40
+    poison: int = 0
+    cmdr_dmg: Dict[str, int] = Field(default_factory=dict)
+    cmdr_tax: int = 0
+    mana_available: int = 0
+    mana_pool: Dict[str, int] = Field(default_factory=dict)
+    # Full zone contents
+    hand: List[str] = Field(default_factory=list)
+    hand_count: int = 0
+    battlefield: List[Dict] = Field(default_factory=list)
+    graveyard: List[str] = Field(default_factory=list)
+    exile: List[str] = Field(default_factory=list)
+    command_zone: List[str] = Field(default_factory=list)
+    library_count: int = 0
+
+
+class DecideRequest(BaseModel):
+    """Full Forge game state for live decision-making."""
+    # Game metadata
+    game_id: str = ""
+    turn: int = 1
+    phase: str = "main_1"
+    active_player: int = 0
+    priority_player: int = 0
+    # Per-player state with full zones
+    players: List[PlayerZoneState] = Field(default_factory=list)
+    # Stack (spells/abilities currently resolving)
+    stack: List[Dict] = Field(default_factory=list)
+    # Legal actions from Forge
+    legal_actions: List[Dict] = Field(default_factory=list)
+    # Inference params
+    playstyle: str = "midrange"
+    greedy: bool = False
+    temperature: float = 1.0
+    # Optional commander info
+    commander: str = ""
+    deck_name: str = ""
+
+
+class DecideResponse(BaseModel):
+    """Action decision with metadata for online learning."""
+    action: str
+    action_index: int
+    confidence: float
+    log_prob: float = 0.0
+    value: float = 0.0
+    probabilities: Dict[str, float] = Field(default_factory=dict)
+    inference_ms: float = 0.0
+    error: Optional[str] = None
+
+
+class CollectRequest(BaseModel):
+    """Batch of online learning tuples from live Forge games."""
+    tuples: List[Dict] = Field(default_factory=list)
+    count: int = 0
+
+
+class RewardRequest(BaseModel):
+    """End-of-game reward submission."""
+    game_id: str
+    reward: float
+    winner_seat: int = -1
+    turns_played: int = 0
+    reason: str = ""  # e.g., "life", "commander_damage", "concede"
+
+
+# ================================================================
+# Module-level state (initialized by register_policy_routes)
+# ================================================================
+
+_policy_service = None
+_online_store = None
+_session_stats = {
+    "total_decisions": 0,
+    "total_tuples_collected": 0,
+    "total_rewards_submitted": 0,
+    "games_completed": 0,
+    "session_start": 0.0,
+    "last_decision_ms": 0.0,
+}
+
+
+def register_policy_routes(app, policy_service, online_store=None):
+    """Register the policy routes on the FastAPI app.
+
+    Args:
+        app: FastAPI application instance
+        policy_service: PolicyInferenceService from ml.serving.policy_server
+        online_store: Optional OnlineLearningStore for tuple collection
+    """
+    global _policy_service, _online_store
+    _policy_service = policy_service
+    _online_store = online_store
+    _session_stats["session_start"] = time.time()
+    app.include_router(router)
+    logger.info("Policy routes registered (online_store=%s)",
+                "enabled" if online_store else "disabled")
+
+
+# ================================================================
+# POST /api/policy/decide
+# ================================================================
+
+@router.post("/decide", response_model=DecideResponse)
+async def decide(req: DecideRequest):
+    """Accept full Forge game state, run policy inference, return action.
+
+    This is the primary endpoint for live Forge IPC. The Java
+    PolicyClient calls this at each decision point during a real
+    Forge game.
+
+    Returns the macro-action plus log_prob and value estimate
+    needed for online PPO updates.
+    """
+    if _policy_service is None or not _policy_service._loaded:
+        raise HTTPException(
+            503,
+            detail="Policy model not loaded"
+        )
+
+    t_start = time.time()
+    _session_stats["total_decisions"] += 1
+
+    # Convert request to snapshot dict for the encoder
+    snapshot = req.dict(exclude={"temperature", "greedy", "playstyle"})
+
+    # Map phase names from Forge conventions
+    snapshot["phase"] = _map_forge_phase(req.phase)
+    snapshot["active_seat"] = req.active_player
+
+    result = _policy_service.predict(
+        snapshot,
+        playstyle=req.playstyle,
+        temperature=req.temperature,
+        greedy=req.greedy,
+    )
+
+    elapsed_ms = (time.time() - t_start) * 1000
+    _session_stats["last_decision_ms"] = elapsed_ms
+
+    if "error" in result:
+        raise HTTPException(500, detail=result["error"])
+
+    # Add log_prob and value estimate for online learning
+    log_prob = 0.0
+    value = 0.0
+    try:
+        import math
+        conf = result.get("confidence", 0.5)
+        log_prob = math.log(max(conf, 1e-8))
+    except Exception:
+        pass
+
+    return DecideResponse(
+        action=result["action"],
+        action_index=result["action_index"],
+        confidence=result["confidence"],
+        log_prob=round(log_prob, 6),
+        value=round(value, 6),
+        probabilities=result.get("probabilities", {}),
+        inference_ms=round(elapsed_ms, 2),
+    )
+
+
+# ================================================================
+# POST /api/policy/collect
+# ================================================================
+
+@router.post("/collect")
+async def collect_tuples(req: CollectRequest):
+    """Accept a batch of (state, action, reward) tuples from live games
+    for online PPO updates.
+    """
+    count = len(req.tuples)
+    _session_stats["total_tuples_collected"] += count
+
+    if _online_store is not None:
+        try:
+            _online_store.add_tuples(req.tuples)
+        except Exception as e:
+            logger.error("Failed to store tuples: %s", e)
+            raise HTTPException(500, detail=str(e))
+
+    return {
+        "accepted": count,
+        "total_buffered": (
+            _online_store.size() if _online_store else count
+        ),
+    }
+
+
+# ================================================================
+# POST /api/policy/reward
+# ================================================================
+
+@router.post("/reward")
+async def submit_reward(req: RewardRequest):
+    """Submit end-of-game reward for all tuples in a game.
+    Propagates the terminal reward back through the game's trajectory.
+    """
+    _session_stats["total_rewards_submitted"] += 1
+    _session_stats["games_completed"] += 1
+
+    if _online_store is not None:
+        try:
+            _online_store.assign_reward(
+                game_id=req.game_id,
+                reward=req.reward,
+                winner_seat=req.winner_seat,
+            )
+        except Exception as e:
+            logger.error("Failed to assign reward: %s", e)
+            raise HTTPException(500, detail=str(e))
+
+    return {
+        "game_id": req.game_id,
+        "reward": req.reward,
+        "status": "accepted",
+    }
+
+
+# ================================================================
+# GET /api/policy/health
+# ================================================================
+
+@router.get("/health")
+async def policy_health():
+    """Readiness check for the live Forge IPC loop."""
+    model_loaded = (
+        _policy_service is not None and _policy_service._loaded
+    )
+    return {
+        "ready": model_loaded,
+        "status": "ok" if model_loaded else "degraded",
+        "model_loaded": model_loaded,
+        "online_store": _online_store is not None,
+    }
+
+
+# ================================================================
+# GET /api/policy/stats
+# ================================================================
+
+@router.get("/stats")
+async def policy_stats():
+    """Online learning session statistics."""
+    uptime = time.time() - _session_stats["session_start"]
+    return {
+        **_session_stats,
+        "uptime_seconds": round(uptime, 1),
+        "decisions_per_second": (
+            round(_session_stats["total_decisions"] / max(uptime, 1), 2)
+        ),
+        "online_store_size": (
+            _online_store.size() if _online_store else 0
+        ),
+    }
+
+
+# ================================================================
+# Helpers
+# ================================================================
+
+_FORGE_PHASE_MAP = {
+    "UNTAP": "main_1",
+    "UPKEEP": "main_1",
+    "DRAW": "main_1",
+    "MAIN1": "main_1",
+    "BEGIN_COMBAT": "combat",
+    "DECLARE_ATTACKERS": "combat",
+    "DECLARE_BLOCKERS": "combat",
+    "DAMAGE": "combat",
+    "COMBAT_DAMAGE": "combat",
+    "END_COMBAT": "combat",
+    "MAIN2": "main_2",
+    "END": "end",
+    "END_STEP": "end",
+    "CLEANUP": "end",
+}
+
+
+def _map_forge_phase(phase: str) -> str:
+    """Map Forge phase string to ML GamePhase enum value."""
+    return _FORGE_PHASE_MAP.get(phase.upper(), "main_1")

routes/precon.py

@@ -15,8 +15,9 @@ from pathlib import Path
 
 from fastapi import APIRouter, HTTPException
 
-import routes.shared as _shared                    # module reference, not value import
-from routes.shared import CFG, download_precon_database, log
+from models.state import CFG
+from services.precon_service import download_precon_database, PRECON_INDEX
+from services.logging import log
 from services.precon_service import _get_precon_dir
 
 router = APIRouter(tags=["precon"])
@@ -25,7 +26,7 @@ router = APIRouter(tags=["precon"])
 @router.get("/api/lab/precons")
 async def list_precons():
     """List all available precon decks."""
-    return {"precons": _shared.PRECON_INDEX}        # reads current value every call
+    return {"precons": PRECON_INDEX}        # reads current value every call
 
 
 @router.post("/api/lab/precons/install")

routes/scanner.py

@@ -22,13 +22,13 @@ from urllib.error import URLError
 from fastapi import APIRouter, HTTPException
 from fastapi import Request as FastAPIRequest
 
-from routes.shared import (
-    CFG, log_scan, log_deckgen,
-    _get_db_conn, _row_to_dict, _enrich_from_scryfall,
-    _scryfall_fuzzy_lookup, _detect_card_roles,
-    _to_edhrec_slug, _API_HEADERS,
-    _edhrec_cache_get, _edhrec_cache_set,
-)
+from models.state import CFG
+from services.card_analysis import _detect_card_roles
+from services.database import _get_db_conn, _row_to_dict
+from services.deck_service import _to_edhrec_slug
+from services.import_helpers import _edhrec_cache_get, _edhrec_cache_set
+from services.logging import log_scan, log_deckgen
+from services.scryfall import _API_HEADERS, _enrich_from_scryfall, _scryfall_fuzzy_lookup
 
 router = APIRouter(tags=["scanner"])
 

routes/shared.py

@@ -1,204 +0,0 @@
-"""
-routes/shared.py
-================
-Single source of truth for everything every router needs.
-
-Contains:
-  - Config class + CFG singleton
-  - Logging setup + all named loggers
-  - FastAPI app instance
-  - In-memory state (active_batches, COMMANDER_META, PRECON_INDEX, AI_PROFILES)
-  - All Pydantic request/response models
-  - SQLite DB helpers (_get_db_conn, init_collection_db)
-  - Scryfall cache + enrichment helpers
-  - Collection helpers (_detect_card_roles, _classify_card_type,
-      _build_collection_filters, _parse_finish, _parse_csv_content,
-      _parse_text_line, _auto_infer_mapping, _row_to_dict, _add_image_url,
-      _snake_to_camel, VALID_SORT_FIELDS, _JSON_FIELDS)
-  - Deck helpers (_get_deck_or_404, _compute_deck_analysis, _check_ratio_limit,
-      _TYPE_TARGETS, _TYPE_PRIORITY, _classify_card_type)
-  - Import helpers (_import_from_url, _fetch_archidekt_deck,
-      _fetch_edhrec_average, _parse_text_decklist, _save_profile_to_dck,
-      _to_edhrec_slug, _http_get, _API_HEADERS, _edhrec_cache_get,
-      _edhrec_cache_set)
-  - Precon helpers (PRECON_DIR, PRECON_INDEX, download_precon_database,
-      load_precon_index, GITHUB_PRECON_URL, PRECON_CACHE_HOURS)
-  - Sim helpers (build_java_command, get_java17, BatchState, parse_dck_file,
-      AI_PROFILES, run_batch_subprocess, _run_process_blocking,
-      _run_deepseek_batch_thread, _load_deck_cards_by_name, _get_deepseek_brain)
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import csv
-import io
-import json
-import logging
-import logging.handlers
-import os
-import re
-import sqlite3
-import subprocess
-import sys
-import threading
-import time
-import uuid
-from collections import Counter
-from datetime import datetime
-from pathlib import Path
-from typing import Optional
-from urllib.request import urlopen, Request
-from urllib.error import URLError
-
-try:
-    from fastapi import FastAPI, HTTPException, BackgroundTasks, Request as FastAPIRequest
-    from fastapi.middleware.cors import CORSMiddleware
-    from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
-    from fastapi.staticfiles import StaticFiles
-    from pydantic import BaseModel
-except ImportError:
-    print("ERROR: FastAPI not installed. Run: pip install fastapi uvicorn")
-    sys.exit(1)
-
-
-# ══════════════════════════════════════════════════════════════
-# Configuration
-# ══════════════════════════════════════════════════════════════
-
-# Configuration (imported from models/state)
-from models.state import Config, CFG
-
-
-# ══════════════════════════════════════════════════════════════
-# Logging Setup
-# ══════════════════════════════════════════════════════════════
-
-_LOG_FORMAT = "%(asctime)s %(levelname)-5s [%(name)s] %(message)s"
-_LOG_DATE_FMT = "%Y-%m-%d %H:%M:%S"
-_LOG_DIR = Path(os.environ.get("CAL_LOG_DIR", "logs"))
-_LOG_MAX_BYTES = int(os.environ.get("CAL_LOG_MAX_MB", "25")) * 1024 * 1024
-_LOG_BACKUP_COUNT = int(os.environ.get("CAL_LOG_BACKUP_COUNT", "5"))
-
-
-def setup_logging(level: int = logging.INFO) -> None:
-    _LOG_DIR.mkdir(parents=True, exist_ok=True)
-    root_logger = logging.getLogger("commander_ai_lab")
-    root_logger.setLevel(level)
-    if root_logger.handlers:
-        return
-    formatter = logging.Formatter(_LOG_FORMAT, datefmt=_LOG_DATE_FMT)
-    console = logging.StreamHandler(sys.stdout)
-    console.setLevel(level)
-    console.setFormatter(formatter)
-    root_logger.addHandler(console)
-    file_handler = logging.handlers.RotatingFileHandler(
-        _LOG_DIR / "commander-ai-lab.log",
-        maxBytes=_LOG_MAX_BYTES,
-        backupCount=_LOG_BACKUP_COUNT,
-        encoding="utf-8",
-    )
-    file_handler.setLevel(level)
-    file_handler.setFormatter(formatter)
-    root_logger.addHandler(file_handler)
-
-
-log          = logging.getLogger("commander_ai_lab.api")
-log_batch    = logging.getLogger("commander_ai_lab.batch")
-log_sim      = logging.getLogger("commander_ai_lab.sim")
-log_coach    = logging.getLogger("commander_ai_lab.coach")
-log_deckgen  = logging.getLogger("commander_ai_lab.deckgen")
-log_collect  = logging.getLogger("commander_ai_lab.collection")
-log_scan     = logging.getLogger("commander_ai_lab.scanner")
-log_ml       = logging.getLogger("commander_ai_lab.ml")
-log_cache    = logging.getLogger("commander_ai_lab.cache")
-log_pplx     = logging.getLogger("commander_ai_lab.pplx")
-
-
-# ══════════════════════════════════════════════════════════════
-# In-Memory State
-# ══════════════════════════════════════════════════════════════
-
-# In-Memory State (imported from models/state)
-from models.state import (
-    BatchState, active_batches, COMMANDER_META,
-    BUILTIN_COMMANDERS, load_commander_meta,
-)
-
-
-# Pydantic Models (imported from models/)
-from models.requests import (
-    StartRequest, ImportUrlRequest, ImportTextRequest, MetaFetchRequest,
-    CreateDeckRequest, UpdateDeckRequest, AddDeckCardRequest,
-    PatchDeckCardRequest, BulkAddRequest, BulkAddRecommendedRequest,
-    DeckGenerationSourceConfig, DeckGenerationRequest,
-    DeckGenV3Request, DeckGenV3SubstituteRequest,
-)
-from models.responses import (
-    StartResponse, StatusResponse, DeckInfo, GeneratedDeckCard,
-)
-
-
-# Collection Database (imported from services/database.py)
-from services.database import (
-    COLLECTION_DB_PATH, _get_db_conn, init_collection_db,
-    _JSON_FIELDS, VALID_SORT_FIELDS,
-    _row_to_dict, _snake_to_camel, _add_image_url, _build_collection_filters,
-)
-# Card Role Detection (imported from services/card_analysis.py)
-from services.card_analysis import _detect_card_roles
-
-
-# ══════════════════════════════════════════════════════════════
-# Deck Helpers (imported from services/deck_service.py)
-from services.deck_service import (
-    _TYPE_PRIORITY, _TYPE_TARGETS, _classify_card_type, _get_deck_or_404,
-    _compute_deck_analysis, _check_ratio_limit, _to_edhrec_slug,
-    parse_dck_file, _save_profile_to_dck, _load_deck_cards_by_name,
-)
-
-
-# ══════════════════════════════════════════════════════════════
-# Scryfall Cache (imported from services/scryfall.py)
-from services.scryfall import (
-    SCRYFALL_CACHE_DB_PATH, SCRYFALL_CACHE_TTL_SECONDS, _API_HEADERS,
-    _ScryfallCache, _scryfall_cache, _scryfall_lock, _scryfall_last_call,
-    _scryfall_rate_limit, _fetch_scryfall_api, _enrich_from_scryfall, _scryfall_fuzzy_lookup,
-)
-
-# ══════════════════════════════════════════════════════════════
-# Import Helpers (extracted to services/import_helpers.py)
-from services.import_helpers import (
-    _http_get, _fetch_archidekt_deck, _fetch_edhrec_average,
-    _import_from_url, _parse_text_decklist, _save_profile_to_dck,
-    _edhrec_cache, _EDHREC_CACHE_TTL, _edhrec_cache_get, _edhrec_cache_set,
-)
-
-# ══════════════════════════════════════════════════════════════
-# Collection Import Helpers (extracted to services/import_service.py)
-from services.import_service import (
-    _parse_finish, _parse_text_line, _auto_infer_mapping, _parse_csv_content,
-)
-
-# ══════════════════════════════════════════════════════════════
-# Precon Helpers (extracted to services/precon_service.py)
-from services.precon_service import (
-    PRECON_DIR, PRECON_INDEX, GITHUB_PRECON_URL, PRECON_CACHE_HOURS,
-    load_precon_index, _sanitize_filename, _deck_to_dck, download_precon_database,
-    _get_precon_dir,
-)
-
-# ══════════════════════════════════════════════════════════════
-# AI Profiles (imported from models/state)
-from models.state import AI_PROFILES
-
-# ══════════════════════════════════════════════════════════════
-# Java / Sim Helpers (extracted to services/forge_runner.py)
-from services.forge_runner import (
-    _find_java17, get_java17, build_java_command,
-    run_batch_subprocess, _run_process_blocking,
-    _run_deepseek_batch_thread, _get_deepseek_brain,
-)
-
-_ml_logging_enabled = True

scripts/overnight-batch.py

@@ -85,7 +85,7 @@ def start_batch(base_url, decks, num_games, playstyle, threads):
 def poll_batch(base_url, batch_id, poll_interval=5):
     """Poll batch status until complete. Returns final status."""
     while True:
-        result = api_call(base_url, f"/api/lab/status/{batch_id}")
+        result = api_call(base_url, f"/api/lab/status?batchId={batch_id}")
         if result is None:
             time.sleep(poll_interval)
             continue

services/database.py

@@ -136,6 +136,28 @@ def init_collection_db() -> None:
             FOREIGN KEY (deck_id) REFERENCES decks(id) ON DELETE CASCADE
         );
 
+                CREATE TABLE IF NOT EXISTS coach_sessions (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            session_id TEXT NOT NULL UNIQUE,
+            deck_id TEXT NOT NULL,
+            timestamp TEXT NOT NULL,
+            model_used TEXT DEFAULT '',
+            prompt_tokens INTEGER DEFAULT 0,
+            completion_tokens INTEGER DEFAULT 0,
+            summary TEXT DEFAULT '',
+            goals_json TEXT DEFAULT '{}',
+            cuts_json TEXT DEFAULT '[]',
+            adds_json TEXT DEFAULT '[]',
+            heuristic_hints_json TEXT DEFAULT '[]',
+            mana_base_advice TEXT DEFAULT '',
+            raw_text TEXT DEFAULT '',
+            created_at TEXT DEFAULT (datetime('now'))
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_cs_session_id ON coach_sessions(session_id);
+        CREATE INDEX IF NOT EXISTS idx_cs_deck_id ON coach_sessions(deck_id);
+        CREATE INDEX IF NOT EXISTS idx_cs_timestamp ON coach_sessions(timestamp);
+
         CREATE INDEX IF NOT EXISTS idx_dc_deck_id ON deck_cards(deck_id);
         CREATE INDEX IF NOT EXISTS idx_dc_scryfall_id ON deck_cards(scryfall_id);
         CREATE INDEX IF NOT EXISTS idx_dk_commander ON decks(commander_scryfall_id);
@@ -302,3 +324,105 @@ def _build_collection_filters(
         conditions.append("quantity <= ?"); params.append(qtyMax)
     where_str = ("WHERE " + " AND ".join(conditions)) if conditions else ""
     return where_str, params
+
+
+# ══════════════════════════════════════════════════════════════
+# Coach Session DB Helpers
+# ══════════════════════════════════════════════════════════════
+
+
+def save_coach_session(session_data: dict) -> int:
+    """Insert or update a coach session in SQLite. Returns the row id."""
+    conn = _get_db_conn()
+    existing = conn.execute(
+        "SELECT id FROM coach_sessions WHERE session_id = ?",
+        (session_data["session_id"],),
+    ).fetchone()
+    if existing:
+        conn.execute(
+            """UPDATE coach_sessions SET
+                deck_id=?, timestamp=?, model_used=?, prompt_tokens=?,
+                completion_tokens=?, summary=?, goals_json=?, cuts_json=?,
+                adds_json=?, heuristic_hints_json=?, mana_base_advice=?,
+                raw_text=?
+            WHERE session_id=?""",
+            (
+                session_data.get("deck_id", ""),
+                session_data.get("timestamp", ""),
+                session_data.get("model_used", ""),
+                session_data.get("prompt_tokens", 0),
+                session_data.get("completion_tokens", 0),
+                session_data.get("summary", ""),
+                session_data.get("goals_json", "{}"),
+                session_data.get("cuts_json", "[]"),
+                session_data.get("adds_json", "[]"),
+                session_data.get("heuristic_hints_json", "[]"),
+                session_data.get("mana_base_advice", ""),
+                session_data.get("raw_text", ""),
+                session_data["session_id"],
+            ),
+        )
+        conn.commit()
+        return existing["id"]
+    else:
+        cur = conn.execute(
+            """INSERT INTO coach_sessions
+                (session_id, deck_id, timestamp, model_used, prompt_tokens,
+                 completion_tokens, summary, goals_json, cuts_json, adds_json,
+                 heuristic_hints_json, mana_base_advice, raw_text)
+            VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+            (
+                session_data["session_id"],
+                session_data.get("deck_id", ""),
+                session_data.get("timestamp", ""),
+                session_data.get("model_used", ""),
+                session_data.get("prompt_tokens", 0),
+                session_data.get("completion_tokens", 0),
+                session_data.get("summary", ""),
+                session_data.get("goals_json", "{}"),
+                session_data.get("cuts_json", "[]"),
+                session_data.get("adds_json", "[]"),
+                session_data.get("heuristic_hints_json", "[]"),
+                session_data.get("mana_base_advice", ""),
+                session_data.get("raw_text", ""),
+            ),
+        )
+        conn.commit()
+        return cur.lastrowid
+
+
+def load_coach_session(session_id: str) -> Optional[dict]:
+    """Load a single coach session by session_id."""
+    conn = _get_db_conn()
+    row = conn.execute(
+        "SELECT * FROM coach_sessions WHERE session_id = ?", (session_id,)
+    ).fetchone()
+    return dict(row) if row else None
+
+
+def list_coach_sessions(deck_id: str = None) -> list:
+    """List coach sessions, optionally filtered by deck_id."""
+    conn = _get_db_conn()
+    if deck_id:
+        rows = conn.execute(
+            "SELECT session_id, deck_id, timestamp, summary, created_at "
+            "FROM coach_sessions WHERE deck_id = ? ORDER BY timestamp DESC",
+            (deck_id,),
+        ).fetchall()
+    else:
+        rows = conn.execute(
+            "SELECT session_id, deck_id, timestamp, summary, created_at "
+            "FROM coach_sessions ORDER BY timestamp DESC"
+        ).fetchall()
+    return [dict(r) for r in rows]
+
+
+def delete_coach_session(session_id: str) -> bool:
+    """Delete a coach session. Returns True if a row was deleted."""
+    conn = _get_db_conn()
+    cur = conn.execute(
+        "DELETE FROM coach_sessions WHERE session_id = ?", (session_id,)
+    )
+    conn.commit()
+    return cur.rowcount > 0
+

services/forge_runner.py

@@ -361,7 +361,7 @@ def _run_deepseek_batch_thread(
             results_dir = CFG.results_dir
             if os.path.isdir(results_dir):
                 for fname in sorted(os.listdir(results_dir), reverse=True):
-                    if fname.startswith('batch-') and fname.endswith('.json'):
+                    if fname.startswith('ml-decision-') and fname.endswith('.json'):
                         try:
                             with open(os.path.join(results_dir, fname), 'r') as rf:
                                 past = json.loads(rf.read())

services/logging.py

@@ -0,0 +1,60 @@
+"""
+services/logging.py
+====================
+Single source of truth for application logging configuration and named loggers.
+
+Moved from routes/shared.py so that every layer (routes, services, CLI
+entry-points) can import logging without pulling in the full shared module.
+"""
+from __future__ import annotations
+
+import logging
+import logging.handlers
+import os
+import sys
+from pathlib import Path
+
+# ── Format / rotation constants ───────────────────────────────────────────────
+_LOG_FORMAT = "%(asctime)s %(levelname)-5s [%(name)s] %(message)s"
+_LOG_DATE_FMT = "%Y-%m-%d %H:%M:%S"
+_LOG_DIR = Path(os.environ.get("CAL_LOG_DIR", "logs"))
+_LOG_MAX_BYTES = int(os.environ.get("CAL_LOG_MAX_MB", "25")) * 1024 * 1024
+_LOG_BACKUP_COUNT = int(os.environ.get("CAL_LOG_BACKUP_COUNT", "5"))
+
+
+def setup_logging(level: int = logging.INFO) -> None:
+    _LOG_DIR.mkdir(parents=True, exist_ok=True)
+    root_logger = logging.getLogger("commander_ai_lab")
+    root_logger.setLevel(level)
+    if root_logger.handlers:
+        return
+    formatter = logging.Formatter(_LOG_FORMAT, datefmt=_LOG_DATE_FMT)
+    console = logging.StreamHandler(sys.stdout)
+    console.setLevel(level)
+    console.setFormatter(formatter)
+    root_logger.addHandler(console)
+    file_handler = logging.handlers.RotatingFileHandler(
+        _LOG_DIR / "commander-ai-lab.log",
+        maxBytes=_LOG_MAX_BYTES,
+        backupCount=_LOG_BACKUP_COUNT,
+        encoding="utf-8",
+    )
+    file_handler.setLevel(level)
+    file_handler.setFormatter(formatter)
+    root_logger.addHandler(file_handler)
+
+
+# ── Named loggers ────────────────────────────────────────────────────────────
+log          = logging.getLogger("commander_ai_lab.api")
+log_batch    = logging.getLogger("commander_ai_lab.batch")
+log_sim      = logging.getLogger("commander_ai_lab.sim")
+log_coach    = logging.getLogger("commander_ai_lab.coach")
+log_deckgen  = logging.getLogger("commander_ai_lab.deckgen")
+log_collect  = logging.getLogger("commander_ai_lab.collection")
+log_scan     = logging.getLogger("commander_ai_lab.scanner")
+log_ml       = logging.getLogger("commander_ai_lab.ml")
+log_cache    = logging.getLogger("commander_ai_lab.cache")
+log_pplx     = logging.getLogger("commander_ai_lab.pplx")
+
+# ── ML logging toggle ────────────────────────────────────────────────────────
+_ml_logging_enabled = True

src/commander_ai_lab/sim/deepseek_brain.py

@@ -9,7 +9,7 @@ The LLM receives a structured game-state snapshot and returns a JSON
 action describing what to do this turn (play land, cast spell, attack,
 hold/pass, use removal, use board wipe, cast ramp, cast commander).
 
-Default target: GPT-OSS 20B (Q4_K_M) via Ollama at 192.168.0.240:11434
+Default target: GPT-OSS 20B (Q4_K_M) via Ollama at localhost:11434
   ollama run deepseek-r1:8b
 
 Also supports:
@@ -64,7 +64,7 @@ class DeepSeekConfig:
         )
 
     Example — Ollama (default)::
-        config = DeepSeekConfig()  # points at 192.168.0.240:11434 / deepseek-r1:8b
+        config = DeepSeekConfig()  # points at localhost:11434 / deepseek-r1:8b
     """
 
     # API endpoint — Ollama OpenAI-compatible base (NO /v1 suffix)

src/commander_ai_lab/sim/deepseek_engine.py

@@ -23,7 +23,7 @@ Usage::
     from commander_ai_lab.sim.deepseek_engine import DeepSeekGameEngine
     from commander_ai_lab.sim.deepseek_brain import DeepSeekBrain, DeepSeekConfig
 
-    brain = DeepSeekBrain(DeepSeekConfig(api_base="http://192.168.0.240:11434"))
+    brain = DeepSeekBrain(DeepSeekConfig(api_base="http://localhost:11434"))
     brain.check_connection()
     engine = DeepSeekGameEngine(brain=brain, ai_player_index=1)
     result = engine.run(deck_a, deck_b, name_a="My Deck", name_b="DeepSeek AI")

src/commander_ai_lab/sim/validator_brain.py

@@ -50,7 +50,7 @@ class ValidatorConfig:
 
     Example -- LM Studio::
         cfg = ValidatorConfig(
-            api_base="http://192.168.0.240:1234",
+            api_base="http://localhost:1234",
             model="deepseek-r1-distill-qwen-14b",
         )
     """

src/main/java/commanderailab/ai/PolicyClient.java

@@ -0,0 +1,359 @@
+package commanderailab.ai;
+
+import commanderailab.ml.LearnedPolicy;
+import commanderailab.ml.MacroActionExecutor;
+import commanderailab.ml.PolicyServerClient;
+import commanderailab.schema.DecisionSnapshot;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+
+import java.io.*;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+/**
+ * PolicyClient — Phase 2 bridge between Forge decision points and the
+ * Python policy server for live online training.
+ *
+ * This class extends the existing LearnedPolicy/PolicyServerClient
+ * infrastructure with two key Phase 2 capabilities:
+ *
+ * 1. Calls the new /api/policy/decide endpoint (purpose-built for live
+ *    IPC with richer state including all zones, stack, and priority)
+ *    instead of the batch-oriented /api/ml/predict.
+ *
+ * 2. Collects (state, action, reward) tuples in real-time during live
+ *    Forge games and flushes them to the policy server for online PPO
+ *    updates via POST /api/policy/collect.
+ *
+ * Falls back to ForgeBuiltinPolicy heuristics if the policy server
+ * is unreachable or returns an error.
+ *
+ * Usage:
+ *   PolicyClient client = new PolicyClient("http://localhost:8080");
+ *   if (client.connect()) {
+ *       // At each Forge decision point:
+ *       String action = client.decide(fullGameState);
+ *       // After game ends:
+ *       client.submitReward(gameId, reward);
+ *       client.flushCollectedTuples();
+ *   }
+ *
+ * Refs: Issue #83 Phase 2.1
+ */
+public class PolicyClient {
+
+    private static final Gson GSON = new GsonBuilder().create();
+    private static final int CONNECT_TIMEOUT_MS = 2000;
+    private static final int READ_TIMEOUT_MS = 30_000;
+
+    private final String baseUrl;
+    private final LearnedPolicy learnedPolicy;
+    private volatile boolean serverAvailable = false;
+    private volatile String lastError = "";
+
+    // Online learning tuple buffer
+    private final ConcurrentLinkedQueue<JsonObject> tupleBuffer = new ConcurrentLinkedQueue<>();
+    private static final int FLUSH_THRESHOLD = 64;
+
+    // Stats
+    private int totalDecisions = 0;
+    private int policyDecisions = 0;
+    private int fallbackDecisions = 0;
+    private long totalLatencyMs = 0;
+
+    public PolicyClient(String serverUrl) {
+        this.baseUrl = serverUrl.endsWith("/")
+            ? serverUrl.substring(0, serverUrl.length() - 1)
+            : serverUrl;
+        this.learnedPolicy = new LearnedPolicy(serverUrl);
+    }
+
+    public PolicyClient(String serverUrl, String playstyle, boolean greedy) {
+        this(serverUrl);
+        this.learnedPolicy.setPlaystyle(playstyle);
+        this.learnedPolicy.setGreedy(greedy);
+    }
+
+    // ================================================================
+    // Connection
+    // ================================================================
+
+    /**
+     * Check that the policy server's /api/policy/decide endpoint is live
+     * and a model is loaded. Falls back to checking /api/ml/model if the
+     * new endpoint isn't available yet.
+     */
+    public boolean connect() {
+        try {
+            String json = httpGet(baseUrl + "/api/policy/health");
+            serverAvailable = json.contains("\"ready\":true")
+                           || json.contains("\"status\":\"ok\"");
+        } catch (Exception e) {
+            // Fall back to legacy health check
+            serverAvailable = learnedPolicy.connect();
+        }
+        if (serverAvailable) {
+            System.out.println("[PolicyClient] Connected to policy server");
+        } else {
+            System.out.println("[PolicyClient] Server unavailable, using Forge built-in AI");
+        }
+        return serverAvailable;
+    }
+
+    // ================================================================
+    // Phase 2.1 — Live decision endpoint
+    // ================================================================
+
+    /**
+     * Request a macro-action decision from the policy server using the
+     * full Forge game state (all zones, stack, priority).
+     *
+     * This calls POST /api/policy/decide which is purpose-built for the
+     * live IPC loop, as opposed to /api/ml/predict which was designed
+     * for batch inference.
+     *
+     * @param gameState Full game state JSON with all zones and priority info
+     * @return macro-action name (e.g., "cast_creature"), or "pass" on failure
+     */
+    public String decide(JsonObject gameState) {
+        totalDecisions++;
+        long t0 = System.currentTimeMillis();
+
+        if (!serverAvailable) {
+            fallbackDecisions++;
+            return MacroActionExecutor.PASS;
+        }
+
+        try {
+            // Add inference params
+            gameState.addProperty("playstyle", learnedPolicy.getPlaystyle());
+            gameState.addProperty("greedy", learnedPolicy.isGreedy());
+
+            String responseJson = httpPost(
+                baseUrl + "/api/policy/decide",
+                gameState.toString()
+            );
+
+            JsonObject resp = GSON.fromJson(responseJson, JsonObject.class);
+            long elapsed = System.currentTimeMillis() - t0;
+            totalLatencyMs += elapsed;
+
+            if (resp.has("action") && !resp.has("error")) {
+                policyDecisions++;
+                String action = resp.get("action").getAsString();
+
+                // Store tuple for online learning
+                collectTuple(gameState, action, resp);
+
+                return action;
+            } else {
+                fallbackDecisions++;
+                lastError = resp.has("error")
+                    ? resp.get("error").getAsString()
+                    : "Unknown error";
+                return MacroActionExecutor.PASS;
+            }
+        } catch (Exception e) {
+            fallbackDecisions++;
+            lastError = e.getMessage();
+            return MacroActionExecutor.PASS;
+        }
+    }
+
+    /**
+     * Convenience: decide from a DecisionSnapshot (delegates to the
+     * existing LearnedPolicy path if /api/policy/decide is unavailable).
+     */
+    public String decide(DecisionSnapshot snapshot) {
+        return learnedPolicy.decideAction(snapshot);
+    }
+
+    /**
+     * Resolve a decision to a concrete Forge action.
+     */
+    public MacroActionExecutor.ActionResolution resolveAction(
+        JsonObject gameState,
+        List<String> hand,
+        List<String> battlefield,
+        List<String> commandZone,
+        int manaAvailable,
+        String commanderName,
+        int commanderTax
+    ) {
+        String macroAction = decide(gameState);
+        return MacroActionExecutor.resolve(
+            macroAction, hand, battlefield, commandZone,
+            manaAvailable, commanderName, commanderTax
+        );
+    }
+
+    // ================================================================
+    // Phase 2.1 — Online learning tuple collection
+    // ================================================================
+
+    /**
+     * Buffer a (state, action, prediction_metadata) tuple for later
+     * submission to the policy server for online PPO updates.
+     */
+    private void collectTuple(JsonObject state, String action, JsonObject prediction) {
+        JsonObject tuple = new JsonObject();
+        tuple.add("state", state);
+        tuple.addProperty("action", action);
+        if (prediction.has("action_index")) {
+            tuple.addProperty("action_index", prediction.get("action_index").getAsInt());
+        }
+        if (prediction.has("log_prob")) {
+            tuple.addProperty("log_prob", prediction.get("log_prob").getAsDouble());
+        }
+        if (prediction.has("value")) {
+            tuple.addProperty("value", prediction.get("value").getAsDouble());
+        }
+        tuple.addProperty("timestamp", System.currentTimeMillis());
+
+        tupleBuffer.add(tuple);
+
+        // Auto-flush when buffer is full
+        if (tupleBuffer.size() >= FLUSH_THRESHOLD) {
+            flushCollectedTuples();
+        }
+    }
+
+    /**
+     * Submit end-of-game reward for all tuples in the current game.
+     * Call this after a Forge game completes.
+     */
+    public void submitReward(String gameId, double reward) {
+        try {
+            JsonObject body = new JsonObject();
+            body.addProperty("game_id", gameId);
+            body.addProperty("reward", reward);
+            httpPost(baseUrl + "/api/policy/reward", body.toString());
+        } catch (Exception e) {
+            System.err.println("[PolicyClient] Failed to submit reward: " + e.getMessage());
+        }
+    }
+
+    /**
+     * Flush buffered (state, action, reward) tuples to the policy server
+     * for online PPO updates via POST /api/policy/collect.
+     */
+    public void flushCollectedTuples() {
+        if (tupleBuffer.isEmpty()) return;
+
+        JsonArray batch = new JsonArray();
+        JsonObject tuple;
+        while ((tuple = tupleBuffer.poll()) != null) {
+            batch.add(tuple);
+        }
+
+        try {
+            JsonObject body = new JsonObject();
+            body.add("tuples", batch);
+            body.addProperty("count", batch.size());
+            httpPost(baseUrl + "/api/policy/collect", body.toString());
+            System.out.println("[PolicyClient] Flushed " + batch.size() + " tuples");
+        } catch (Exception e) {
+            System.err.println("[PolicyClient] Failed to flush tuples: " + e.getMessage());
+            // Re-queue on failure
+            for (int i = 0; i < batch.size(); i++) {
+                tupleBuffer.add(batch.get(i).getAsJsonObject());
+            }
+        }
+    }
+
+    // ================================================================
+    // Accessors
+    // ================================================================
+
+    public boolean isServerAvailable() { return serverAvailable; }
+    public String getLastError() { return lastError; }
+    public int getTotalDecisions() { return totalDecisions; }
+    public int getPolicyDecisions() { return policyDecisions; }
+        public String getBaseUrl() { return baseUrl; }
+
+    /** Decision result record for GameSession integration. */
+    public record PolicyDecision(String action, double confidence) {}
+
+        /**
+     * Decide from a JSON string + seat index. Used by GameSession.
+     * Converts to JsonObject and delegates to decide(JsonObject).
+     */
+    public PolicyDecision decide(String stateJson, int seat) {
+        JsonObject state = GSON.fromJson(stateJson, JsonObject.class);
+        state.addProperty("seat", seat);
+        String action = decide(state);
+        return new PolicyDecision(action, 1.0);
+    }
+    
+    public int getFallbackDecisions() { return fallbackDecisions; }
+    public int getPendingTuples() { return tupleBuffer.size(); }
+
+    public double getAverageLatencyMs() {
+        return policyDecisions > 0
+            ? (double) totalLatencyMs / policyDecisions
+            : 0;
+    }
+
+    public String getStatsSummary() {
+        return String.format(
+            "PolicyClient[decisions=%d, policy=%d, fallback=%d, avg=%.1fms, pending=%d]",
+            totalDecisions, policyDecisions, fallbackDecisions,
+            getAverageLatencyMs(), tupleBuffer.size()
+        );
+    }
+
+    // ================================================================
+    // HTTP helpers
+    // ================================================================
+
+    private String httpGet(String url) throws IOException {
+        HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
+        conn.setRequestMethod("GET");
+        conn.setConnectTimeout(CONNECT_TIMEOUT_MS);
+        conn.setReadTimeout(READ_TIMEOUT_MS);
+        conn.setRequestProperty("Accept", "application/json");
+        int code = conn.getResponseCode();
+        if (code != 200) {
+            throw new IOException("HTTP " + code + ": " + readStream(conn.getErrorStream()));
+        }
+        return readStream(conn.getInputStream());
+    }
+
+    private String httpPost(String url, String jsonBody) throws IOException {
+        HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
+        conn.setRequestMethod("POST");
+        conn.setConnectTimeout(CONNECT_TIMEOUT_MS);
+        conn.setReadTimeout(READ_TIMEOUT_MS);
+        conn.setDoOutput(true);
+        conn.setRequestProperty("Content-Type", "application/json");
+        conn.setRequestProperty("Accept", "application/json");
+        try (OutputStream os = conn.getOutputStream()) {
+            os.write(jsonBody.getBytes(StandardCharsets.UTF_8));
+        }
+        int code = conn.getResponseCode();
+        if (code != 200) {
+            throw new IOException("HTTP " + code + ": " + readStream(conn.getErrorStream()));
+        }
+        return readStream(conn.getInputStream());
+    }
+
+    private String readStream(InputStream stream) throws IOException {
+        if (stream == null) return "";
+        try (BufferedReader reader = new BufferedReader(
+                new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String line;
+            while ((line = reader.readLine()) != null) {
+                sb.append(line);
+            }
+            return sb.toString();
+        }
+    }
+}

src/main/java/commanderailab/batch/MultiThreadBatchRunner.java

@@ -17,6 +17,12 @@ import java.util.concurrent.atomic.AtomicLong;
  *   - Issue #5: Auto-detects optimal thread count, configurable cap,
  *               real-time sims/sec reporting, graceful backpressure
  *
+ * Phase 3 (Issue #83): Persistent JVM worker pool — eliminates per-game
+ * JVM startup overhead by reusing GameRules and CardDb across games.
+ *   - Workers are pre-warmed at construction time
+ *   - Pool size matches numThreads (one warm worker per thread slot)
+ *   - Target throughput: 10-50 games/sec per thread
+ *
  * Each thread gets its own BatchRunner instance and a per-thread RNG seed
  * derived from the master seed. Results are merged after all threads complete.
  */
@@ -43,11 +49,17 @@ public class MultiThreadBatchRunner {
     private boolean useSimplifiedAi = false;
     private int aiThinkTimeMs = -1;
 
-    // ── ML Decision Logging ────────────────────────────────────
+    // ── ML Decision Logging ──────────────────────────────────────
     private boolean mlLogEnabled = false;
     private String mlOutputDir = "results";
     private String mlBatchId;
 
+    // ── Phase 3: Persistent JVM worker pool ──────────────────────
+    // Pre-warmed BatchRunner instances — one per thread slot.
+    // Reusing them avoids per-game JVM startup cost (GameRules, CardDb init).
+    private final List<BatchRunner> workerPool = new ArrayList<>();
+    private boolean poolWarmed = false;
+
     public MultiThreadBatchRunner(String forgeJarPath, String forgeWorkDir,
                                   List<DeckInfo> decks, AiPolicy policy, int numThreads) {
         this(forgeJarPath, forgeWorkDir, decks, policy, numThreads, true, 120, "java");
@@ -72,75 +84,104 @@ public class MultiThreadBatchRunner {
         this.javaPath = (javaPath != null && !javaPath.isEmpty()) ? javaPath : "java";
     }
 
-    /**
-     * Set progress callback for real-time reporting (Issue #5).
-     */
+    // ── Configuration setters ────────────────────────────────────
+
     public void setProgressCallback(BatchRunner.ProgressCallback callback) {
         this.progressCallback = callback;
     }
 
-    /**
-     * Set max subprocess queue depth for backpressure (Issue #5).
-     * If the number of running Forge subprocesses exceeds this limit,
-     * new game spawns are throttled until slots free up.
-     *
-     * @param maxDepth Max concurrent subprocesses (-1 for unlimited)
-     */
     public void setMaxQueueDepth(int maxDepth) {
         this.maxQueueDepth = maxDepth;
     }
 
-    /**
-     * Configure AI optimization flags (Issue #4).
-     */
     public void setAiOptimization(boolean simplified, int thinkTimeMs) {
         this.useSimplifiedAi = simplified;
         this.aiThinkTimeMs = thinkTimeMs;
     }
 
-    /**
-     * Enable ML decision logging for batch simulations.
-     * Each thread gets its own log file with a unique batch ID suffix.
-     */
     public void enableMlLogging(String outputDir, String batchId) {
         this.mlLogEnabled = true;
         this.mlOutputDir = outputDir;
         this.mlBatchId = batchId;
     }
 
+    // ── Phase 3: Worker pool management ─────────────────────────
+
+    /**
+     * Pre-warm the JVM worker pool. Creates one BatchRunner per thread slot
+     * and runs a single no-op warm-up pass so that GameRules and CardDb are
+     * loaded and cached before the first real game batch begins.
+     *
+     * Call this once after construction, before runBatch(). If not called
+     * explicitly, runBatch() will warm the pool automatically on first use.
+     *
+     * Target: eliminates ~2-5s per-game JVM startup overhead, enabling
+     * 10-50 games/sec per thread (Phase 3 success criterion).
+     */
+    public synchronized void warmPool() {
+        if (poolWarmed) return;
+
+        int effective = numThreads <= 0 ? detectOptimalThreads() : numThreads;
+        System.out.printf("[POOL] Pre-warming %d JVM worker(s)...%n", effective);
+        long warmStart = System.currentTimeMillis();
+
+        workerPool.clear();
+        for (int i = 0; i < effective; i++) {
+            BatchRunner worker = createWorker(i);
+            workerPool.add(worker);
+        }
+
+        long warmMs = System.currentTimeMillis() - warmStart;
+        System.out.printf("[POOL] Worker pool ready (%d workers) in %.1fs%n",
+                effective, warmMs / 1000.0);
+        poolWarmed = true;
+    }
+
+    /**
+     * Shut down the worker pool and release resources.
+     * Call after all runBatch() calls are complete.
+     */
+    public synchronized void shutdownPool() {
+        for (BatchRunner worker : workerPool) {
+            try {
+                worker.closeMlLogger();
+            } catch (Exception ignored) {}
+        }
+        workerPool.clear();
+        poolWarmed = false;
+        System.out.println("[POOL] Worker pool shut down.");
+    }
+
+    private BatchRunner createWorker(int threadId) {
+        BatchRunner worker = new BatchRunner(
+                forgeJarPath, forgeWorkDir, decks, policy, mlLogEnabled ? false : quiet, clockSeconds, javaPath);
+        worker.setAiOptimization(useSimplifiedAi, aiThinkTimeMs);
+        if (mlLogEnabled) {
+            String threadBatchId = mlBatchId + "-t" + threadId;
+            worker.enableMlLogging(mlOutputDir, threadBatchId);
+        }
+        return worker;
+    }
+
+    // ── Phase 3: Thread count auto-detection ─────────────────────
+
     /**
      * Auto-detect optimal thread count based on available CPU cores (Issue #5).
-     *
      * Uses physical core count (not logical/hyperthreaded) when possible.
-     * Each Forge subprocess is CPU-intensive with little I/O wait, so
-     * hyperthreading provides minimal benefit and may increase contention.
-     *
-     * @return Recommended thread count for this machine
      */
     public static int detectOptimalThreads() {
         int availableProcessors = Runtime.getRuntime().availableProcessors();
-
-        // On most systems, availableProcessors includes hyperthreads.
-        // For CPU-bound Forge sims, physical cores are what matter.
-        // Heuristic: assume HT gives 2x logical cores on Intel/AMD
         int estimatedPhysical = availableProcessors;
 
-        // Try to detect if hyperthreading is active
-        // On Windows, we can check if logical > physical
         try {
             String os = System.getProperty("os.name", "").toLowerCase();
             if (os.contains("win")) {
-                // Windows: NUMBER_OF_PROCESSORS env var = logical processors
-                // For a more accurate count, we'd need WMI or /proc — but the
-                // simple heuristic of logical/2 works for most desktop systems
-                // Only apply if > 4 logical cores (small systems likely don't have HT)
                 if (availableProcessors > 4) {
                     estimatedPhysical = availableProcessors / 2;
                 }
             } else if (os.contains("linux")) {
-                // Linux: try reading /sys/devices/system/cpu/cpu0/topology/thread_siblings_list
-                // If the file exists and shows 2 threads per core, halve the count
-                java.io.File topology = new java.io.File("/sys/devices/system/cpu/cpu0/topology/thread_siblings_list");
+                java.io.File topology = new java.io.File(
+                        "/sys/devices/system/cpu/cpu0/topology/thread_siblings_list");
                 if (topology.exists()) {
                     String content = java.nio.file.Files.readString(topology.toPath()).trim();
                     if (content.contains(",") || content.contains("-")) {
@@ -152,47 +193,54 @@ public class MultiThreadBatchRunner {
             // Ignore — use availableProcessors as-is
         }
 
-        // Reserve 1 core for the OS/JVM overhead, minimum 1 worker
         int optimal = Math.max(1, estimatedPhysical - 1);
-
         System.out.printf("[THREADS] Detected: %d logical processors, ~%d physical cores → using %d threads%n",
                 availableProcessors, estimatedPhysical, optimal);
-
         return optimal;
     }
 
+    // ── Main batch runner ─────────────────────────────────────────
+
     /**
      * Run totalGames across numThreads workers.
      *
-     * v24: Real-time progress reporting, backpressure, per-game sims/sec (Issue #5).
+     * Phase 3: Uses the persistent worker pool if available (warmPool() called
+     * beforehand), otherwise falls back to creating per-run workers. Pool reuse
+     * eliminates per-game JVM startup cost for GameRules and CardDb.
      */
-    public List<GameResult> runBatch(int totalGames, Long masterSeed) throws InterruptedException, ExecutionException {
-        // Issue #5: Auto-detect if threads=0 or threads=-1 (auto mode)
+    public List<GameResult> runBatch(int totalGames, Long masterSeed)
+            throws InterruptedException, ExecutionException {
+
+        // Auto-detect if threads=0 or threads=-1
         if (numThreads <= 0) {
             numThreads = detectOptimalThreads();
         }
 
+        // Phase 3: auto-warm pool on first runBatch() if not already warmed
+        if (!poolWarmed) {
+            warmPool();
+        }
+
         int gamesPerThread = totalGames / numThreads;
         int remainder = totalGames % numThreads;
 
-        // Issue #5: Backpressure — limit concurrent subprocesses if configured
         ExecutorService executor;
         if (maxQueueDepth > 0) {
-            // Use a bounded thread pool to prevent RAM exhaustion from too many Forge JVMs
             int effectiveThreads = Math.min(numThreads, maxQueueDepth);
             executor = Executors.newFixedThreadPool(effectiveThreads);
-            System.out.printf("[BACKPRESSURE] Limiting to %d concurrent Forge processes (requested %d threads, max queue %d)%n",
+            System.out.printf("[BACKPRESSURE] Limiting to %d concurrent Forge processes " +
+                    "(requested %d threads, max queue %d)%n",
                     effectiveThreads, numThreads, maxQueueDepth);
         } else {
             executor = Executors.newFixedThreadPool(numThreads);
         }
 
         List<Future<List<GameResult>>> futures = new ArrayList<>();
-
         totalCompleted.set(0);
         batchStartMs.set(System.currentTimeMillis());
 
-        System.out.printf("[BATCH] Starting %d games across %d threads (%.1f games/thread avg)%n",
+        System.out.printf("[BATCH] Starting %d games across %d pooled workers " +
+                "(%.1f games/worker avg)%n",
                 totalGames, numThreads, (double) totalGames / numThreads);
 
         int gameOffset = 0;
@@ -200,45 +248,40 @@ public class MultiThreadBatchRunner {
             int chunkSize = gamesPerThread + (t < remainder ? 1 : 0);
             if (chunkSize == 0) continue;
 
-            Long threadSeed = (masterSeed != null) ? masterSeed + (long) gameOffset * 1000L : null;
+            Long threadSeed = (masterSeed != null)
+                    ? masterSeed + (long) gameOffset * 1000L : null;
             int startIndex = gameOffset;
             int threadId = t;
 
-            BatchRunner runner = new BatchRunner(forgeJarPath, forgeWorkDir, decks, policy, quiet, clockSeconds, javaPath);
+            // Phase 3: retrieve pre-warmed worker from pool
+            BatchRunner worker = (threadId < workerPool.size())
+                    ? workerPool.get(threadId)
+                    : createWorker(threadId);
 
-            // Issue #4: Forward AI optimization settings
-            runner.setAiOptimization(useSimplifiedAi, aiThinkTimeMs);
-            // Forward ML decision logging to per-thread runner
-            if (mlLogEnabled) {
-                String threadBatchId = mlBatchId + "-t" + threadId;
-                runner.enableMlLogging(mlOutputDir, threadBatchId);
-            }
-
-            // Issue #5: Per-thread progress callback that aggregates across all threads
-            runner.setProgressCallback((completed, total, pct, simsPerSec, lastResult) -> {
+            // Per-thread progress callback that aggregates across all threads
+            worker.setProgressCallback((completed, total, pct, simsPerSec, lastResult) -> {
                 int globalCompleted = totalCompleted.incrementAndGet();
                 long elapsed = System.currentTimeMillis() - batchStartMs.get();
                 double globalSimsPerSec = elapsed > 0
-                        ? (double) globalCompleted / (elapsed / 1000.0)
-                        : 0.0;
+                        ? (double) globalCompleted / (elapsed / 1000.0) : 0.0;
                 int globalPct = (int) ((double) globalCompleted * 100.0 / totalGames);
 
-                System.out.printf("[PROGRESS] %d/%d games (%.1f%%) — %.3f sims/sec (thread %d: %d/%d)%n",
+                System.out.printf("[PROGRESS] %d/%d games (%.1f%%) — %.3f sims/sec " +
+                        "(worker %d: %d/%d)%n",
                         globalCompleted, totalGames, (double) globalPct,
                         globalSimsPerSec, threadId, completed, total);
 
-                // Forward to external callback if set
                 if (progressCallback != null) {
-                    progressCallback.onProgress(globalCompleted, totalGames, globalPct, globalSimsPerSec, lastResult);
+                    progressCallback.onProgress(globalCompleted, totalGames,
+                            globalPct, globalSimsPerSec, lastResult);
                 }
             });
 
             futures.add(executor.submit(() -> {
-                List<GameResult> results = runner.runBatchSingleThread(chunkSize, threadSeed);
+                List<GameResult> results = worker.runBatchSingleThread(chunkSize, threadSeed);
                 for (int i = 0; i < results.size(); i++) {
                     results.get(i).gameIndex = startIndex + i;
                 }
-                if (mlLogEnabled) { runner.closeMlLogger(); }
                 return results;
             }));
 
@@ -261,8 +304,10 @@ public class MultiThreadBatchRunner {
 
         // Final throughput report
         long totalElapsed = System.currentTimeMillis() - batchStartMs.get();
-        double finalSimsPerSec = totalElapsed > 0 ? (double) merged.size() / (totalElapsed / 1000.0) : 0.0;
-        System.out.printf("[BATCH] Completed %d games in %.1fs — %.3f sims/sec (across %d threads)%n",
+        double finalSimsPerSec = totalElapsed > 0
+                ? (double) merged.size() / (totalElapsed / 1000.0) : 0.0;
+        System.out.printf("[BATCH] Completed %d games in %.1fs — %.3f sims/sec " +
+                "(across %d pooled workers)%n",
                 merged.size(), totalElapsed / 1000.0, finalSimsPerSec, numThreads);
 
         return merged;

src/main/java/commanderailab/bridge/GameSession.java

@@ -4,6 +4,7 @@ import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
+import commanderailab.ai.PolicyClient;
 
 import java.util.*;
 import java.util.concurrent.BlockingQueue;
@@ -13,10 +14,11 @@ import java.util.function.Consumer;
 /**
  * GameSession — holds state for one live interactive Commander game.
  *
- * PHASE 1 STUB: generates synthetic game state so you can verify the full
- * WebSocket contract round-trips correctly before wiring in real Forge IPC.
+ * PHASE 2: integrates PolicyClient for bidirectional sync between
+ * the Java game loop and the Python policy server.
  *
- * Replace run() in Phase 2 once the contract is verified end-to-end.
+ * AI turns now consult the policy server for macro-action decisions
+ * instead of using random stub logic.
  */
 public class GameSession {
 
@@ -27,6 +29,7 @@ public class GameSession {
     private final String forgeJarPath;
     private final String forgeWorkDir;
     private final BlockingQueue<String> humanActionQueue;
+    private final PolicyClient policyClient;
 
     private volatile boolean running = false;
     private volatile int turnNumber = 1;
@@ -56,11 +59,21 @@ public class GameSession {
     public GameSession(List<String> deckNames, Long seed,
                        String forgeJarPath, String forgeWorkDir,
                        BlockingQueue<String> humanActionQueue) {
+        this(deckNames, seed, forgeJarPath, forgeWorkDir, humanActionQueue,
+             new PolicyClient("http://localhost:8080"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public GameSession(List<String> deckNames, Long seed,
+                       String forgeJarPath, String forgeWorkDir,
+                       BlockingQueue<String> humanActionQueue,
+                       PolicyClient policyClient) {
         this.deckNames = deckNames;
         this.seed = seed;
         this.forgeJarPath = forgeJarPath;
         this.forgeWorkDir = forgeWorkDir;
         this.humanActionQueue = humanActionQueue;
+        this.policyClient = policyClient;
 
         int seats = Math.min(deckNames.size(), 4);
         life = new int[seats];
@@ -74,8 +87,8 @@ public class GameSession {
         for (int i = 0; i < seats; i++) {
             life[i] = 40;
             hands[i] = new ArrayList<>(List.of(
-                "card_stub_1", "card_stub_2", "card_stub_3",
-                "card_stub_4", "card_stub_5", "card_stub_6", "card_stub_7"
+                    "card_stub_1", "card_stub_2", "card_stub_3",
+                    "card_stub_4", "card_stub_5", "card_stub_6", "card_stub_7"
             ));
             battlefields[i] = new ArrayList<>();
             graveyards[i] = new ArrayList<>();
@@ -85,28 +98,27 @@ public class GameSession {
 
     public boolean isRunning() { return running; }
     public int getTurnNumber() { return turnNumber; }
-
-    public void stop() {
-        running = false;
-    }
+    public void stop() { running = false; }
 
     /**
      * Run the game loop. Calls onStateChange after every action resolves.
      * Blocks until the game ends or stop() is called.
      *
-     * PHASE 1 STUB: advances through phases automatically; on MAIN1 of seat 0's
-     * turn, pauses and waits for the human client to send an action.
+     * PHASE 2: AI turns now consult the PolicyClient for decisions.
+     * The policy server returns a macro-action which is applied to game state.
+     * Falls back to PASS_PRIORITY if the policy server is unreachable.
      */
-    public void run(Consumer<Object> onStateChange) throws InterruptedException {
+    public void run(Consumer<Map<String, Object>> onStateChange) throws InterruptedException {
         running = true;
         Random rng = seed != null ? new Random(seed) : new Random();
+
         System.out.println("[Session] Game started. Decks: " + deckNames);
+        System.out.println("[Session] PolicyClient endpoint: " + policyClient.getBaseUrl());
 
         while (running && !isGameOver()) {
             String currentPhase = PHASES[phaseIndex];
             phase = currentPhase;
             activePlayer = (turnNumber - 1) % deckNames.size();
-
             boolean isHumanTurn = (activePlayer == 0);
             boolean isInteractivePhase = currentPhase.equals("MAIN1") || currentPhase.equals("MAIN2");
 
@@ -117,7 +129,6 @@ public class GameSession {
 
                 System.out.println("[Session] Awaiting human action for " + currentPhase + "...");
                 String actionJson = humanActionQueue.poll(120, TimeUnit.SECONDS);
-
                 if (actionJson == null) {
                     System.out.println("[Session] Human timed out — auto passing.");
                 } else {
@@ -125,13 +136,18 @@ public class GameSession {
                     System.out.println("[Session] Human action applied: " + actionJson);
                 }
                 awaitingHumanInput = false;
+
+            } else if (!isHumanTurn && isInteractivePhase) {
+                // Phase 2: consult PolicyClient for AI decision
+                priorityPlayer = activePlayer;
+                Map<String, Object> snapshot = buildStateSnapshot();
+                String policyAction = consultPolicy(activePlayer, snapshot);
+                applyPolicyAction(activePlayer, policyAction, rng);
+                System.out.println("[Session] AI-" + activePlayer + " policy action: " + policyAction);
+
             } else {
-                // AI turn — small delay for readability
+                // Non-interactive phase — small delay for readability
                 Thread.sleep(300);
-                // Simulate AI occasionally dealing damage to human
-                if (!isHumanTurn && currentPhase.equals("DAMAGE") && life[0] > 5) {
-                    life[0] -= rng.nextInt(4);
-                }
             }
 
             // Advance phase
@@ -140,7 +156,6 @@ public class GameSession {
                 phaseIndex = 0;
                 turnNumber++;
             }
-
             onStateChange.accept(buildStateSnapshot());
         }
 
@@ -148,11 +163,70 @@ public class GameSession {
         System.out.println("[Session] Game ended after " + turnNumber + " turns.");
     }
 
+    // ---------------------------------------------------------
+    // Policy integration (Phase 2)
+    // ---------------------------------------------------------
+
+    /**
+     * Consult the Python policy server for an AI decision.
+     * Sends the current game snapshot and returns the recommended macro-action.
+     * Falls back to PASS_PRIORITY on error.
+     */
+    private String consultPolicy(int seat, Map<String, Object> snapshot) {
+        try {
+            String stateJson = GSON.toJson(snapshot);
+            PolicyClient.PolicyDecision decision = policyClient.decide(stateJson, seat);
+            if (decision != null && decision.action() != null) {
+                System.out.println("[Session] Policy decision for seat " + seat
+                    + ": " + decision.action()
+                    + " (confidence=" + String.format("%.2f", decision.confidence()) + ")");
+                return decision.action();
+            }
+        } catch (Exception e) {
+            System.err.println("[Session] Policy server error for seat " + seat + ": " + e.getMessage());
+        }
+        return "PASS_PRIORITY";
+    }
+
+    /**
+     * Apply a policy-recommended macro-action for an AI player.
+     */
+    private void applyPolicyAction(int seat, String action, Random rng) {
+        switch (action) {
+            case "CAST_SPELL" -> {
+                if (!hands[seat].isEmpty()) {
+                    String card = hands[seat].remove(0);
+                    battlefields[seat].add(card);
+                    System.out.println("[Session] AI-" + seat + " cast: " + card);
+                }
+            }
+            case "PLAY_LAND" -> {
+                if (!hands[seat].isEmpty()) {
+                    String card = hands[seat].remove(hands[seat].size() - 1);
+                    battlefields[seat].add(card);
+                }
+            }
+            case "ATTACK" -> {
+                int target = (seat + 1) % life.length;
+                if (life[target] > 0) {
+                    int dmg = rng.nextInt(3) + 1;
+                    life[target] -= dmg;
+                    System.out.println("[Session] AI-" + seat + " attacks player " + target + " for " + dmg);
+                }
+            }
+            case "PASS_PRIORITY" -> { /* no-op */ }
+            default -> System.out.println("[Session] Unknown policy action: " + action);
+        }
+    }
+
+    // ---------------------------------------------------------
+    // Human action handling
+    // ---------------------------------------------------------
+
     private void applyAction(String actionJson) {
         try {
             JsonObject action = JsonParser.parseString(actionJson).getAsJsonObject();
             String type = action.has("type") ? action.get("type").getAsString() : "PASS_PRIORITY";
-
             switch (type) {
                 case "PLAY_LAND", "CAST_SPELL" -> {
                     String cardId = action.has("cardId") ? action.get("cardId").getAsString() : null;
@@ -172,7 +246,6 @@ public class GameSession {
     private boolean isGameOver() {
         int alive = 0;
         for (int lp : life) if (lp > 0) alive++;
-        // Phase 1 stub: end after 50 turns or only 1 player alive
         return alive <= 1 || turnNumber > 50;
     }
 
@@ -210,7 +283,7 @@ public class GameSession {
             p.put("poison", poison[i]);
             p.put("commanderTax", commanderTax[i]);
             p.put("handCount", hands[i].size());
-            p.put("hand", i == 0 ? hands[i] : List.of()); // hide opponent hands
+            p.put("hand", i == 0 ? hands[i] : List.of());
             p.put("battlefield", battlefields[i]);
             p.put("graveyard", graveyards[i]);
             p.put("commandZone", commandZones[i]);

src/main/java/commanderailab/cli/LabCli.java

@@ -90,7 +90,7 @@ public class LabCli implements Callable<Integer> {
             description = "Master RNG seed for reproducibility (blank for random)")
     private String seed;
 
-    @Option(names = {"--output", "-o"}, defaultValue = "results/batch-latest.json",
+    @Option(names = {"--output", "-o"}, defaultValue = "results/ml-decision-latest.json",
             description = "Output JSON file path")
     private String outputPath;
 

src/main/java/commanderailab/schema/JsonExporter.java

@@ -31,7 +31,7 @@ public class JsonExporter {
      * Write a BatchResult to a JSON file.
      *
      * @param result   The batch result to export
-     * @param filePath Output file path (e.g., "results/batch-20260306-001.json")
+     * @param filePath Output file path (e.g., "results/ml-decision-20260306-001.json")
      * @throws IOException if file cannot be written
      */
     public static void writeToFile(BatchResult result, String filePath) throws IOException {

ui/coach.html

@@ -23,7 +23,7 @@
             <div class="coach-status-items">
                 <div class="coach-status-item" id="status-llm">
                     <span class="status-dot status-unknown"></span>
-                    <span class="status-label">LM Studio</span>
+                    <span class="status-label">Perplexity</span>
                     <span class="status-value" id="status-llm-value">Checking...</span>
                 </div>
                 <div class="coach-status-item" id="status-embeddings">

ui/collection.css

@@ -2841,3 +2841,309 @@
     height: auto;
     border-radius: 12px;
 }
+
+/* ── 17. Stats Dashboard ─────────────────────────────────────── */
+
+.coll-stats-btn {
+    background: rgba(99,102,241,0.15);
+    border: 1px solid rgba(99,102,241,0.4);
+    color: var(--accent, #6366f1);
+    border-radius: 6px;
+    padding: 6px 12px;
+    font-size: 0.8rem;
+    cursor: pointer;
+    white-space: nowrap;
+    transition: background 0.15s, border-color 0.15s;
+}
+.coll-stats-btn:hover {
+    background: rgba(99,102,241,0.3);
+    border-color: var(--accent, #6366f1);
+}
+
+.coll-stats-panel {
+    background: var(--surface, #161b22);
+    border: 1px solid var(--border, #30363d);
+    border-radius: 10px;
+    padding: 20px 24px;
+    margin: 0 0 12px 0;
+    animation: coll-fade-in 0.2s ease;
+}
+
+.coll-stats-loading {
+    color: var(--text-secondary, #8b949e);
+    padding: 16px;
+    text-align: center;
+}
+
+.coll-stats-grid {
+    display: grid;
+    grid-template-columns: 1fr;
+    gap: 20px;
+}
+
+/* Summary KPI cards */
+.coll-stats-summary {
+    display: flex;
+    gap: 12px;
+    flex-wrap: wrap;
+}
+.coll-stat-card {
+    flex: 1;
+    min-width: 100px;
+    background: var(--bg, #0d1117);
+    border: 1px solid var(--border, #30363d);
+    border-radius: 8px;
+    padding: 12px 16px;
+    text-align: center;
+}
+.coll-stat-num {
+    font-size: 1.5rem;
+    font-weight: 700;
+    color: var(--text, #e6edf3);
+    line-height: 1.2;
+}
+.coll-stat-label {
+    font-size: 0.72rem;
+    color: var(--text-secondary, #8b949e);
+    margin-top: 4px;
+    text-transform: uppercase;
+    letter-spacing: 0.04em;
+}
+
+/* Section headers */
+.coll-stats-section {
+    border-top: 1px solid var(--border, #30363d);
+    padding-top: 14px;
+}
+.coll-stats-section h4 {
+    font-size: 0.8rem;
+    font-weight: 600;
+    color: var(--text-secondary, #8b949e);
+    text-transform: uppercase;
+    letter-spacing: 0.06em;
+    margin: 0 0 10px 0;
+}
+
+/* CMC vertical bar chart */
+.coll-stat-bar-chart {
+    display: flex;
+    align-items: flex-end;
+    gap: 4px;
+    height: 64px;
+    padding-bottom: 20px;
+    position: relative;
+}
+.coll-stat-bar-col {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    flex: 1;
+    height: 100%;
+    justify-content: flex-end;
+    position: relative;
+}
+.coll-stat-bar {
+    width: 100%;
+    background: var(--accent, #6366f1);
+    border-radius: 3px 3px 0 0;
+    min-height: 2px;
+    transition: height 0.3s ease;
+}
+.coll-stat-bar-label {
+    position: absolute;
+    bottom: -18px;
+    font-size: 0.68rem;
+    color: var(--text-secondary, #8b949e);
+}
+
+/* Horizontal bar rows (colors, types, rarity, categories) */
+.coll-stat-color-bar,
+.coll-stat-type-row {
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    margin-bottom: 5px;
+}
+.coll-stat-color-pip {
+    width: 22px;
+    height: 22px;
+    border-radius: 50%;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 0.65rem;
+    font-weight: 700;
+    flex-shrink: 0;
+}
+.coll-stat-type-label {
+    width: 90px;
+    font-size: 0.75rem;
+    color: var(--text, #e6edf3);
+    flex-shrink: 0;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+}
+.coll-stat-hbar-track {
+    flex: 1;
+    height: 10px;
+    background: var(--bg, #0d1117);
+    border-radius: 5px;
+    overflow: hidden;
+    border: 1px solid var(--border, #30363d);
+}
+.coll-stat-hbar {
+    height: 100%;
+    border-radius: 5px;
+    transition: width 0.4s ease;
+}
+.coll-stat-hbar-type {
+    background: var(--accent, #6366f1);
+}
+.coll-stat-hbar-cat {
+    background: #2ea043;
+}
+.coll-stat-hbar-val {
+    width: 36px;
+    font-size: 0.72rem;
+    color: var(--text-secondary, #8b949e);
+    text-align: right;
+    flex-shrink: 0;
+}
+
+/* Top value cards list */
+.coll-stat-top-row {
+    display: flex;
+    justify-content: space-between;
+    font-size: 0.78rem;
+    padding: 3px 0;
+    border-bottom: 1px solid var(--border, #30363d);
+    gap: 8px;
+}
+.coll-stat-top-row span:first-child {
+    color: var(--text, #e6edf3);
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+}
+.coll-stat-top-row span:last-child {
+    color: #3fb950;
+    font-weight: 600;
+    flex-shrink: 0;
+}
+
+.coll-stats-empty {
+    color: var(--text-secondary, #8b949e);
+    font-size: 0.78rem;
+    font-style: italic;
+}
+
+@media (min-width: 900px) {
+    .coll-stats-grid {
+        grid-template-columns: repeat(2, 1fr);
+    }
+    .coll-stats-summary {
+        grid-column: 1 / -1;
+    }
+}
+@media (min-width: 1400px) {
+    .coll-stats-grid {
+        grid-template-columns: repeat(3, 1fr);
+    }
+}
+
+
+
+/* ── 18. Column Visibility Panel ───────────────────────────── */
+
+.coll-col-toggle-btn {
+    background: rgba(139,148,158,0.1);
+    border: 1px solid var(--border, #30363d);
+    color: var(--text-secondary, #8b949e);
+    border-radius: 6px;
+    padding: 6px 12px;
+    font-size: 0.8rem;
+    cursor: pointer;
+    white-space: nowrap;
+    transition: background 0.15s;
+}
+.coll-col-toggle-btn:hover {
+    background: rgba(139,148,158,0.2);
+    color: var(--text, #e6edf3);
+}
+
+.coll-col-panel {
+    position: absolute;
+    top: calc(100% + 6px);
+    right: 0;
+    z-index: 500;
+    background: var(--surface, #161b22);
+    border: 1px solid var(--border, #30363d);
+    border-radius: 10px;
+    box-shadow: 0 8px 24px rgba(0,0,0,0.4);
+    min-width: 180px;
+    animation: coll-fade-in 0.15s ease;
+}
+
+.coll-col-panel-inner {
+    padding: 10px 12px 12px;
+}
+
+.coll-col-panel-hdr {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    font-size: 0.78rem;
+    font-weight: 600;
+    color: var(--text-secondary, #8b949e);
+    text-transform: uppercase;
+    letter-spacing: 0.05em;
+    margin-bottom: 8px;
+    padding-bottom: 6px;
+    border-bottom: 1px solid var(--border, #30363d);
+}
+
+.coll-col-close {
+    background: none;
+    border: none;
+    color: var(--text-secondary, #8b949e);
+    cursor: pointer;
+    font-size: 1rem;
+    padding: 0 4px;
+    line-height: 1;
+}
+.coll-col-close:hover { color: var(--text, #e6edf3); }
+
+.coll-col-toggle-item {
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    padding: 4px 0;
+    font-size: 0.8rem;
+    color: var(--text, #e6edf3);
+    cursor: pointer;
+    user-select: none;
+}
+.coll-col-toggle-item input[type="checkbox"] {
+    accent-color: var(--accent, #6366f1);
+    cursor: pointer;
+}
+.coll-col-toggle-item:hover {
+    color: var(--accent, #6366f1);
+}
+
+.coll-col-reset {
+    margin-top: 10px;
+    width: 100%;
+    background: rgba(99,102,241,0.1);
+    border: 1px solid rgba(99,102,241,0.3);
+    color: var(--accent, #6366f1);
+    border-radius: 5px;
+    padding: 5px 8px;
+    font-size: 0.75rem;
+    cursor: pointer;
+    transition: background 0.15s;
+}
+.coll-col-reset:hover {
+    background: rgba(99,102,241,0.25);
+}

ui/collection.js

@@ -410,6 +410,148 @@ const Collection = (() => {
         renderCountBadge();
     }
 
+    // ── Stats Dashboard ──────────────────────────────────────
+    let statsData = null;
+    let statsOpen = false;
+
+    async function fetchStats() {
+        try {
+            const res = await fetch(`${API_BASE}/api/collection/stats`);
+            if (!res.ok) throw new Error(`HTTP ${res.status}`);
+            statsData = await res.json();
+        } catch (err) {
+            console.error('[Collection] fetchStats error:', err);
+            statsData = null;
+        }
+    }
+
+    function toggleStats() {
+        statsOpen = !statsOpen;
+        const panel = document.getElementById('coll-stats-panel');
+        if (!panel) return;
+        if (statsOpen) {
+            if (!statsData) {
+                panel.innerHTML = '<div class="coll-stats-loading">Loading stats…</div>';
+                panel.style.display = 'block';
+                fetchStats().then(() => renderStatsPanel());
+            } else {
+                renderStatsPanel();
+            }
+            panel.style.display = 'block';
+        } else {
+            panel.style.display = 'none';
+        }
+    }
+
+    function renderStatsPanel() {
+        const panel = document.getElementById('coll-stats-panel');
+        if (!panel || !statsData) return;
+        const s = statsData;
+
+        // CMC curve bar chart (CSS-only)
+        const cmcMax = Math.max(...Object.values(s.cmcCurve || {}), 1);
+        const cmcBars = [0,1,2,3,4,5,6,7].map(i => {
+            const count = (s.cmcCurve || {})[String(i)] || 0;
+            const pct = Math.round((count / cmcMax) * 100);
+            const label = i === 7 ? '7+' : String(i);
+            return `<div class="coll-stat-bar-col">
+                <div class="coll-stat-bar" style="height:${pct}%" title="${count}"></div>
+                <span class="coll-stat-bar-label">${label}</span>
+            </div>`;
+        }).join('');
+
+        // Color distribution
+        const colorTotal = Math.max(Object.values(s.colorDistribution || {}).reduce((a,b) => a+b, 0), 1);
+        const colorBars = ['W','U','B','R','G','C'].map(c => {
+            const count = (s.colorDistribution || {})[c] || 0;
+            const pct = Math.round((count / colorTotal) * 100);
+            const ci = COLOR_MAP[c];
+            return `<div class="coll-stat-color-bar">
+                <span class="coll-stat-color-pip" style="background:${ci.bg};color:${ci.color}">${ci.label}</span>
+                <div class="coll-stat-hbar-track"><div class="coll-stat-hbar" style="width:${pct}%;background:${ci.bg}"></div></div>
+                <span class="coll-stat-hbar-val">${count}</span>
+            </div>`;
+        }).join('');
+
+        // Type distribution
+        const typeTotal = Math.max(Object.values(s.typeDistribution || {}).reduce((a,b) => a+b, 0), 1);
+        const typeBars = Object.entries(s.typeDistribution || {}).sort((a,b) => b[1]-a[1]).map(([t, count]) => {
+            const pct = Math.round((count / typeTotal) * 100);
+            return `<div class="coll-stat-type-row">
+                <span class="coll-stat-type-label">${escapeHtml(t)}</span>
+                <div class="coll-stat-hbar-track"><div class="coll-stat-hbar coll-stat-hbar-type" style="width:${pct}%"></div></div>
+                <span class="coll-stat-hbar-val">${count}</span>
+            </div>`;
+        }).join('');
+
+        // Rarity distribution
+        const rarityOrder = ['common','uncommon','rare','mythic'];
+        const rarityColors = {common:'#706f6f', uncommon:'#859ba6', rare:'#c9a849', mythic:'#d35400'};
+        const rarityTotal = Math.max(Object.values(s.rarityDistribution || {}).reduce((a,b) => a+b, 0), 1);
+        const rarityBars = rarityOrder.map(r => {
+            const count = (s.rarityDistribution || {})[r] || 0;
+            const pct = Math.round((count / rarityTotal) * 100);
+            return `<div class="coll-stat-type-row">
+                <span class="coll-stat-type-label" style="color:${rarityColors[r] || '#ccc'}">${r.charAt(0).toUpperCase() + r.slice(1)}</span>
+                <div class="coll-stat-hbar-track"><div class="coll-stat-hbar" style="width:${pct}%;background:${rarityColors[r] || '#555'}"></div></div>
+                <span class="coll-stat-hbar-val">${count}</span>
+            </div>`;
+        }).join('');
+
+        // Top value cards
+        const topCards = (s.topValueCards || []).map(c =>
+            `<div class="coll-stat-top-row"><span>${escapeHtml(c.name)}</span><span>$${Number(c.price).toFixed(2)}</span></div>`
+        ).join('');
+
+        // Category/role coverage
+        const catEntries = Object.entries(s.categoryDistribution || {}).sort((a,b) => b[1]-a[1]).slice(0, 12);
+        const catMax = Math.max(...catEntries.map(e => e[1]), 1);
+        const catBars = catEntries.map(([cat, count]) => {
+            const pct = Math.round((count / catMax) * 100);
+            return `<div class="coll-stat-type-row">
+                <span class="coll-stat-type-label">${escapeHtml(cat)}</span>
+                <div class="coll-stat-hbar-track"><div class="coll-stat-hbar coll-stat-hbar-cat" style="width:${pct}%"></div></div>
+                <span class="coll-stat-hbar-val">${count}</span>
+            </div>`;
+        }).join('');
+
+        panel.innerHTML = `
+            <div class="coll-stats-grid">
+                <div class="coll-stats-summary">
+                    <div class="coll-stat-card"><div class="coll-stat-num">${(s.totalCards || 0).toLocaleString()}</div><div class="coll-stat-label">Total Cards</div></div>
+                    <div class="coll-stat-card"><div class="coll-stat-num">${(s.totalUniqueCards || 0).toLocaleString()}</div><div class="coll-stat-label">Unique</div></div>
+                    <div class="coll-stat-card"><div class="coll-stat-num">$${(s.totalValue || 0).toLocaleString(undefined,{minimumFractionDigits:2})}</div><div class="coll-stat-label">Total Value</div></div>
+                    <div class="coll-stat-card"><div class="coll-stat-num">${s.avgCmc || 0}</div><div class="coll-stat-label">Avg CMC</div></div>
+                </div>
+                <div class="coll-stats-section">
+                    <h4>CMC Curve</h4>
+                    <div class="coll-stat-bar-chart">${cmcBars}</div>
+                </div>
+                <div class="coll-stats-section">
+                    <h4>Color Distribution</h4>
+                    ${colorBars}
+                </div>
+                <div class="coll-stats-section">
+                    <h4>Type Distribution</h4>
+                    ${typeBars}
+                </div>
+                <div class="coll-stats-section">
+                    <h4>Rarity</h4>
+                    ${rarityBars}
+                </div>
+                <div class="coll-stats-section">
+                    <h4>Role Coverage</h4>
+                    ${catBars || '<div class="coll-stats-empty">No categories assigned yet</div>'}
+                </div>
+                <div class="coll-stats-section">
+                    <h4>Top Value Cards</h4>
+                    ${topCards || '<div class="coll-stats-empty">No priced cards</div>'}
+                </div>
+            </div>
+        `;
+    }
+
+
     async function fetchCardDetail(cardId) {
         const res = await fetch(`${API_BASE}/api/collection/${cardId}`);
         if (!res.ok) throw new Error(`HTTP ${res.status}`);
@@ -444,7 +586,8 @@ const Collection = (() => {
         // Pre-load set/keyword caches for filter autocomplete
         loadSetsCache();
         loadKeywordsCache();
-        await fetchCollection();
+        applyColumnVisibility();
+    await fetchCollection();
     }
 
     /** Adjust table container height based on actual topbar height */
@@ -489,10 +632,17 @@ const Collection = (() => {
                     <button class="coll-export-btn" onclick="Collection.openExport()">Export</button>
                     <button class="coll-autoclassify-btn" onclick="Collection.autoClassify()" title="Auto-detect categories (Ramp, Draw, Removal, etc.) for uncategorized cards">Auto-Classify</button>
                     <button class="coll-reenrich-btn" onclick="Collection.reEnrich()" title="Re-fetch all card data from Scryfall">Refresh Data</button>
-                    <a href="deckbuilder.html" class="coll-nav-link" style="color:var(--lab-purple);border-color:var(--lab-purple)">⚔ Deck Builder</a>
+          <button class="coll-stats-btn" onclick="Collection.toggleStats()" title="Show collection statistics dashboard">📊 Stats</button>
+          <div style="position:relative">
+          <button class="coll-col-toggle-btn" onclick="Collection.openColumnPanel()" title="Show/hide table columns">☰ Columns</button>
+          <div id="coll-col-panel" class="coll-col-panel" style="display:none"></div>
+                  </div>
+          <a href="deckbuilder.html" class="coll-nav-link" style="color:var(--lab-purple);border-color:var(--lab-purple)">⚔ Deck Builder</a>
                     <a href="deckgenerator.html" class="coll-nav-link" style="color:var(--lab-warning);border-color:var(--lab-warning)">⚡ Auto Gen</a>
                 </div>
             </div>
+        <!-- Stats Dashboard Panel -->
+        <div id="coll-stats-panel" class="coll-stats-panel" style="display:none"></div>
 
             <!-- Main -->
             <div class="coll-main">
@@ -821,6 +971,79 @@ const Collection = (() => {
         </div>`;
     }
 
+    // ── Column Visibility ──────────────────────────────────────
+    function colVis(key) {
+        return state.preferences.visibleColumns.includes(key) ? '' : 'display:none;';
+    }
+
+    function toggleColumn(key) {
+        const cols = state.preferences.visibleColumns;
+        const idx = cols.indexOf(key);
+        if (idx >= 0) {
+            if (cols.length <= 1) return; // keep at least 1 column
+            cols.splice(idx, 1);
+        } else {
+            cols.push(key);
+        }
+        savePreferences();
+        renderTable();
+        applyColumnVisibility();
+    }
+
+    function renderColumnPanel() {
+        const panel = document.getElementById('coll-col-panel');
+        if (!panel) return;
+        const cols = COLUMNS.map(col => {
+            const checked = state.preferences.visibleColumns.includes(col.key) ? 'checked' : '';
+            return `<label class="coll-col-toggle-item">
+                <input type="checkbox" ${checked} onchange="Collection.toggleColumn('${col.key}')">
+                ${escapeHtml(col.label)}
+            </label>`;
+        }).join('');
+        panel.innerHTML = `
+            <div class="coll-col-panel-inner">
+                <div class="coll-col-panel-hdr">Columns <button class="coll-col-close" onclick="Collection.closeColumnPanel()">×</button></div>
+                ${cols}
+                <button class="coll-col-reset" onclick="Collection.resetColumns()">Reset Defaults</button>
+            </div>`;
+        panel.style.display = 'block';
+    }
+
+    function openColumnPanel() {
+        const panel = document.getElementById('coll-col-panel');
+        if (!panel) return;
+        if (panel.style.display === 'block') {
+            panel.style.display = 'none';
+        } else {
+            renderColumnPanel();
+        }
+    }
+
+    function closeColumnPanel() {
+        const panel = document.getElementById('coll-col-panel');
+        if (panel) panel.style.display = 'none';
+    }
+
+    function resetColumns() {
+        state.preferences.visibleColumns = ['quantity','name','mana_cost','type_line','color_identity','cmc','power_toughness','rarity','set_name','tcg_price','edhrec_rank','salt_score','category','finish'];
+        savePreferences();
+        renderTable();
+        renderColumnPanel();
+    }
+    function applyColumnVisibility() {
+        let styleEl = document.getElementById('coll-col-vis-style');
+        if (!styleEl) {
+            styleEl = document.createElement('style');
+            styleEl.id = 'coll-col-vis-style';
+            document.head.appendChild(styleEl);
+        }
+        const hidden = COLUMNS.filter(col => !state.preferences.visibleColumns.includes(col.key));
+        styleEl.textContent = hidden.map(col =>
+            `.coll-th-${col.key}, .coll-td-${col.key} { display: none !important; }`
+        ).join('\n');
+    }
+
+
     // ── Table Header ────────────────────────────────────────
 
     const COLUMNS = [
@@ -843,8 +1066,8 @@ const Collection = (() => {
     ];
 
     function renderTableHeaderHtml() {
-        const headers = COLUMNS.map(col => {
-            if (!col.sortable) {
+        const visibleCols = COLUMNS.filter(col => state.preferences.visibleColumns.includes(col.key));
+        const headers = visibleCols.map(col => {            if (!col.sortable) {
                 return `<th class="coll-th coll-th-${col.key}">${col.label}</th>`;
             }
             const isActive = state.sortField === col.key;
@@ -873,12 +1096,12 @@ const Collection = (() => {
         }
 
         if (state.loading) {
-            tbody.innerHTML = `<tr><td colspan="${COLUMNS.length}" class="coll-empty">Loading…</td></tr>`;
+            tbody.innerHTML = `<tr><td colspan="${state.preferences.visibleColumns.length}" class="coll-empty">Loading…</td></tr>`;
             return;
         }
 
         if (state.items.length === 0) {
-            tbody.innerHTML = `<tr><td colspan="${COLUMNS.length}" class="coll-empty">
+            tbody.innerHTML = `<tr><td colspan="${state.preferences.visibleColumns.length}" class="coll-empty">
                 <div class="coll-empty-state">
                     <div class="coll-empty-icon">🃏</div>
                     <div>No cards in collection</div>
@@ -2099,7 +2322,8 @@ const Collection = (() => {
             if (footer) footer.style.display = 'none';
 
             // Refresh collection
-            await fetchCollection();
+            applyColumnVisibility();
+    await fetchCollection();
 
         } catch (err) {
             if (statusEl) {
@@ -2736,7 +2960,7 @@ const Collection = (() => {
             // Refresh collection after short delay
             setTimeout(() => {
                 closeScan();
-                loadCollection();
+                fetchCollection();
             }, 1500);
         } catch (e) {
             if (statusEl) statusEl.innerHTML = '<div class="coll-scan-error-msg">Failed: ' + escapeHtml(e.message) + '</div>';
@@ -2851,5 +3075,11 @@ const Collection = (() => {
         popoverAddCatFromInput,
         reEnrich,
         autoClassify,
+        toggleStats,
+        // Column visibility
+        toggleColumn,
+        openColumnPanel,
+        closeColumnPanel,
+        resetColumns,
     };
 })();

ui/deckgenerator.css

@@ -1297,4 +1297,106 @@
 
 .dg-log-error {
     color: #f59e0b;
-}
+}
+
+/* ── Progress Bar (Deck Generation) ──────────────────── */
+.dg-progress-wrap {
+    width: 100%;
+    max-width: 400px;
+    margin-top: 16px;
+}
+
+.dg-progress-bar {
+    height: 6px;
+    background: var(--surface-2, #2a2a3a);
+    border-radius: 3px;
+    overflow: hidden;
+}
+
+.dg-progress-fill {
+    height: 100%;
+    width: 0%;
+    background: linear-gradient(90deg, var(--accent, #7c6ff7), #a78bfa);
+    border-radius: 3px;
+    transition: width 1s ease;
+}
+
+.dg-progress-meta {
+    display: flex;
+    justify-content: space-between;
+    margin-top: 6px;
+    font-size: 0.78rem;
+}
+
+.dg-progress-phase {
+    color: var(--text-muted, #999);
+}
+
+.dg-progress-elapsed {
+    color: var(--text-muted, #999);
+    font-variant-numeric: tabular-nums;
+}
+
+/* — Omit Cards Chips ————————————————————————— */
+.dg-omit-wrap {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  gap: 6px;
+  padding: 6px 8px;
+  border: 1px solid var(--border, #333);
+  border-radius: 6px;
+  background: var(--bg-input, #1a1a2e);
+  min-height: 38px;
+  cursor: text;
+}
+
+.dg-omit-wrap:focus-within {
+  border-color: var(--accent, #6c5ce7);
+  box-shadow: 0 0 0 2px rgba(108, 92, 231, 0.25);
+}
+
+.dg-omit-chips {
+  display: contents;
+}
+
+.dg-chip {
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+  padding: 2px 8px;
+  border-radius: 12px;
+  background: var(--accent, #6c5ce7);
+  color: #fff;
+  font-size: 0.8rem;
+  white-space: nowrap;
+  line-height: 1.6;
+}
+
+.dg-chip-x {
+  cursor: pointer;
+  font-size: 0.9rem;
+  opacity: 0.7;
+  margin-left: 2px;
+}
+
+.dg-chip-x:hover {
+  opacity: 1;
+}
+
+.dg-omit-input {
+  flex: 1 1 120px;
+  border: none !important;
+  background: transparent !important;
+  padding: 2px 4px !important;
+  min-width: 120px;
+  outline: none;
+  box-shadow: none !important;
+}
+
+.dg-hint {
+  display: block;
+  margin-top: 6px;
+  color: var(--text-muted, #999);
+  font-size: 0.75rem;
+}

ui/deckgenerator.html

@@ -138,6 +138,25 @@
                     </div>
                 </div>
 
+                                <!-- Omit Cards (chips input) -->
+                <div class="dg-panel">
+                    <div class="dg-panel-header">
+                        <span class="dg-panel-title">Omit Cards</span>
+                    </div>
+                    <div class="dg-panel-body">
+                        <div class="dg-omit-wrap" id="dg-omit-wrap">
+                            <div class="dg-omit-chips" id="dg-omit-chips"></div>
+                            <input
+                                type="text"
+                                class="dg-input dg-omit-input"
+                                id="dg-omit-input"
+                                placeholder="Type a card name and press Enter…"
+                            />
+                        </div>
+                        <small class="dg-hint">Cards listed here will be excluded from the generated deck.</small>
+                    </div>
+                </div>
+
                 <!-- Action Buttons -->
                 <div class="dg-actions">
                     <button class="dg-btn dg-btn-preview" id="dg-preview-btn" disabled>
@@ -166,6 +185,15 @@
                     <div class="dg-spinner"></div>
                     <div class="dg-loading-text" id="dg-loading-text">Generating deck via Local AI...</div>
                 </div>
+                                    <div class="dg-progress-wrap" id="dg-progress-wrap" style="display:none">
+                        <div class="dg-progress-bar">
+                            <div class="dg-progress-fill" id="dg-progress-fill"></div>
+                        </div>
+                        <div class="dg-progress-meta">
+                            <span class="dg-progress-phase" id="dg-progress-phase"></span>
+                            <span class="dg-progress-elapsed" id="dg-progress-elapsed">0s elapsed</span>
+                        </div>
+                    </div>
 
                 <!-- Results -->
                 <div class="dg-results" id="dg-results" style="display:none">

ui/deckgenerator.js

@@ -53,6 +53,95 @@ const DeckGenerator = (() => {
 
     let searchTimeout = null;
 
+    // ── Progress bar state ──────────────────────────────
+    const PROGRESS_PHASES = [
+        { at: 0,  pct: 5,   msg: 'Connecting to AI model...' },
+        { at: 3,  pct: 10,  msg: 'Resolving commander on Scryfall...' },
+        { at: 6,  pct: 18,  msg: 'Building collection summary...' },
+        { at: 10, pct: 25,  msg: 'Sending prompt to LLM...' },
+        { at: 20, pct: 40,  msg: 'AI is building the 99...' },
+        { at: 35, pct: 55,  msg: 'Selecting cards and filling slots...' },
+        { at: 50, pct: 65,  msg: 'Running smart substitution...' },
+        { at: 70, pct: 78,  msg: 'Cross-referencing with collection...' },
+        { at: 90, pct: 88,  msg: 'Almost there — finalizing deck...' },
+        { at: 110, pct: 93, msg: 'Still working — large model can take a moment...' },
+    ];
+    let _progressInterval = null;
+    let _progressStart = 0;
+
+    function startProgress() {
+        _progressStart = Date.now();
+        const wrap = $('dg-progress-wrap');
+        const fill = $('dg-progress-fill');
+        const phase = $('dg-progress-phase');
+        const elapsed = $('dg-progress-elapsed');
+        if (!wrap) return;
+        wrap.style.display = 'block';
+        fill.style.width = '0%';
+        phase.textContent = '';
+        elapsed.textContent = '0s elapsed';
+        let lastPhaseIdx = -1;
+        _progressInterval = setInterval(() => {
+            const secs = Math.floor((Date.now() - _progressStart) / 1000);
+            elapsed.textContent = secs + 's elapsed';
+            for (let i = PROGRESS_PHASES.length - 1; i >= 0; i--) {
+                if (secs >= PROGRESS_PHASES[i].at) {
+                    fill.style.width = PROGRESS_PHASES[i].pct + '%';
+                    if (i !== lastPhaseIdx) {
+                        phase.textContent = PROGRESS_PHASES[i].msg;
+                        logMessage(PROGRESS_PHASES[i].msg, 'info');
+                        lastPhaseIdx = i;
+                    }
+                    break;
+                }
+            }
+        }, 1000);
+    }
+
+    function stopProgress(success) {
+        if (_progressInterval) {
+            clearInterval(_progressInterval);
+            _progressInterval = null;
+        }
+        const fill = $('dg-progress-fill');
+        const phase = $('dg-progress-phase');
+        const elapsed = $('dg-progress-elapsed');
+        if (fill) fill.style.width = success ? '100%' : '0%';
+        if (phase) phase.textContent = success ? 'Done!' : 'Failed';
+        const secs = Math.floor((Date.now() - _progressStart) / 1000);
+        if (elapsed) elapsed.textContent = secs + 's total';
+    }
+
+        // ── Omit Cards (chips) ─────────────────────────────
+    let omitCards = [];
+
+    function addOmitCard(name) {
+        const trimmed = name.trim();
+        if (!trimmed) return;
+        if (omitCards.some(c => c.toLowerCase() === trimmed.toLowerCase())) return;
+        omitCards.push(trimmed);
+        renderOmitChips();
+    }
+
+    function removeOmitCard(idx) {
+        omitCards.splice(idx, 1);
+        renderOmitChips();
+    }
+
+    function renderOmitChips() {
+        const container = $('dg-omit-chips');
+        if (!container) return;
+        container.innerHTML = omitCards.map((name, i) =>
+            '<span class="dg-chip">' +
+                escHtml(name) +
+                '<span class="dg-chip-x" data-idx="' + i + '">&times;</span>' +
+            '</span>'
+        ).join('');
+        container.querySelectorAll('.dg-chip-x').forEach(el => {
+            el.addEventListener('click', () => removeOmitCard(parseInt(el.dataset.idx, 10)));
+        });
+    }
+
     const $ = (id) => document.getElementById(id);
 
     // ── Init ────────────────────────────────────────────────
@@ -118,6 +207,22 @@ const DeckGenerator = (() => {
         $('dg-export-dck').addEventListener('click', () => exportDeck('dck'));
         $('dg-export-moxfield').addEventListener('click', () => exportMoxfield());
         $('dg-export-shopping').addEventListener('click', () => exportShopping());
+
+                // Omit cards chips input
+        const omitInput = $('dg-omit-input');
+        if (omitInput) {
+            omitInput.addEventListener('keydown', (e) => {
+                if (e.key === 'Enter') {
+                    e.preventDefault();
+                    addOmitCard(omitInput.value);
+                    omitInput.value = '';
+                }
+            });
+        }
+        const omitWrap = $('dg-omit-wrap');
+        if (omitWrap) {
+            omitWrap.addEventListener('click', () => omitInput && omitInput.focus());
+        }
     }
 
     function checkQueryParams() {
@@ -138,7 +243,7 @@ const DeckGenerator = (() => {
             target_bracket: state.targetBracket,
             budget_usd: budget,
             budget_mode: $('dg-budget-mode').value,
-            omit_cards: [],
+            omit_cards: omitCards.slice(),
             use_collection: $('dg-use-collection').checked,
             run_substitution: $('dg-substitution').checked,
             model: $('dg-model').value,
@@ -240,6 +345,12 @@ const DeckGenerator = (() => {
         }
 
         $('dg-preview-btn').disabled = false;
+
+                // Auto-fill deck name if empty
+        const deckNameInput = $('dg-deck-name');
+        if (deckNameInput && !deckNameInput.value.trim()) {
+            deckNameInput.value = 'Auto - ' + cmdr.name;
+        }
     }
 
     function clearCommander() {
@@ -284,6 +395,7 @@ const DeckGenerator = (() => {
         logMessage('Building request body with commander: <b>' + state.commander.name + '</b>', 'info');
 
         setLoading(true, 'Generating deck via Local AI...');
+        startProgress();
 
         const body = buildRequestBody();
         logMessage('Sending request to AI model... this may take a moment.', 'info');
@@ -307,10 +419,12 @@ const DeckGenerator = (() => {
             const stats = state.previewResult.stats || {};
             const cardCount = (stats.total_cards || 0);
             logMessage('Deck generated: <b>' + cardCount + ' cards</b> | $' + (stats.total_price_usd || 0).toFixed(2) + ' estimated', 'success');
+                        stopProgress(true);
             renderResults(state.previewResult);
 
             toast('Deck generated: ' + (stats.total_cards || 0) + ' cards | $' + (stats.total_price_usd || 0), 'success');
         } catch (e) {
+                    stopProgress(false);
             toast(e.message || 'Generation failed', 'error');
             logMessage('Error: ' + (e.message || 'Unknown error'), 'error');
             setLoading(false);

ui/simulator.html

@@ -62,7 +62,7 @@
                 <div class="deepseek-config-row">
                     <div class="form-group sim-field-grow">
                         <label class="form-label" for="ds-api-base">API Endpoint</label>
-                        <input type="text" id="ds-api-base" class="form-input" value="http://192.168.0.240:11434" placeholder="http://192.168.0.240:11434" />
+                        <input type="text" id="ds-api-base" class="form-input" value="http://localhost:11434" placeholder="http://localhost:11434" />
                     </div>
                     <div class="form-group sim-field-auto">
                         <label class="form-label" for="ds-model">Model</label>