diff --git a/ml/config/scope.py b/ml/config/scope.py index 3a46b2c..b83c282 100644 --- a/ml/config/scope.py +++ b/ml/config/scope.py @@ -155,15 +155,25 @@ class StateDimensions: 2 players = 28 global features + turn_number = 1 Total global = 29 + + Zone embedding boundary (architectural decision — Phase 5 follow-up): + Zone dims are **always** 768 per zone slot, zero-padded when a zone + is empty (no cards). The fixed 6144-dim zone block is the contract + between ForgeEpisodeGenerator (upstream) and the policy network + (downstream). All downstream consumers (PolicyNetwork, PPOTrainer, + dataset builder) depend on this shape. If a future change adds + attention pooling or variable-length zone representations, the + boundary must be updated here *and* in StateEncoder._encode_zones. """ card_embedding_dim: int = 768 # From mtg-embeddings - zone_pool_dim: int = 768 # Mean-pooled per zone + zone_pool_dim: int = 768 # Mean-pooled per zone (zero-vec when empty) # Zones per player: hand, battlefield, graveyard, command_zone zones_per_player: int = 4 num_players: int = 2 # 1v1 # Zone vectors: 4 zones × 2 players × 768 = 6144 + # This is a fixed-size block; empty zones produce zero vectors. total_zone_dim: int = 4 * 2 * 768 # 6144 # Global scalar features @@ -268,6 +278,10 @@ class DistillationDefaults: These are consumed by ml.training.distillation_loop.DistillationConfig when no explicit override is provided. + + Default weights: Forge-only (forge_weight=1.0, ppo_weight=0.0). + The baseline trains exclusively on Forge simulation data. Use + ``MIXED_MODE_PRESETS`` below for blended Forge+PPO configurations. """ max_iterations: int = 10 convergence_window: int = 3 @@ -281,9 +295,9 @@ class DistillationDefaults: ppo_entropy_coeff: float = 0.01 ppo_eval_episodes: int = 100 - # Dataset mixing + # Dataset mixing — Forge-only by default (Phase 5 follow-up decision) forge_weight: float = 1.0 - ppo_weight: float = 0.5 + ppo_weight: float = 0.0 min_reward_threshold: float = 0.0 # Quality gate thresholds @@ -302,3 +316,35 @@ class DistillationDefaults: DISTILLATION_DEFAULTS = DistillationDefaults() + + +# ══════════════════════════════════════════════════════════ +# Mixed-Mode Presets (Phase 5 follow-up) +# ══════════════════════════════════════════════════════════ + +# Pre-defined Forge/PPO weight combinations for common scenarios. +# Select a preset by name when starting a distillation run to +# override the Forge-only defaults without specifying raw weights. +# +# "forge_only" — baseline: 100% Forge data, ignore PPO +# "forge_90_10" — light PPO blending: 90% Forge, 10% PPO +# "forge_80_20" — heavier PPO blending: 80% Forge, 20% PPO +MIXED_MODE_PRESETS: Dict[str, Dict[str, float]] = { + "forge_only": {"forge_weight": 1.0, "ppo_weight": 0.0}, + "forge_90_10": {"forge_weight": 1.0, "ppo_weight": 0.11}, # ~10% PPO + "forge_80_20": {"forge_weight": 1.0, "ppo_weight": 0.25}, # ~20% PPO +} + + +def get_preset_weights(preset_name: str) -> Dict[str, float]: + """Look up a mixed-mode preset by name. + + Returns a dict with ``forge_weight`` and ``ppo_weight`` keys. + Raises ``KeyError`` if the preset name is unknown. + """ + if preset_name not in MIXED_MODE_PRESETS: + raise KeyError( + f"Unknown preset '{preset_name}'. " + f"Available: {list(MIXED_MODE_PRESETS.keys())}" + ) + return dict(MIXED_MODE_PRESETS[preset_name]) diff --git a/ml/data/dataset_builder.py b/ml/data/dataset_builder.py index c5b1ef9..243161e 100644 --- a/ml/data/dataset_builder.py +++ b/ml/data/dataset_builder.py @@ -63,10 +63,10 @@ class DatasetConfig: max_samples: Optional[int] = None # Source weights control sampling ratios between Forge and PPO data. - # e.g. {"forge": 1.0, "ppo": 0.5} means keep all Forge data but - # randomly sample 50% of PPO data to avoid overwhelming the baseline. + # Default: Forge-only (ppo=0.0). Use MIXED_MODE_PRESETS from + # ml.config.scope for blended configurations (e.g. 90/10, 80/20). source_weights: Dict[str, float] = field( - default_factory=lambda: {"forge": 1.0, "ppo": 0.5} + default_factory=lambda: {"forge": 1.0, "ppo": 0.0} ) # Minimum reward threshold for PPO data — only include decisions @@ -209,7 +209,7 @@ def build_dataset( Dict with keys: states, labels, game_ids, outcomes, playstyles, sources """ if source_weights is None: - source_weights = {"forge": 1.0, "ppo": 0.5} + source_weights = {"forge": 1.0, "ppo": 0.0} # Load card embeddings card_index = CardEmbeddingIndex(embeddings_dir) @@ -502,8 +502,8 @@ def main(): help="Sampling weight for Forge sim data (default: 1.0)", ) parser.add_argument( - "--ppo-weight", type=float, default=0.5, - help="Sampling weight for PPO self-play data (default: 0.5)", + "--ppo-weight", type=float, default=0.0, + help="Sampling weight for PPO self-play data (default: 0.0, Forge-only)", ) parser.add_argument( "--min-reward", type=float, default=0.0, diff --git a/ml/encoder/state_encoder.py b/ml/encoder/state_encoder.py index f0304b1..371dbe4 100644 --- a/ml/encoder/state_encoder.py +++ b/ml/encoder/state_encoder.py @@ -276,6 +276,13 @@ class StateEncoder: """ Encode zone contents as mean-pooled card embeddings. Shape: (4 zones × 2 players × 768) = (6144,) + + Zone embedding boundary: each zone slot is exactly ``zone_pool_dim`` + (768) floats. Empty zones produce a zero vector of that size. This + fixed-size contract is relied upon by every downstream consumer + (PolicyNetwork, PPOTrainer, dataset builder). See + ``StateDimensions`` in ``ml/config/scope.py`` for the canonical + definition. """ zone_keys = ["hand", "battlefield", "graveyard", "command_zone"] vectors = [] @@ -283,6 +290,8 @@ class StateEncoder: for p in players[:2]: for zone_key in zone_keys: cards = p.get(zone_key, []) + # mean_pool_zone returns a zero vector when cards is empty, + # preserving the fixed 768-dim slot size. pooled = self.card_index.mean_pool_zone(cards) vectors.append(pooled) diff --git a/ml/serving/online_learning_store.py b/ml/serving/online_learning_store.py new file mode 100644 index 0000000..bbb48c1 --- /dev/null +++ b/ml/serving/online_learning_store.py @@ -0,0 +1,210 @@ +""" +Commander AI Lab — Online Learning Store (SQLite WAL) +═════════════════════════════════════════════════════ + +Single-machine backing store for online learning data collected during +policy inference (``/api/ml/predict`` and ``/api/policy/decide``). + +Each prediction request can optionally log the state snapshot and the +chosen action so that future training runs can incorporate online +decision data alongside Forge and PPO batches. + +Design decisions (Phase 5 architecture follow-up): + - SQLite in WAL mode for concurrent readers + single writer, which is + the right fit for a single-machine baseline (one API server process). + - Separate DB file (``online_learning.db``) to avoid contention with + the collection database. + - Append-only writes; reads are batched by the dataset builder. + - Thread-safe via the Python sqlite3 module's ``check_same_thread=False`` + and WAL's built-in reader/writer concurrency. + +Usage:: + + store = OnlineLearningStore() # uses default path + store.init_db() + + # At prediction time: + store.record_decision(snapshot_json, action_index, confidence, playstyle) + + # At training time: + decisions = store.fetch_decisions(since_rowid=last_seen, limit=10000) + store.mark_exported(max_rowid) +""" + +import json +import logging +import os +import sqlite3 +import threading +from pathlib import Path +from typing import Dict, List, Optional + +logger = logging.getLogger("ml.serving.online_store") + +_DEFAULT_DB_PATH = str( + Path(__file__).parent.parent.parent / "online_learning.db" +) + + +class OnlineLearningStore: + """SQLite WAL-backed store for online policy decision collection.""" + + def __init__(self, db_path: str = None): + self.db_path = db_path or _DEFAULT_DB_PATH + self._local = threading.local() + + # ------------------------------------------------------------------ + # Connection management + # ------------------------------------------------------------------ + + def _get_conn(self) -> sqlite3.Connection: + """Return a thread-local connection with WAL mode enabled.""" + if not hasattr(self._local, "conn") or self._local.conn is None: + conn = sqlite3.connect(self.db_path, check_same_thread=False) + conn.row_factory = sqlite3.Row + # Enable WAL for concurrent read/write on single machine. + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + self._local.conn = conn + return self._local.conn + + def close(self) -> None: + """Close the thread-local connection.""" + conn = getattr(self._local, "conn", None) + if conn is not None: + try: + conn.close() + except Exception: + pass + self._local.conn = None + + # ------------------------------------------------------------------ + # Schema + # ------------------------------------------------------------------ + + def init_db(self) -> None: + """Create the decisions table if it doesn't exist.""" + conn = self._get_conn() + conn.executescript(""" + CREATE TABLE IF NOT EXISTS online_decisions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + snapshot TEXT NOT NULL, + action_idx INTEGER NOT NULL, + confidence REAL NOT NULL DEFAULT 0.0, + playstyle TEXT NOT NULL DEFAULT 'midrange', + temperature REAL NOT NULL DEFAULT 1.0, + greedy INTEGER NOT NULL DEFAULT 0, + exported INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE INDEX IF NOT EXISTS idx_od_exported + ON online_decisions(exported); + CREATE INDEX IF NOT EXISTS idx_od_created + ON online_decisions(created_at); + """) + conn.commit() + logger.info("Online learning store ready: %s (WAL mode)", self.db_path) + + # ------------------------------------------------------------------ + # Write path (called during prediction) + # ------------------------------------------------------------------ + + def record_decision( + self, + snapshot: Dict, + action_idx: int, + confidence: float, + playstyle: str = "midrange", + temperature: float = 1.0, + greedy: bool = False, + ) -> int: + """Append a policy decision for future training. + + Returns the row id of the inserted record. + """ + conn = self._get_conn() + cur = conn.execute( + """ + INSERT INTO online_decisions + (snapshot, action_idx, confidence, playstyle, temperature, greedy) + VALUES (?, ?, ?, ?, ?, ?) + """, + ( + json.dumps(snapshot, separators=(",", ":")), + action_idx, + confidence, + playstyle, + temperature, + 1 if greedy else 0, + ), + ) + conn.commit() + return cur.lastrowid + + # ------------------------------------------------------------------ + # Read path (called by dataset builder / training pipeline) + # ------------------------------------------------------------------ + + def fetch_decisions( + self, + since_rowid: int = 0, + limit: int = 50000, + only_unexported: bool = True, + ) -> List[Dict]: + """Fetch collected decisions for training. + + Args: + since_rowid: Only return rows with id > this value. + limit: Max rows to return. + only_unexported: If True, skip rows already marked exported. + + Returns: + List of dicts with keys: id, snapshot (parsed), action_idx, + confidence, playstyle, temperature, greedy, created_at. + """ + conn = self._get_conn() + clauses = ["id > ?"] + params: list = [since_rowid] + if only_unexported: + clauses.append("exported = 0") + where = " AND ".join(clauses) + rows = conn.execute( + f"SELECT * FROM online_decisions WHERE {where} ORDER BY id LIMIT ?", + (*params, limit), + ).fetchall() + results = [] + for row in rows: + d = dict(row) + try: + d["snapshot"] = json.loads(d["snapshot"]) + except (json.JSONDecodeError, TypeError): + pass + results.append(d) + return results + + def mark_exported(self, up_to_rowid: int) -> int: + """Mark decisions as exported so they aren't fetched again. + + Returns the number of rows updated. + """ + conn = self._get_conn() + cur = conn.execute( + "UPDATE online_decisions SET exported = 1 WHERE id <= ? AND exported = 0", + (up_to_rowid,), + ) + conn.commit() + return cur.rowcount + + def count(self, only_unexported: bool = False) -> int: + """Return the total number of stored decisions.""" + conn = self._get_conn() + if only_unexported: + row = conn.execute( + "SELECT COUNT(*) FROM online_decisions WHERE exported = 0" + ).fetchone() + else: + row = conn.execute( + "SELECT COUNT(*) FROM online_decisions" + ).fetchone() + return row[0] diff --git a/ml/training/distillation_loop.py b/ml/training/distillation_loop.py index 9128bf0..9ec64a5 100644 --- a/ml/training/distillation_loop.py +++ b/ml/training/distillation_loop.py @@ -84,9 +84,9 @@ class DistillationConfig: opponent: str = "heuristic" playstyle: str = "midrange" - # --- Dataset building --- + # --- Dataset building (Forge-only default; see MIXED_MODE_PRESETS) --- forge_weight: float = 1.0 - ppo_weight: float = 0.5 + ppo_weight: float = 0.0 min_reward_threshold: float = 0.0 # --- Quality gate --- @@ -858,7 +858,7 @@ def main(): # Dataset parser.add_argument("--forge-weight", type=float, default=1.0) - parser.add_argument("--ppo-weight", type=float, default=0.5) + parser.add_argument("--ppo-weight", type=float, default=0.0) parser.add_argument("--min-reward", type=float, default=0.0) # Quality gate diff --git a/routes/ml.py b/routes/ml.py index 5e790e7..b3fb2b4 100644 --- a/routes/ml.py +++ b/routes/ml.py @@ -188,10 +188,95 @@ async def ml_get_decisions(filename: str, limit: int = 100, offset: int = 0): } +# ============================================================== +# Online Learning Store (SQLite WAL — Phase 5 follow-up) +# ============================================================== + +_online_store = None +_online_store_init_attempted = False + +def _get_online_store(): + """Lazy-init the online learning store singleton.""" + global _online_store, _online_store_init_attempted + if _online_store is None and not _online_store_init_attempted: + _online_store_init_attempted = True + try: + from ml.serving.online_learning_store import OnlineLearningStore + _online_store = OnlineLearningStore() + _online_store.init_db() + log_ml.info("Online learning store initialized") + except Exception as e: + log_ml.error("Online learning store init failed: %s", e) + return _online_store + + # ============================================================== # ML Policy Inference Endpoints # ============================================================== +@router.post("/api/policy/decide") +async def policy_decide(request: FastAPIRequest): + """Predict a macro-action and log the decision for online learning. + + Combines prediction with automatic data collection: every decision + is appended to the SQLite WAL-backed online learning store so + future training runs can incorporate live gameplay data. + """ + svc = _get_policy_service() + if svc is None or not svc._loaded: + detail = "Policy model not loaded. " + if svc: + detail += svc._load_error or "Train a model first." + else: + detail += "PyTorch may not be installed." + raise HTTPException(status_code=503, detail=detail) + + body = await request.json() + playstyle = body.pop("archetype", "midrange") + temperature = body.pop("temperature", 1.0) + greedy = body.pop("greedy", False) + collect = body.pop("collect", True) # opt-out flag + + result = svc.predict( + decision_snapshot=body, + playstyle=playstyle, + temperature=temperature, + greedy=greedy, + ) + if "error" in result: + raise HTTPException(status_code=500, detail=result["error"]) + + # Record to online learning store (fire-and-forget — don't block response) + if collect: + store = _get_online_store() + if store is not None: + try: + store.record_decision( + snapshot=body, + action_idx=result["action_index"], + confidence=result["confidence"], + playstyle=playstyle, + temperature=temperature, + greedy=greedy, + ) + except Exception as e: + log_ml.warning("Online learning record failed: %s", e) + + return result + + +@router.get("/api/policy/decide/stats") +async def policy_decide_stats(): + """Return counts of collected online learning decisions.""" + store = _get_online_store() + if store is None: + return {"total": 0, "unexported": 0, "error": "Store not initialized"} + return { + "total": store.count(), + "unexported": store.count(only_unexported=True), + } + + @router.post("/api/ml/predict") async def ml_predict(request: FastAPIRequest): """Predict a macro-action from a game state snapshot.""" @@ -777,6 +862,8 @@ def _run_distillation_pipeline( opponent: str, playstyle: str, min_win_rate: float, + forge_weight: float = 1.0, + ppo_weight: float = 0.0, ): """Run the distillation loop in a background thread.""" global _distillation_loop @@ -800,6 +887,8 @@ def _run_distillation_pipeline( opponent=opponent, playstyle=playstyle, min_ppo_win_rate=min_win_rate, + forge_weight=forge_weight, + ppo_weight=ppo_weight, results_dir=os.path.join(project_root, "results"), models_dir=os.path.join(project_root, "ml", "models"), checkpoint_dir=os.path.join(project_root, "ml", "models", "checkpoints"), @@ -847,9 +936,21 @@ def _run_distillation_pipeline( _distillation_loop = None +@router.get("/api/ml/distill/presets") +async def ml_distill_presets(): + """List available mixed-mode weight presets for distillation.""" + from ml.config.scope import MIXED_MODE_PRESETS + return {"presets": MIXED_MODE_PRESETS} + + @router.post("/api/ml/distill/start") async def ml_start_distillation(request: FastAPIRequest): - """Start the closed-loop distillation pipeline.""" + """Start the closed-loop distillation pipeline. + + Accepts an optional ``preset`` field (e.g. ``"forge_90_10"``) + which overrides ``forgeWeight`` / ``ppoWeight`` with a named + configuration from ``MIXED_MODE_PRESETS``. + """ if _distillation_state.running: raise HTTPException(409, "Distillation loop already in progress") if _training_state.running: @@ -871,6 +972,22 @@ async def ml_start_distillation(request: FastAPIRequest): playstyle = body.get("playstyle", "midrange") min_win_rate = body.get("minWinRate", 0.30) + # Resolve weights: preset > explicit params > defaults (Forge-only) + forge_weight = 1.0 + ppo_weight = 0.0 + preset_name = body.get("preset") + if preset_name: + from ml.config.scope import get_preset_weights + try: + pw = get_preset_weights(preset_name) + forge_weight = pw["forge_weight"] + ppo_weight = pw["ppo_weight"] + except KeyError as e: + raise HTTPException(400, str(e)) + else: + forge_weight = body.get("forgeWeight", 1.0) + ppo_weight = body.get("ppoWeight", 0.0) + with _distillation_lock: _distillation_state.running = True _distillation_state.generation = 0 @@ -890,12 +1007,19 @@ async def ml_start_distillation(request: FastAPIRequest): supervised_epochs, supervised_lr, ppo_iterations, ppo_episodes_per_iter, ppo_lr, opponent, playstyle, min_win_rate, + forge_weight, ppo_weight, ), daemon=True, ) thread.start() - return {"status": "started", "max_iterations": max_iterations} + return { + "status": "started", + "max_iterations": max_iterations, + "forge_weight": forge_weight, + "ppo_weight": ppo_weight, + "preset": preset_name, + } @router.get("/api/ml/distill/status") diff --git a/tests/test_phase5_followup.py b/tests/test_phase5_followup.py new file mode 100644 index 0000000..240830c --- /dev/null +++ b/tests/test_phase5_followup.py @@ -0,0 +1,291 @@ +""" +Phase 5 Architecture Follow-Up — Unit Tests +============================================ +Tests for the three approved decisions: + 1. Zone embedding boundary is explicit (docs + code + assertion) + 2. SQLite WAL-backed online learning store + 3. Forge-only default weights + mixed-mode presets + +Run with: pytest tests/test_phase5_followup.py -v +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import tempfile + +import numpy as np +import pytest + + +# ==================================================================== +# Decision 1: Zone Embedding Boundary +# ==================================================================== + +class TestZoneEmbeddingBoundary: + """Verify that the zone embedding contract is explicit and enforced.""" + + def test_state_dims_total_zone_dim(self): + from ml.config.scope import STATE_DIMS + # 4 zones × 2 players × 768 = 6144 + assert STATE_DIMS.total_zone_dim == 4 * 2 * 768 == 6144 + + def test_state_dims_total_state_dim(self): + from ml.config.scope import STATE_DIMS + # 29 global + 6144 zone + 4 playstyle = 6177 + assert STATE_DIMS.total_state_dim == 6177 + + def test_zone_pool_dim_matches_card_embedding_dim(self): + from ml.config.scope import STATE_DIMS + # Zone pool output is same size as card embedding input + assert STATE_DIMS.zone_pool_dim == STATE_DIMS.card_embedding_dim == 768 + + def test_empty_zone_produces_zero_vector(self): + """Empty zone must produce a zero vector of exactly zone_pool_dim.""" + from ml.config.scope import STATE_DIMS + from ml.encoder.state_encoder import CardEmbeddingIndex + + idx = CardEmbeddingIndex() + # Even without loading, mean_pool_zone returns a zero vec + vec = idx.mean_pool_zone([]) + assert vec.shape == (STATE_DIMS.zone_pool_dim,) + assert np.all(vec == 0.0) + + def test_encode_zones_shape_with_empty_players(self): + """_encode_zones must always return exactly total_zone_dim floats.""" + from ml.config.scope import STATE_DIMS + from ml.encoder.state_encoder import CardEmbeddingIndex, StateEncoder + + idx = CardEmbeddingIndex() + enc = StateEncoder(idx) + + # Two players with empty zones + players = [ + {"hand": [], "battlefield": [], "graveyard": [], "command_zone": []}, + {"hand": [], "battlefield": [], "graveyard": [], "command_zone": []}, + ] + zone_vec = enc._encode_zones(players) + assert zone_vec.shape == (STATE_DIMS.total_zone_dim,) + # All zeros because all zones are empty + assert np.all(zone_vec == 0.0) + + def test_full_encode_shape(self): + """Full state vector must be exactly total_state_dim.""" + from ml.config.scope import STATE_DIMS + from ml.encoder.state_encoder import CardEmbeddingIndex, StateEncoder + + idx = CardEmbeddingIndex() + enc = StateEncoder(idx) + + decision = { + "turn": 3, + "phase": "main_1", + "active_seat": 0, + "players": [ + { + "seat": 0, "life": 40, "cmdr_dmg": 0, "mana": 4, + "cmdr_tax": 0, "creatures": 2, "lands": 4, + "hand": ["Island", "Counterspell"], + "battlefield": ["Tundra"], + "graveyard": [], + "command_zone": [], + }, + { + "seat": 1, "life": 38, "cmdr_dmg": 0, "mana": 3, + "cmdr_tax": 0, "creatures": 1, "lands": 3, + "hand": [], + "battlefield": [], + "graveyard": [], + "command_zone": [], + }, + ], + } + state = enc.encode(decision, playstyle="control") + assert state.shape == (STATE_DIMS.total_state_dim,) + assert state.dtype == np.float32 + + def test_boundary_docstring_present(self): + """StateDimensions must document the zone embedding boundary.""" + from ml.config.scope import StateDimensions + assert "zone embedding boundary" in StateDimensions.__doc__.lower() + + +# ==================================================================== +# Decision 2: SQLite WAL Online Learning Store +# ==================================================================== + +class TestOnlineLearningStore: + """Verify the SQLite WAL-backed online learning store.""" + + @pytest.fixture + def store(self, tmp_path): + from ml.serving.online_learning_store import OnlineLearningStore + db_path = str(tmp_path / "test_online.db") + s = OnlineLearningStore(db_path=db_path) + s.init_db() + return s + + def test_init_creates_db(self, store): + assert os.path.exists(store.db_path) + + def test_wal_mode_enabled(self, store): + conn = store._get_conn() + mode = conn.execute("PRAGMA journal_mode").fetchone()[0] + assert mode == "wal" + + def test_record_and_fetch(self, store): + snapshot = {"turn": 1, "phase": "main_1", "players": []} + row_id = store.record_decision( + snapshot=snapshot, + action_idx=3, + confidence=0.85, + playstyle="control", + temperature=0.8, + greedy=False, + ) + assert row_id >= 1 + + decisions = store.fetch_decisions() + assert len(decisions) == 1 + d = decisions[0] + assert d["action_idx"] == 3 + assert d["confidence"] == pytest.approx(0.85) + assert d["playstyle"] == "control" + assert d["temperature"] == pytest.approx(0.8) + assert d["greedy"] == 0 + assert d["snapshot"]["turn"] == 1 + + def test_count(self, store): + assert store.count() == 0 + store.record_decision({"t": 1}, 0, 0.5) + store.record_decision({"t": 2}, 1, 0.6) + assert store.count() == 2 + assert store.count(only_unexported=True) == 2 + + def test_mark_exported(self, store): + store.record_decision({"t": 1}, 0, 0.5) + store.record_decision({"t": 2}, 1, 0.6) + store.record_decision({"t": 3}, 2, 0.7) + + # Mark first two as exported + updated = store.mark_exported(up_to_rowid=2) + assert updated == 2 + assert store.count(only_unexported=True) == 1 + + # Fetch only unexported + remaining = store.fetch_decisions(only_unexported=True) + assert len(remaining) == 1 + assert remaining[0]["action_idx"] == 2 + + def test_fetch_since_rowid(self, store): + ids = [] + for i in range(5): + ids.append(store.record_decision({"t": i}, i, 0.5)) + + # Fetch only rows after id 3 + results = store.fetch_decisions(since_rowid=ids[2], only_unexported=False) + assert len(results) == 2 # ids[3] and ids[4] + assert results[0]["action_idx"] == 3 + + def test_fetch_limit(self, store): + for i in range(10): + store.record_decision({"t": i}, i % 8, 0.5) + results = store.fetch_decisions(limit=3) + assert len(results) == 3 + + def test_close_and_reopen(self, store): + store.record_decision({"t": 1}, 0, 0.5) + store.close() + # Re-open by calling _get_conn again + assert store.count() == 1 + + def test_snapshot_stored_as_compact_json(self, store): + """Snapshots should be stored as compact JSON (no extra whitespace).""" + store.record_decision({"key": "value", "num": 42}, 0, 0.5) + conn = store._get_conn() + raw = conn.execute( + "SELECT snapshot FROM online_decisions WHERE id=1" + ).fetchone()[0] + # Compact JSON has no spaces after separators + assert " " not in raw + assert json.loads(raw) == {"key": "value", "num": 42} + + +# ==================================================================== +# Decision 3: Forge-Only Defaults + Mixed-Mode Presets +# ==================================================================== + +class TestForgeOnlyDefaults: + """Verify default weights are Forge-only (ppo_weight=0.0).""" + + def test_distillation_defaults_forge_only(self): + from ml.config.scope import DISTILLATION_DEFAULTS + assert DISTILLATION_DEFAULTS.forge_weight == 1.0 + assert DISTILLATION_DEFAULTS.ppo_weight == 0.0 + + def test_distillation_config_forge_only(self): + from ml.training.distillation_loop import DistillationConfig + cfg = DistillationConfig() + assert cfg.forge_weight == 1.0 + assert cfg.ppo_weight == 0.0 + + def test_dataset_config_forge_only(self): + from ml.data.dataset_builder import DatasetConfig + cfg = DatasetConfig() + assert cfg.source_weights["forge"] == 1.0 + assert cfg.source_weights["ppo"] == 0.0 + + +class TestMixedModePresets: + """Verify mixed-mode weight presets exist and are correct.""" + + def test_presets_dict_exists(self): + from ml.config.scope import MIXED_MODE_PRESETS + assert isinstance(MIXED_MODE_PRESETS, dict) + assert len(MIXED_MODE_PRESETS) == 3 + + def test_forge_only_preset(self): + from ml.config.scope import MIXED_MODE_PRESETS + p = MIXED_MODE_PRESETS["forge_only"] + assert p["forge_weight"] == 1.0 + assert p["ppo_weight"] == 0.0 + + def test_forge_90_10_preset(self): + from ml.config.scope import MIXED_MODE_PRESETS + p = MIXED_MODE_PRESETS["forge_90_10"] + assert p["forge_weight"] == 1.0 + # ppo_weight ~0.11 gives approximately 10% PPO in the mix + assert 0.1 <= p["ppo_weight"] <= 0.15 + + def test_forge_80_20_preset(self): + from ml.config.scope import MIXED_MODE_PRESETS + p = MIXED_MODE_PRESETS["forge_80_20"] + assert p["forge_weight"] == 1.0 + # ppo_weight ~0.25 gives approximately 20% PPO + assert 0.2 <= p["ppo_weight"] <= 0.3 + + def test_get_preset_weights_valid(self): + from ml.config.scope import get_preset_weights + w = get_preset_weights("forge_90_10") + assert "forge_weight" in w + assert "ppo_weight" in w + + def test_get_preset_weights_returns_copy(self): + """get_preset_weights must return a copy, not the original dict.""" + from ml.config.scope import get_preset_weights, MIXED_MODE_PRESETS + w = get_preset_weights("forge_only") + w["forge_weight"] = 999 + assert MIXED_MODE_PRESETS["forge_only"]["forge_weight"] == 1.0 + + def test_get_preset_weights_invalid(self): + from ml.config.scope import get_preset_weights + with pytest.raises(KeyError, match="Unknown preset"): + get_preset_weights("nonexistent") + + def test_all_presets_have_both_keys(self): + from ml.config.scope import MIXED_MODE_PRESETS + for name, preset in MIXED_MODE_PRESETS.items(): + assert "forge_weight" in preset, f"Preset '{name}' missing forge_weight" + assert "ppo_weight" in preset, f"Preset '{name}' missing ppo_weight"