feat: Phase 5 architecture follow-up — zone boundary, WAL store, Forge-only defaults (#87)
Implements the three approved Phase 5 architecture decisions: 1. Zone embedding boundary: add explicit documentation in StateDimensions and StateEncoder._encode_zones making the 6144-dim zero-padded zone contract visible to future maintainers. 2. SQLite WAL online learning store: new OnlineLearningStore class backed by WAL-mode SQLite for concurrent read/write on single-machine baseline. New /api/policy/decide endpoint collects decisions during inference. Stats endpoint at /api/policy/decide/stats. 3. Forge-only default weights: change ppo_weight default from 0.5 to 0.0 across scope.py, DistillationConfig, DatasetConfig, and CLI defaults. Add MIXED_MODE_PRESETS dict with forge_only, forge_90_10, forge_80_20 presets. Distill/start endpoint now accepts a "preset" parameter. 27 new tests covering all three decisions (all passing). Co-authored-by: Claude <claude@anthropic.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
7 files changed
+694
-14
No files matched your search
+49
-3
@@ -155,15 +155,25 @@ class StateDimensions:
|
||||
2 players = 28 global features
|
||||
+ turn_number = 1
|
||||
Total global = 29
|
||||
|
||||
Zone embedding boundary (architectural decision — Phase 5 follow-up):
|
||||
Zone dims are **always** 768 per zone slot, zero-padded when a zone
|
||||
is empty (no cards). The fixed 6144-dim zone block is the contract
|
||||
between ForgeEpisodeGenerator (upstream) and the policy network
|
||||
(downstream). All downstream consumers (PolicyNetwork, PPOTrainer,
|
||||
dataset builder) depend on this shape. If a future change adds
|
||||
attention pooling or variable-length zone representations, the
|
||||
boundary must be updated here *and* in StateEncoder._encode_zones.
|
||||
"""
|
||||
card_embedding_dim: int = 768 # From mtg-embeddings
|
||||
zone_pool_dim: int = 768 # Mean-pooled per zone
|
||||
zone_pool_dim: int = 768 # Mean-pooled per zone (zero-vec when empty)
|
||||
|
||||
# Zones per player: hand, battlefield, graveyard, command_zone
|
||||
zones_per_player: int = 4
|
||||
num_players: int = 2 # 1v1
|
||||
|
||||
# Zone vectors: 4 zones × 2 players × 768 = 6144
|
||||
# This is a fixed-size block; empty zones produce zero vectors.
|
||||
total_zone_dim: int = 4 * 2 * 768 # 6144
|
||||
|
||||
# Global scalar features
|
||||
@@ -268,6 +278,10 @@ class DistillationDefaults:
|
||||
|
||||
These are consumed by ml.training.distillation_loop.DistillationConfig
|
||||
when no explicit override is provided.
|
||||
|
||||
Default weights: Forge-only (forge_weight=1.0, ppo_weight=0.0).
|
||||
The baseline trains exclusively on Forge simulation data. Use
|
||||
``MIXED_MODE_PRESETS`` below for blended Forge+PPO configurations.
|
||||
"""
|
||||
max_iterations: int = 10
|
||||
convergence_window: int = 3
|
||||
@@ -281,9 +295,9 @@ class DistillationDefaults:
|
||||
ppo_entropy_coeff: float = 0.01
|
||||
ppo_eval_episodes: int = 100
|
||||
|
||||
# Dataset mixing
|
||||
# Dataset mixing — Forge-only by default (Phase 5 follow-up decision)
|
||||
forge_weight: float = 1.0
|
||||
ppo_weight: float = 0.5
|
||||
ppo_weight: float = 0.0
|
||||
min_reward_threshold: float = 0.0
|
||||
|
||||
# Quality gate thresholds
|
||||
@@ -302,3 +316,35 @@ class DistillationDefaults:
|
||||
|
||||
|
||||
DISTILLATION_DEFAULTS = DistillationDefaults()
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════
|
||||
# Mixed-Mode Presets (Phase 5 follow-up)
|
||||
# ══════════════════════════════════════════════════════════
|
||||
|
||||
# Pre-defined Forge/PPO weight combinations for common scenarios.
|
||||
# Select a preset by name when starting a distillation run to
|
||||
# override the Forge-only defaults without specifying raw weights.
|
||||
#
|
||||
# "forge_only" — baseline: 100% Forge data, ignore PPO
|
||||
# "forge_90_10" — light PPO blending: 90% Forge, 10% PPO
|
||||
# "forge_80_20" — heavier PPO blending: 80% Forge, 20% PPO
|
||||
MIXED_MODE_PRESETS: Dict[str, Dict[str, float]] = {
|
||||
"forge_only": {"forge_weight": 1.0, "ppo_weight": 0.0},
|
||||
"forge_90_10": {"forge_weight": 1.0, "ppo_weight": 0.11}, # ~10% PPO
|
||||
"forge_80_20": {"forge_weight": 1.0, "ppo_weight": 0.25}, # ~20% PPO
|
||||
}
|
||||
|
||||
|
||||
def get_preset_weights(preset_name: str) -> Dict[str, float]:
|
||||
"""Look up a mixed-mode preset by name.
|
||||
|
||||
Returns a dict with ``forge_weight`` and ``ppo_weight`` keys.
|
||||
Raises ``KeyError`` if the preset name is unknown.
|
||||
"""
|
||||
if preset_name not in MIXED_MODE_PRESETS:
|
||||
raise KeyError(
|
||||
f"Unknown preset '{preset_name}'. "
|
||||
f"Available: {list(MIXED_MODE_PRESETS.keys())}"
|
||||
)
|
||||
return dict(MIXED_MODE_PRESETS[preset_name])
|
||||
@@ -63,10 +63,10 @@ class DatasetConfig:
|
||||
max_samples: Optional[int] = None
|
||||
|
||||
# Source weights control sampling ratios between Forge and PPO data.
|
||||
# e.g. {"forge": 1.0, "ppo": 0.5} means keep all Forge data but
|
||||
# randomly sample 50% of PPO data to avoid overwhelming the baseline.
|
||||
# Default: Forge-only (ppo=0.0). Use MIXED_MODE_PRESETS from
|
||||
# ml.config.scope for blended configurations (e.g. 90/10, 80/20).
|
||||
source_weights: Dict[str, float] = field(
|
||||
default_factory=lambda: {"forge": 1.0, "ppo": 0.5}
|
||||
default_factory=lambda: {"forge": 1.0, "ppo": 0.0}
|
||||
)
|
||||
|
||||
# Minimum reward threshold for PPO data — only include decisions
|
||||
@@ -209,7 +209,7 @@ def build_dataset(
|
||||
Dict with keys: states, labels, game_ids, outcomes, playstyles, sources
|
||||
"""
|
||||
if source_weights is None:
|
||||
source_weights = {"forge": 1.0, "ppo": 0.5}
|
||||
source_weights = {"forge": 1.0, "ppo": 0.0}
|
||||
|
||||
# Load card embeddings
|
||||
card_index = CardEmbeddingIndex(embeddings_dir)
|
||||
@@ -502,8 +502,8 @@ def main():
|
||||
help="Sampling weight for Forge sim data (default: 1.0)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ppo-weight", type=float, default=0.5,
|
||||
help="Sampling weight for PPO self-play data (default: 0.5)",
|
||||
"--ppo-weight", type=float, default=0.0,
|
||||
help="Sampling weight for PPO self-play data (default: 0.0, Forge-only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-reward", type=float, default=0.0,
|
||||
|
||||
@@ -276,6 +276,13 @@ class StateEncoder:
|
||||
"""
|
||||
Encode zone contents as mean-pooled card embeddings.
|
||||
Shape: (4 zones × 2 players × 768) = (6144,)
|
||||
|
||||
Zone embedding boundary: each zone slot is exactly ``zone_pool_dim``
|
||||
(768) floats. Empty zones produce a zero vector of that size. This
|
||||
fixed-size contract is relied upon by every downstream consumer
|
||||
(PolicyNetwork, PPOTrainer, dataset builder). See
|
||||
``StateDimensions`` in ``ml/config/scope.py`` for the canonical
|
||||
definition.
|
||||
"""
|
||||
zone_keys = ["hand", "battlefield", "graveyard", "command_zone"]
|
||||
vectors = []
|
||||
@@ -283,6 +290,8 @@ class StateEncoder:
|
||||
for p in players[:2]:
|
||||
for zone_key in zone_keys:
|
||||
cards = p.get(zone_key, [])
|
||||
# mean_pool_zone returns a zero vector when cards is empty,
|
||||
# preserving the fixed 768-dim slot size.
|
||||
pooled = self.card_index.mean_pool_zone(cards)
|
||||
vectors.append(pooled)
|
||||
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""
|
||||
Commander AI Lab — Online Learning Store (SQLite WAL)
|
||||
═════════════════════════════════════════════════════
|
||||
|
||||
Single-machine backing store for online learning data collected during
|
||||
policy inference (``/api/ml/predict`` and ``/api/policy/decide``).
|
||||
|
||||
Each prediction request can optionally log the state snapshot and the
|
||||
chosen action so that future training runs can incorporate online
|
||||
decision data alongside Forge and PPO batches.
|
||||
|
||||
Design decisions (Phase 5 architecture follow-up):
|
||||
- SQLite in WAL mode for concurrent readers + single writer, which is
|
||||
the right fit for a single-machine baseline (one API server process).
|
||||
- Separate DB file (``online_learning.db``) to avoid contention with
|
||||
the collection database.
|
||||
- Append-only writes; reads are batched by the dataset builder.
|
||||
- Thread-safe via the Python sqlite3 module's ``check_same_thread=False``
|
||||
and WAL's built-in reader/writer concurrency.
|
||||
|
||||
Usage::
|
||||
|
||||
store = OnlineLearningStore() # uses default path
|
||||
store.init_db()
|
||||
|
||||
# At prediction time:
|
||||
store.record_decision(snapshot_json, action_index, confidence, playstyle)
|
||||
|
||||
# At training time:
|
||||
decisions = store.fetch_decisions(since_rowid=last_seen, limit=10000)
|
||||
store.mark_exported(max_rowid)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger("ml.serving.online_store")
|
||||
|
||||
_DEFAULT_DB_PATH = str(
|
||||
Path(__file__).parent.parent.parent / "online_learning.db"
|
||||
)
|
||||
|
||||
|
||||
class OnlineLearningStore:
|
||||
"""SQLite WAL-backed store for online policy decision collection."""
|
||||
|
||||
def __init__(self, db_path: str = None):
|
||||
self.db_path = db_path or _DEFAULT_DB_PATH
|
||||
self._local = threading.local()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Connection management
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _get_conn(self) -> sqlite3.Connection:
|
||||
"""Return a thread-local connection with WAL mode enabled."""
|
||||
if not hasattr(self._local, "conn") or self._local.conn is None:
|
||||
conn = sqlite3.connect(self.db_path, check_same_thread=False)
|
||||
conn.row_factory = sqlite3.Row
|
||||
# Enable WAL for concurrent read/write on single machine.
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
self._local.conn = conn
|
||||
return self._local.conn
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the thread-local connection."""
|
||||
conn = getattr(self._local, "conn", None)
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._local.conn = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Schema
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def init_db(self) -> None:
|
||||
"""Create the decisions table if it doesn't exist."""
|
||||
conn = self._get_conn()
|
||||
conn.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS online_decisions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
snapshot TEXT NOT NULL,
|
||||
action_idx INTEGER NOT NULL,
|
||||
confidence REAL NOT NULL DEFAULT 0.0,
|
||||
playstyle TEXT NOT NULL DEFAULT 'midrange',
|
||||
temperature REAL NOT NULL DEFAULT 1.0,
|
||||
greedy INTEGER NOT NULL DEFAULT 0,
|
||||
exported INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_od_exported
|
||||
ON online_decisions(exported);
|
||||
CREATE INDEX IF NOT EXISTS idx_od_created
|
||||
ON online_decisions(created_at);
|
||||
""")
|
||||
conn.commit()
|
||||
logger.info("Online learning store ready: %s (WAL mode)", self.db_path)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Write path (called during prediction)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def record_decision(
|
||||
self,
|
||||
snapshot: Dict,
|
||||
action_idx: int,
|
||||
confidence: float,
|
||||
playstyle: str = "midrange",
|
||||
temperature: float = 1.0,
|
||||
greedy: bool = False,
|
||||
) -> int:
|
||||
"""Append a policy decision for future training.
|
||||
|
||||
Returns the row id of the inserted record.
|
||||
"""
|
||||
conn = self._get_conn()
|
||||
cur = conn.execute(
|
||||
"""
|
||||
INSERT INTO online_decisions
|
||||
(snapshot, action_idx, confidence, playstyle, temperature, greedy)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
json.dumps(snapshot, separators=(",", ":")),
|
||||
action_idx,
|
||||
confidence,
|
||||
playstyle,
|
||||
temperature,
|
||||
1 if greedy else 0,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
return cur.lastrowid
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Read path (called by dataset builder / training pipeline)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def fetch_decisions(
|
||||
self,
|
||||
since_rowid: int = 0,
|
||||
limit: int = 50000,
|
||||
only_unexported: bool = True,
|
||||
) -> List[Dict]:
|
||||
"""Fetch collected decisions for training.
|
||||
|
||||
Args:
|
||||
since_rowid: Only return rows with id > this value.
|
||||
limit: Max rows to return.
|
||||
only_unexported: If True, skip rows already marked exported.
|
||||
|
||||
Returns:
|
||||
List of dicts with keys: id, snapshot (parsed), action_idx,
|
||||
confidence, playstyle, temperature, greedy, created_at.
|
||||
"""
|
||||
conn = self._get_conn()
|
||||
clauses = ["id > ?"]
|
||||
params: list = [since_rowid]
|
||||
if only_unexported:
|
||||
clauses.append("exported = 0")
|
||||
where = " AND ".join(clauses)
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM online_decisions WHERE {where} ORDER BY id LIMIT ?",
|
||||
(*params, limit),
|
||||
).fetchall()
|
||||
results = []
|
||||
for row in rows:
|
||||
d = dict(row)
|
||||
try:
|
||||
d["snapshot"] = json.loads(d["snapshot"])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
results.append(d)
|
||||
return results
|
||||
|
||||
def mark_exported(self, up_to_rowid: int) -> int:
|
||||
"""Mark decisions as exported so they aren't fetched again.
|
||||
|
||||
Returns the number of rows updated.
|
||||
"""
|
||||
conn = self._get_conn()
|
||||
cur = conn.execute(
|
||||
"UPDATE online_decisions SET exported = 1 WHERE id <= ? AND exported = 0",
|
||||
(up_to_rowid,),
|
||||
)
|
||||
conn.commit()
|
||||
return cur.rowcount
|
||||
|
||||
def count(self, only_unexported: bool = False) -> int:
|
||||
"""Return the total number of stored decisions."""
|
||||
conn = self._get_conn()
|
||||
if only_unexported:
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) FROM online_decisions WHERE exported = 0"
|
||||
).fetchone()
|
||||
else:
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) FROM online_decisions"
|
||||
).fetchone()
|
||||
return row[0]
|
||||
@@ -84,9 +84,9 @@ class DistillationConfig:
|
||||
opponent: str = "heuristic"
|
||||
playstyle: str = "midrange"
|
||||
|
||||
# --- Dataset building ---
|
||||
# --- Dataset building (Forge-only default; see MIXED_MODE_PRESETS) ---
|
||||
forge_weight: float = 1.0
|
||||
ppo_weight: float = 0.5
|
||||
ppo_weight: float = 0.0
|
||||
min_reward_threshold: float = 0.0
|
||||
|
||||
# --- Quality gate ---
|
||||
@@ -858,7 +858,7 @@ def main():
|
||||
|
||||
# Dataset
|
||||
parser.add_argument("--forge-weight", type=float, default=1.0)
|
||||
parser.add_argument("--ppo-weight", type=float, default=0.5)
|
||||
parser.add_argument("--ppo-weight", type=float, default=0.0)
|
||||
parser.add_argument("--min-reward", type=float, default=0.0)
|
||||
|
||||
# Quality gate
|
||||
|
||||
Reference in new issue
Block a user