Phase 6: ELO Tracking & Model Tournament (#70)
- Create ml/eval/elo_tracker.py:
- EloTracker class with standard ELO formula (K=32, start=1200)
- EloResult dataclass for tournament output
- EloHistory class for JSON persistence (data/elo_history.json)
- run_elo_tournament(): round-robin ELO-rated matches using
existing run_match() from tournament.py
- run_generation_tournament(): discovers gen-*/best_policy.pt
checkpoints and runs cross-generation ELO tournament
- CLI entry point: python -m ml.eval.elo_tracker
- Add ELO API endpoints to routes/ml.py:
- POST /api/ml/elo/tournament — start background ELO tournament
- GET /api/ml/elo/tournament/status — poll progress
- GET /api/ml/elo/history — load ELO history from disk
- Integrate ELO convergence in distillation_loop.py:
- New config fields: elo_convergence_enabled (default False),
elo_convergence_threshold (20.0), elo_tournament_episodes (30)
- New field: GenerationRecord.elo_rating
- _run_elo_mini_tournament() runs after quality gate pass
- _check_convergence() checks ELO delta when enabled
- Add ELO UI to distillation page:
- ELO Ratings panel with canvas multi-line chart showing per-model
ELO progression across generations
- Run Tournament button with progress indicator
- ELO leaderboard table with rank, rating, and visual bars
- Responsive chart resize handling
Closes #70
This commit is contained in:
rgriffith committed
1 parent
dd2e945214
commit
e843528d8b
6 files changed
+953
-1
No files matched your search
@@ -0,0 +1,407 @@
|
||||
"""
|
||||
Commander AI Lab — ELO Tracker & Cross-Generation Tournament
|
||||
═══════════════════════════════════════════════════════════════
|
||||
Phase 6 (Issue #70): Quantify improvement across distillation
|
||||
generations using ELO ratings.
|
||||
|
||||
- Runs round-robin tournaments between generation checkpoints
|
||||
and baselines (heuristic, random)
|
||||
- Computes ELO ratings using the standard formula
|
||||
- Persists ELO history to `data/elo_history.json`
|
||||
- Provides ELO delta for convergence detection
|
||||
|
||||
Reuses `run_match`, `LearnedPolicy`, `HeuristicPolicy`,
|
||||
`RandomPolicy` from `ml/eval/tournament.py`.
|
||||
|
||||
Usage:
|
||||
python -m ml.eval.elo_tracker
|
||||
python -m ml.eval.elo_tracker --episodes 50 --output data/elo_history.json
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from itertools import combinations
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
project_root = str(Path(__file__).parent.parent.parent)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from ml.eval.tournament import (
|
||||
run_match, LearnedPolicy, HeuristicPolicy, RandomPolicy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("ml.eval.elo")
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# ELO Rating Engine
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
DEFAULT_K = 32
|
||||
DEFAULT_START_ELO = 1200.0
|
||||
|
||||
|
||||
class EloTracker:
|
||||
"""Standard ELO rating tracker."""
|
||||
|
||||
def __init__(self, k_factor: float = DEFAULT_K, start_elo: float = DEFAULT_START_ELO):
|
||||
self.k = k_factor
|
||||
self.start_elo = start_elo
|
||||
self.ratings: Dict[str, float] = {}
|
||||
|
||||
def ensure_player(self, name: str):
|
||||
if name not in self.ratings:
|
||||
self.ratings[name] = self.start_elo
|
||||
|
||||
def expected_score(self, rating_a: float, rating_b: float) -> float:
|
||||
return 1.0 / (1.0 + math.pow(10, (rating_b - rating_a) / 400.0))
|
||||
|
||||
def update(self, winner: str, loser: str):
|
||||
"""Update ratings after a decisive match."""
|
||||
self.ensure_player(winner)
|
||||
self.ensure_player(loser)
|
||||
|
||||
e_win = self.expected_score(self.ratings[winner], self.ratings[loser])
|
||||
e_lose = self.expected_score(self.ratings[loser], self.ratings[winner])
|
||||
|
||||
self.ratings[winner] += self.k * (1.0 - e_win)
|
||||
self.ratings[loser] += self.k * (0.0 - e_lose)
|
||||
|
||||
def update_draw(self, player_a: str, player_b: str):
|
||||
"""Update ratings after a draw."""
|
||||
self.ensure_player(player_a)
|
||||
self.ensure_player(player_b)
|
||||
|
||||
e_a = self.expected_score(self.ratings[player_a], self.ratings[player_b])
|
||||
e_b = self.expected_score(self.ratings[player_b], self.ratings[player_a])
|
||||
|
||||
self.ratings[player_a] += self.k * (0.5 - e_a)
|
||||
self.ratings[player_b] += self.k * (0.5 - e_b)
|
||||
|
||||
def get_ratings(self) -> Dict[str, float]:
|
||||
"""Return a copy of current ratings, rounded to 1 decimal."""
|
||||
return {k: round(v, 1) for k, v in self.ratings.items()}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# ELO Result
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
@dataclass
|
||||
class EloResult:
|
||||
"""Result of an ELO-rated tournament."""
|
||||
ratings: Dict[str, float] = field(default_factory=dict)
|
||||
match_count: int = 0
|
||||
total_time_s: float = 0.0
|
||||
policies: List[str] = field(default_factory=list)
|
||||
rating_progression: List[Dict[str, float]] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"ratings": self.ratings,
|
||||
"match_count": self.match_count,
|
||||
"total_time_s": round(self.total_time_s, 1),
|
||||
"policies": self.policies,
|
||||
"rating_progression": self.rating_progression,
|
||||
}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# ELO History (persistence)
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
class EloHistory:
|
||||
"""Persists ELO ratings across tournament runs."""
|
||||
|
||||
def __init__(self, path: str = "data/elo_history.json"):
|
||||
self.path = path
|
||||
self.entries: List[dict] = []
|
||||
self._load()
|
||||
|
||||
def _load(self):
|
||||
if os.path.exists(self.path):
|
||||
try:
|
||||
with open(self.path, "r") as f:
|
||||
data = json.load(f)
|
||||
self.entries = data.get("entries", [])
|
||||
except Exception:
|
||||
self.entries = []
|
||||
|
||||
def save(self):
|
||||
os.makedirs(os.path.dirname(self.path) or ".", exist_ok=True)
|
||||
with open(self.path, "w") as f:
|
||||
json.dump({"entries": self.entries}, f, indent=2)
|
||||
|
||||
def append(self, generation: int, ratings: Dict[str, float]):
|
||||
"""Add a new ELO snapshot after a tournament."""
|
||||
self.entries.append({
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"generation": generation,
|
||||
"ratings": {k: round(v, 1) for k, v in ratings.items()},
|
||||
})
|
||||
self.save()
|
||||
|
||||
def get_elo_delta(self, window: int = 2) -> Optional[float]:
|
||||
"""
|
||||
Compute average ELO gain of the latest generation model
|
||||
over the last `window` entries.
|
||||
|
||||
Returns None if insufficient data.
|
||||
"""
|
||||
if len(self.entries) < window:
|
||||
return None
|
||||
|
||||
recent = self.entries[-window:]
|
||||
deltas = []
|
||||
|
||||
for i in range(1, len(recent)):
|
||||
prev_ratings = recent[i - 1]["ratings"]
|
||||
curr_ratings = recent[i]["ratings"]
|
||||
|
||||
# Find the latest generation model name in current entry
|
||||
gen_models = [k for k in curr_ratings if k.startswith("gen-")]
|
||||
if not gen_models:
|
||||
continue
|
||||
|
||||
latest_gen = sorted(gen_models)[-1]
|
||||
curr_elo = curr_ratings.get(latest_gen, DEFAULT_START_ELO)
|
||||
|
||||
# Compare to previous entry's latest gen model, or heuristic
|
||||
prev_gen_models = [k for k in prev_ratings if k.startswith("gen-")]
|
||||
if prev_gen_models:
|
||||
prev_latest = sorted(prev_gen_models)[-1]
|
||||
prev_elo = prev_ratings.get(prev_latest, DEFAULT_START_ELO)
|
||||
else:
|
||||
prev_elo = prev_ratings.get("heuristic", DEFAULT_START_ELO)
|
||||
|
||||
deltas.append(curr_elo - prev_elo)
|
||||
|
||||
if not deltas:
|
||||
return None
|
||||
|
||||
return sum(deltas) / len(deltas)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {"entries": self.entries}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict, path: str = "data/elo_history.json") -> "EloHistory":
|
||||
h = cls.__new__(cls)
|
||||
h.path = path
|
||||
h.entries = data.get("entries", [])
|
||||
return h
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# Tournament Functions
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def run_elo_tournament(
|
||||
policies: Dict[str, object],
|
||||
episodes_per_matchup: int = 50,
|
||||
playstyle: str = "midrange",
|
||||
k_factor: float = DEFAULT_K,
|
||||
) -> EloResult:
|
||||
"""
|
||||
Run a round-robin ELO-rated tournament.
|
||||
|
||||
Uses `run_match` from tournament.py for match execution.
|
||||
|
||||
Args:
|
||||
policies: Dict mapping name → policy object (with select_action method)
|
||||
episodes_per_matchup: Games per pair (played both sides)
|
||||
playstyle: Game playstyle
|
||||
k_factor: ELO K-factor
|
||||
|
||||
Returns:
|
||||
EloResult with final ratings, match count, and progression
|
||||
"""
|
||||
t_start = time.time()
|
||||
names = list(policies.keys())
|
||||
tracker = EloTracker(k_factor=k_factor)
|
||||
|
||||
# Initialize all players
|
||||
for name in names:
|
||||
tracker.ensure_player(name)
|
||||
|
||||
progression = [tracker.get_ratings().copy()]
|
||||
total_matches = 0
|
||||
|
||||
logger.info("ELO Tournament: %d policies, %d games/matchup", len(names), episodes_per_matchup)
|
||||
|
||||
for name_a, name_b in combinations(names, 2):
|
||||
policy_a = policies[name_a]
|
||||
policy_b = policies[name_b]
|
||||
|
||||
for ep in range(episodes_per_matchup):
|
||||
# Game 1: A as seat 0
|
||||
m1 = run_match(policy_a, name_a, policy_b, name_b, playstyle=playstyle)
|
||||
total_matches += 1
|
||||
if m1.winner == name_a:
|
||||
tracker.update(name_a, name_b)
|
||||
elif m1.winner == name_b:
|
||||
tracker.update(name_b, name_a)
|
||||
else:
|
||||
tracker.update_draw(name_a, name_b)
|
||||
|
||||
# Game 2: B as seat 0
|
||||
m2 = run_match(policy_b, name_b, policy_a, name_a, playstyle=playstyle)
|
||||
total_matches += 1
|
||||
if m2.winner == name_a:
|
||||
tracker.update(name_a, name_b)
|
||||
elif m2.winner == name_b:
|
||||
tracker.update(name_b, name_a)
|
||||
else:
|
||||
tracker.update_draw(name_a, name_b)
|
||||
|
||||
# Record progression after each matchup
|
||||
progression.append(tracker.get_ratings().copy())
|
||||
|
||||
logger.info(
|
||||
" %s (%.0f) vs %s (%.0f)",
|
||||
name_a, tracker.ratings[name_a],
|
||||
name_b, tracker.ratings[name_b],
|
||||
)
|
||||
|
||||
return EloResult(
|
||||
ratings=tracker.get_ratings(),
|
||||
match_count=total_matches,
|
||||
total_time_s=time.time() - t_start,
|
||||
policies=names,
|
||||
rating_progression=progression,
|
||||
)
|
||||
|
||||
|
||||
def run_generation_tournament(
|
||||
checkpoint_dir: str = "ml/models/checkpoints",
|
||||
episodes_per_matchup: int = 50,
|
||||
playstyle: str = "midrange",
|
||||
k_factor: float = DEFAULT_K,
|
||||
) -> EloResult:
|
||||
"""
|
||||
Discover generation checkpoints and run an ELO tournament.
|
||||
|
||||
Looks for `gen-*/best_policy.pt` under checkpoint_dir.
|
||||
Also includes heuristic and random baselines.
|
||||
"""
|
||||
policies: Dict[str, object] = {
|
||||
"heuristic": HeuristicPolicy(),
|
||||
"random": RandomPolicy(),
|
||||
}
|
||||
|
||||
ckpt_path = Path(checkpoint_dir)
|
||||
if ckpt_path.exists():
|
||||
for gen_dir in sorted(ckpt_path.glob("gen-*")):
|
||||
model_path = gen_dir / "best_policy.pt"
|
||||
if model_path.exists():
|
||||
name = gen_dir.name # e.g., "gen-001"
|
||||
policies[name] = LearnedPolicy(str(model_path))
|
||||
logger.info("Loaded %s: %s", name, model_path)
|
||||
|
||||
# Also check standard checkpoints
|
||||
sup_path = ckpt_path / "best_policy.pt"
|
||||
if sup_path.exists() and "supervised" not in policies:
|
||||
policies["supervised"] = LearnedPolicy(str(sup_path))
|
||||
|
||||
ppo_path = ckpt_path / "best_ppo.pt"
|
||||
if ppo_path.exists() and "ppo" not in policies:
|
||||
policies["ppo"] = LearnedPolicy(str(ppo_path))
|
||||
|
||||
if len(policies) < 2:
|
||||
logger.warning("Need at least 2 policies for ELO tournament")
|
||||
return EloResult(
|
||||
ratings={k: DEFAULT_START_ELO for k in policies},
|
||||
policies=list(policies.keys()),
|
||||
)
|
||||
|
||||
logger.info("Running ELO tournament with %d policies: %s", len(policies), list(policies.keys()))
|
||||
|
||||
return run_elo_tournament(
|
||||
policies=policies,
|
||||
episodes_per_matchup=episodes_per_matchup,
|
||||
playstyle=playstyle,
|
||||
k_factor=k_factor,
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# CLI
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def main():
|
||||
"""CLI entry point for ELO tournament."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Commander AI Lab — ELO Tournament"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--episodes", type=int, default=50,
|
||||
help="Episodes per matchup (default: 50)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--checkpoint-dir", default="ml/models/checkpoints",
|
||||
help="Checkpoint directory (default: ml/models/checkpoints)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--playstyle", default="midrange",
|
||||
help="Playstyle (default: midrange)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", default="data/elo_history.json",
|
||||
help="Output path for ELO history (default: data/elo_history.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--k-factor", type=float, default=32,
|
||||
help="ELO K-factor (default: 32)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
|
||||
result = run_generation_tournament(
|
||||
checkpoint_dir=args.checkpoint_dir,
|
||||
episodes_per_matchup=args.episodes,
|
||||
playstyle=args.playstyle,
|
||||
k_factor=args.k_factor,
|
||||
)
|
||||
|
||||
# Print results
|
||||
logger.info("")
|
||||
logger.info("=" * 50)
|
||||
logger.info(" ELO RATINGS")
|
||||
logger.info("=" * 50)
|
||||
sorted_ratings = sorted(result.ratings.items(), key=lambda x: x[1], reverse=True)
|
||||
for name, elo in sorted_ratings:
|
||||
logger.info(" %-20s %7.1f", name, elo)
|
||||
logger.info("")
|
||||
logger.info(" %d matches in %.1f s", result.match_count, result.total_time_s)
|
||||
logger.info("=" * 50)
|
||||
|
||||
# Save to history
|
||||
history = EloHistory(path=args.output)
|
||||
# Use the latest generation number, or 0 if no gen models
|
||||
gen_models = [k for k in result.ratings if k.startswith("gen-")]
|
||||
gen_num = 0
|
||||
if gen_models:
|
||||
try:
|
||||
gen_num = max(int(g.split("-")[1]) for g in gen_models)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
history.append(gen_num, result.ratings)
|
||||
logger.info("ELO history saved: %s", args.output)
|
||||
|
||||
print(json.dumps(result.to_dict(), indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -99,6 +99,11 @@ class DistillationConfig:
|
||||
retry_lr_factor: float = 0.5 # Halve PPO LR on retry
|
||||
retry_entropy_boost: float = 0.005 # Increase entropy coeff on retry
|
||||
|
||||
# --- ELO convergence (Phase 6) ---
|
||||
elo_convergence_enabled: bool = False # Off by default
|
||||
elo_convergence_threshold: float = 20.0 # Stop if ELO gain < this
|
||||
elo_tournament_episodes: int = 30 # Games per matchup for ELO mini-tournament
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# Generation metadata
|
||||
@@ -137,6 +142,7 @@ class GenerationRecord:
|
||||
|
||||
retries: int = 0
|
||||
error: str = ""
|
||||
elo_rating: float = 0.0
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {k: v for k, v in self.__dict__.items()}
|
||||
@@ -393,6 +399,15 @@ class DistillationLoop:
|
||||
"in next generation's dataset build."
|
||||
)
|
||||
record.status = "passed"
|
||||
|
||||
# ─── ELO mini-tournament (Phase 6) ────────────────
|
||||
if cfg.elo_convergence_enabled and gen_num > 1:
|
||||
elo = self._run_elo_mini_tournament(
|
||||
gen_checkpoint_dir, gen_num
|
||||
)
|
||||
if elo is not None:
|
||||
record.elo_rating = elo
|
||||
logger.info(" ELO rating: %.1f", elo)
|
||||
else:
|
||||
record.status = "failed"
|
||||
|
||||
@@ -611,6 +626,66 @@ class DistillationLoop:
|
||||
},
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# ELO mini-tournament (Phase 6)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run_elo_mini_tournament(
|
||||
self, gen_checkpoint_dir: str, gen_num: int
|
||||
) -> Optional[float]:
|
||||
"""
|
||||
Run a small ELO tournament between current gen, previous gen,
|
||||
and heuristic baseline. Returns the current gen's ELO rating.
|
||||
"""
|
||||
cfg = self.config
|
||||
try:
|
||||
from ml.eval.elo_tracker import (
|
||||
run_elo_tournament, EloHistory, HeuristicPolicy, LearnedPolicy,
|
||||
)
|
||||
|
||||
policies = {"heuristic": HeuristicPolicy()}
|
||||
|
||||
# Current generation checkpoint
|
||||
curr_path = os.path.join(gen_checkpoint_dir, "best_policy.pt")
|
||||
curr_name = f"gen-{gen_num:03d}"
|
||||
if os.path.exists(curr_path):
|
||||
policies[curr_name] = LearnedPolicy(curr_path)
|
||||
else:
|
||||
logger.warning(" ELO: current gen checkpoint not found: %s", curr_path)
|
||||
return None
|
||||
|
||||
# Previous generation checkpoint
|
||||
if gen_num > 1:
|
||||
prev_tag = f"gen-{gen_num - 1:03d}"
|
||||
prev_dir = os.path.join(cfg.checkpoint_dir, prev_tag)
|
||||
prev_path = os.path.join(prev_dir, "best_policy.pt")
|
||||
if os.path.exists(prev_path):
|
||||
policies[prev_tag] = LearnedPolicy(prev_path)
|
||||
|
||||
logger.info(
|
||||
" [ELO] Running mini-tournament: %s (%d episodes/matchup)",
|
||||
list(policies.keys()), cfg.elo_tournament_episodes,
|
||||
)
|
||||
|
||||
result = run_elo_tournament(
|
||||
policies=policies,
|
||||
episodes_per_matchup=cfg.elo_tournament_episodes,
|
||||
playstyle=cfg.playstyle,
|
||||
)
|
||||
|
||||
# Save to ELO history
|
||||
history_path = os.path.join(
|
||||
os.path.dirname(cfg.history_dir), "elo_history.json"
|
||||
) if cfg.history_dir else "data/elo_history.json"
|
||||
history = EloHistory(path=history_path)
|
||||
history.append(gen_num, result.ratings)
|
||||
|
||||
return result.ratings.get(curr_name, 0.0)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(" ELO mini-tournament failed: %s", e)
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Convergence detection
|
||||
# ------------------------------------------------------------------
|
||||
@@ -618,6 +693,7 @@ class DistillationLoop:
|
||||
def _check_convergence(self) -> bool:
|
||||
"""
|
||||
Check if win rate has plateaued over recent generations.
|
||||
Also checks ELO delta when elo_convergence_enabled is True.
|
||||
|
||||
Returns True if the loop should stop.
|
||||
"""
|
||||
@@ -642,7 +718,22 @@ class DistillationLoop:
|
||||
cfg.convergence_threshold,
|
||||
)
|
||||
|
||||
return improvement < cfg.convergence_threshold
|
||||
win_rate_converged = improvement < cfg.convergence_threshold
|
||||
|
||||
# ELO convergence check (Phase 6)
|
||||
if cfg.elo_convergence_enabled and len(passed) >= 2:
|
||||
elo_ratings = [g.elo_rating for g in passed[-2:] if g.elo_rating > 0]
|
||||
if len(elo_ratings) >= 2:
|
||||
elo_delta = elo_ratings[-1] - elo_ratings[-2]
|
||||
logger.info(
|
||||
" ELO convergence check: delta = %.1f (threshold = %.1f)",
|
||||
elo_delta, cfg.elo_convergence_threshold,
|
||||
)
|
||||
if elo_delta < cfg.elo_convergence_threshold:
|
||||
logger.info(" ELO delta below threshold — converged.")
|
||||
return True
|
||||
|
||||
return win_rate_converged
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Device detection
|
||||
|
||||
+129
@@ -945,3 +945,132 @@ async def ml_distillation_history():
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ==============================================================
|
||||
# ELO Tournament Endpoints
|
||||
# ==============================================================
|
||||
|
||||
@dataclasses.dataclass
|
||||
class _EloTournamentState:
|
||||
running: bool = False
|
||||
phase: str = "idle"
|
||||
message: str = ""
|
||||
result: object = None
|
||||
error: object = None
|
||||
|
||||
def snapshot(self) -> dict:
|
||||
return {
|
||||
"running": self.running,
|
||||
"phase": self.phase,
|
||||
"message": self.message,
|
||||
"result": self.result,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
_elo_tournament_state = _EloTournamentState()
|
||||
_elo_tournament_lock = threading.Lock()
|
||||
|
||||
|
||||
def _run_elo_tournament_pipeline(episodes: int, playstyle: str):
|
||||
"""Run ELO tournament in a background thread."""
|
||||
try:
|
||||
import sys
|
||||
project_root = str(Path(__file__).resolve().parent.parent)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
with _elo_tournament_lock:
|
||||
_elo_tournament_state.running = True
|
||||
_elo_tournament_state.phase = "running"
|
||||
_elo_tournament_state.message = "Running ELO tournament..."
|
||||
_elo_tournament_state.error = None
|
||||
_elo_tournament_state.result = None
|
||||
|
||||
from ml.eval.elo_tracker import (
|
||||
run_generation_tournament, EloHistory,
|
||||
)
|
||||
|
||||
ckpt_dir = os.path.join(project_root, "ml", "models", "checkpoints")
|
||||
result = run_generation_tournament(
|
||||
checkpoint_dir=ckpt_dir,
|
||||
episodes_per_matchup=episodes,
|
||||
playstyle=playstyle,
|
||||
)
|
||||
|
||||
# Save to ELO history
|
||||
history_path = os.path.join(project_root, "data", "elo_history.json")
|
||||
history = EloHistory(path=history_path)
|
||||
gen_models = [k for k in result.ratings if k.startswith("gen-")]
|
||||
gen_num = 0
|
||||
if gen_models:
|
||||
try:
|
||||
gen_num = max(int(g.split("-")[1]) for g in gen_models)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
history.append(gen_num, result.ratings)
|
||||
|
||||
with _elo_tournament_lock:
|
||||
_elo_tournament_state.phase = "done"
|
||||
_elo_tournament_state.running = False
|
||||
_elo_tournament_state.result = result.to_dict()
|
||||
_elo_tournament_state.message = (
|
||||
f"ELO tournament complete! {result.match_count} matches in {result.total_time_s:.1f}s"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
with _elo_tournament_lock:
|
||||
_elo_tournament_state.phase = "error"
|
||||
_elo_tournament_state.running = False
|
||||
_elo_tournament_state.error = str(e)
|
||||
_elo_tournament_state.message = f"ELO tournament failed: {e}"
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
@router.post("/api/ml/elo/tournament")
|
||||
async def ml_start_elo_tournament(request: FastAPIRequest):
|
||||
"""Start an ELO-rated tournament across all generation checkpoints."""
|
||||
if _elo_tournament_state.running:
|
||||
raise HTTPException(409, "ELO tournament already in progress")
|
||||
|
||||
body = await request.json() if await request.body() else {}
|
||||
episodes = body.get("episodes", 50)
|
||||
playstyle = body.get("playstyle", "midrange")
|
||||
|
||||
with _elo_tournament_lock:
|
||||
_elo_tournament_state.running = True
|
||||
_elo_tournament_state.phase = "starting"
|
||||
_elo_tournament_state.message = "Initializing ELO tournament..."
|
||||
_elo_tournament_state.result = None
|
||||
_elo_tournament_state.error = None
|
||||
|
||||
thread = threading.Thread(
|
||||
target=_run_elo_tournament_pipeline,
|
||||
args=(episodes, playstyle),
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
|
||||
return {"status": "started", "episodes": episodes}
|
||||
|
||||
|
||||
@router.get("/api/ml/elo/tournament/status")
|
||||
async def ml_elo_tournament_status():
|
||||
"""Get ELO tournament status."""
|
||||
with _elo_tournament_lock:
|
||||
return _elo_tournament_state.snapshot()
|
||||
|
||||
|
||||
@router.get("/api/ml/elo/history")
|
||||
async def ml_elo_history():
|
||||
"""Get ELO rating history."""
|
||||
project_root = Path(__file__).resolve().parent.parent
|
||||
history_path = project_root / "data" / "elo_history.json"
|
||||
if history_path.exists():
|
||||
try:
|
||||
with open(history_path, "r") as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
pass
|
||||
return {"entries": []}
|
||||
@@ -187,6 +187,21 @@
|
||||
}
|
||||
}
|
||||
|
||||
/* ── ELO Panel ─────────────────────────────────────────── */
|
||||
.elo-panel-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
.elo-table {
|
||||
margin-top: var(--sp-3);
|
||||
}
|
||||
|
||||
.elo-table td {
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
/* Refresh button spin animation */
|
||||
@keyframes spin {
|
||||
from { transform: rotate(0deg); }
|
||||
|
||||
@@ -161,6 +161,32 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ELO Rating Chart -->
|
||||
<section class="panel" id="elo-panel">
|
||||
<div class="panel-header elo-panel-header">
|
||||
<h2>ELO Ratings</h2>
|
||||
<button id="btn-run-elo" class="btn btn-sm btn-secondary" onclick="runEloTournament()">
|
||||
<span class="btn-icon">🏆</span> Run Tournament
|
||||
</button>
|
||||
</div>
|
||||
<div id="elo-chart" class="chart-container">
|
||||
<canvas id="elo-canvas"></canvas>
|
||||
<div id="elo-empty" class="empty-state">
|
||||
<p>No ELO data yet. Run a tournament or enable ELO tracking in distillation settings.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div id="elo-progress" class="train-progress hidden">
|
||||
<div class="progress-header">
|
||||
<span id="elo-progress-phase" class="phase-badge running">RUNNING</span>
|
||||
<span id="elo-progress-message">Running ELO tournament...</span>
|
||||
</div>
|
||||
<div class="progress-bar-container progress-track">
|
||||
<div class="progress-bar progress-fill" id="elo-progress-bar" style="width:50%"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="elo-leaderboard" class="mt-4"></div>
|
||||
</section>
|
||||
|
||||
<!-- Generation History Table -->
|
||||
<section class="panel" id="history-panel">
|
||||
<div class="panel-header">
|
||||
|
||||
@@ -30,6 +30,7 @@ async function refreshAll() {
|
||||
try {
|
||||
await refreshStatus();
|
||||
await loadHistory();
|
||||
await loadEloHistory();
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
icon.style.animation = '';
|
||||
@@ -627,6 +628,286 @@ function renderHistoryTable(generations) {
|
||||
container.innerHTML = html;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════
|
||||
// ELO Tournament
|
||||
// ═══════════════════════════════════════════
|
||||
|
||||
let eloPollingInterval = null;
|
||||
let eloHistoryData = null;
|
||||
|
||||
async function runEloTournament() {
|
||||
const btn = document.getElementById('btn-run-elo');
|
||||
btn.disabled = true;
|
||||
btn.innerHTML = '<span class="btn-icon">⌛</span> Running...';
|
||||
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}/api/ml/elo/tournament`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ episodes: 50, playstyle: 'midrange' }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = await res.json();
|
||||
throw new Error(err.detail || `HTTP ${res.status}`);
|
||||
}
|
||||
|
||||
document.getElementById('elo-progress').classList.remove('hidden');
|
||||
startEloPolling();
|
||||
|
||||
} catch (err) {
|
||||
alert('Failed to start ELO tournament: ' + err.message);
|
||||
btn.disabled = false;
|
||||
btn.innerHTML = '<span class="btn-icon">🏆</span> Run Tournament';
|
||||
}
|
||||
}
|
||||
|
||||
function startEloPolling() {
|
||||
if (eloPollingInterval) clearInterval(eloPollingInterval);
|
||||
eloPollingInterval = setInterval(pollEloStatus, 3000);
|
||||
}
|
||||
|
||||
async function pollEloStatus() {
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}/api/ml/elo/tournament/status`);
|
||||
const data = await res.json();
|
||||
|
||||
const phase = data.phase || 'idle';
|
||||
const phaseEl = document.getElementById('elo-progress-phase');
|
||||
phaseEl.textContent = phase.toUpperCase();
|
||||
phaseEl.className = 'phase-badge ' + phase;
|
||||
document.getElementById('elo-progress-message').textContent = data.message || '';
|
||||
|
||||
const bar = document.getElementById('elo-progress-bar');
|
||||
if (phase === 'done') {
|
||||
bar.style.width = '100%';
|
||||
bar.className = 'progress-bar progress-fill done';
|
||||
} else if (phase === 'error') {
|
||||
bar.style.width = '100%';
|
||||
bar.className = 'progress-bar progress-fill error';
|
||||
}
|
||||
|
||||
if (!data.running && phase !== 'idle') {
|
||||
clearInterval(eloPollingInterval);
|
||||
eloPollingInterval = null;
|
||||
|
||||
const btn = document.getElementById('btn-run-elo');
|
||||
btn.disabled = false;
|
||||
btn.innerHTML = '<span class="btn-icon">🏆</span> Run Tournament';
|
||||
|
||||
if (phase === 'done' && data.result) {
|
||||
renderEloLeaderboard(data.result.ratings);
|
||||
}
|
||||
|
||||
// Hide progress after a moment, reload history
|
||||
setTimeout(() => {
|
||||
document.getElementById('elo-progress').classList.add('hidden');
|
||||
loadEloHistory();
|
||||
}, 2000);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('ELO poll error:', err);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadEloHistory() {
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}/api/ml/elo/history`);
|
||||
if (!res.ok) return;
|
||||
const data = await res.json();
|
||||
|
||||
if (data.entries && data.entries.length > 0) {
|
||||
eloHistoryData = data.entries;
|
||||
renderEloChart(data.entries);
|
||||
|
||||
// Render leaderboard from latest entry
|
||||
const latest = data.entries[data.entries.length - 1];
|
||||
if (latest.ratings) {
|
||||
renderEloLeaderboard(latest.ratings);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to load ELO history:', err);
|
||||
}
|
||||
}
|
||||
|
||||
function renderEloChart(entries) {
|
||||
const canvas = document.getElementById('elo-canvas');
|
||||
const empty = document.getElementById('elo-empty');
|
||||
|
||||
if (!entries || entries.length === 0) {
|
||||
canvas.style.display = 'none';
|
||||
empty.style.display = '';
|
||||
return;
|
||||
}
|
||||
|
||||
canvas.style.display = 'block';
|
||||
empty.style.display = 'none';
|
||||
|
||||
const ctx = canvas.getContext('2d');
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
const rect = canvas.parentElement.getBoundingClientRect();
|
||||
const w = rect.width;
|
||||
const h = 280;
|
||||
|
||||
canvas.width = w * dpr;
|
||||
canvas.height = h * dpr;
|
||||
canvas.style.width = w + 'px';
|
||||
canvas.style.height = h + 'px';
|
||||
ctx.scale(dpr, dpr);
|
||||
ctx.clearRect(0, 0, w, h);
|
||||
|
||||
const pad = { top: 30, right: 30, bottom: 40, left: 55 };
|
||||
const cw = w - pad.left - pad.right;
|
||||
const ch = h - pad.top - pad.bottom;
|
||||
|
||||
// Collect all model names across entries
|
||||
const allModels = new Set();
|
||||
entries.forEach(e => Object.keys(e.ratings).forEach(k => allModels.add(k)));
|
||||
const modelNames = Array.from(allModels).sort();
|
||||
|
||||
// Color palette for models
|
||||
const colors = {
|
||||
'heuristic': '#fbbf24',
|
||||
'random': '#8b90a0',
|
||||
'supervised': '#5b9ef0',
|
||||
'ppo': '#a78bfa',
|
||||
};
|
||||
const genColors = ['#4ade80', '#f472b6', '#22d3ee', '#fb923c', '#e879f9', '#34d399', '#f87171', '#60a5fa'];
|
||||
let genColorIdx = 0;
|
||||
modelNames.forEach(name => {
|
||||
if (!colors[name]) {
|
||||
colors[name] = genColors[genColorIdx % genColors.length];
|
||||
genColorIdx++;
|
||||
}
|
||||
});
|
||||
|
||||
// Find ELO range
|
||||
let minElo = Infinity, maxElo = -Infinity;
|
||||
entries.forEach(e => {
|
||||
Object.values(e.ratings).forEach(v => {
|
||||
minElo = Math.min(minElo, v);
|
||||
maxElo = Math.max(maxElo, v);
|
||||
});
|
||||
});
|
||||
const eloPad = 50;
|
||||
minElo = Math.floor((minElo - eloPad) / 50) * 50;
|
||||
maxElo = Math.ceil((maxElo + eloPad) / 50) * 50;
|
||||
const eloRange = maxElo - minElo || 100;
|
||||
|
||||
// Grid lines
|
||||
ctx.strokeStyle = '#222533';
|
||||
ctx.lineWidth = 1;
|
||||
const ySteps = 5;
|
||||
for (let i = 0; i <= ySteps; i++) {
|
||||
const y = pad.top + (ch / ySteps) * i;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(pad.left, y);
|
||||
ctx.lineTo(pad.left + cw, y);
|
||||
ctx.stroke();
|
||||
|
||||
const val = maxElo - (eloRange * i / ySteps);
|
||||
ctx.fillStyle = '#8b90a0';
|
||||
ctx.font = '11px Inter, sans-serif';
|
||||
ctx.textAlign = 'right';
|
||||
ctx.fillText(Math.round(val).toString(), pad.left - 8, y + 4);
|
||||
}
|
||||
|
||||
// X axis labels
|
||||
ctx.textAlign = 'center';
|
||||
ctx.fillStyle = '#8b90a0';
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
const x = pad.left + (cw / Math.max(1, entries.length - 1)) * i;
|
||||
ctx.fillText('G' + entries[i].generation, x, h - 8);
|
||||
}
|
||||
|
||||
// Draw lines for each model
|
||||
modelNames.forEach(name => {
|
||||
const points = [];
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
if (entries[i].ratings[name] !== undefined) {
|
||||
const x = entries.length === 1
|
||||
? pad.left + cw / 2
|
||||
: pad.left + (cw / (entries.length - 1)) * i;
|
||||
const y = pad.top + ch - ((entries[i].ratings[name] - minElo) / eloRange) * ch;
|
||||
points.push({ x, y });
|
||||
}
|
||||
}
|
||||
|
||||
if (points.length === 0) return;
|
||||
|
||||
ctx.strokeStyle = colors[name] || '#5b9ef0';
|
||||
ctx.lineWidth = 2;
|
||||
ctx.lineJoin = 'round';
|
||||
ctx.lineCap = 'round';
|
||||
|
||||
if (points.length === 1) {
|
||||
ctx.fillStyle = colors[name] || '#5b9ef0';
|
||||
ctx.beginPath();
|
||||
ctx.arc(points[0].x, points[0].y, 4, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
} else {
|
||||
ctx.beginPath();
|
||||
points.forEach((p, i) => i === 0 ? ctx.moveTo(p.x, p.y) : ctx.lineTo(p.x, p.y));
|
||||
ctx.stroke();
|
||||
|
||||
// Points
|
||||
points.forEach(p => {
|
||||
ctx.fillStyle = colors[name] || '#5b9ef0';
|
||||
ctx.beginPath();
|
||||
ctx.arc(p.x, p.y, 3, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Legend
|
||||
ctx.font = '11px Inter, sans-serif';
|
||||
let legendX = pad.left + 10;
|
||||
const legendY = pad.top + 12;
|
||||
modelNames.forEach(name => {
|
||||
ctx.fillStyle = colors[name] || '#5b9ef0';
|
||||
ctx.fillRect(legendX, legendY - 5, 10, 10);
|
||||
ctx.fillStyle = '#8b90a0';
|
||||
ctx.textAlign = 'left';
|
||||
const label = name.length > 10 ? name.substring(0, 10) : name;
|
||||
ctx.fillText(label, legendX + 14, legendY + 4);
|
||||
legendX += ctx.measureText(label).width + 28;
|
||||
});
|
||||
}
|
||||
|
||||
function renderEloLeaderboard(ratings) {
|
||||
const container = document.getElementById('elo-leaderboard');
|
||||
if (!ratings || Object.keys(ratings).length === 0) {
|
||||
container.innerHTML = '';
|
||||
return;
|
||||
}
|
||||
|
||||
const sorted = Object.entries(ratings).sort((a, b) => b[1] - a[1]);
|
||||
|
||||
let html = '<table class="data-table elo-table">';
|
||||
html += '<thead><tr><th>Rank</th><th>Policy</th><th>ELO Rating</th><th></th></tr></thead><tbody>';
|
||||
|
||||
sorted.forEach(([name, elo], i) => {
|
||||
const medal = i === 0 ? '🥇' : i === 1 ? '🥈' : i === 2 ? '🥉' : (i + 1);
|
||||
const barPct = Math.max(0, Math.min(100, ((elo - 1000) / 400) * 100));
|
||||
const barColor = elo >= 1250 ? 'var(--success)' : elo >= 1150 ? 'var(--accent)' : elo >= 1050 ? 'var(--warning)' : 'var(--error)';
|
||||
|
||||
html += `<tr>
|
||||
<td style="text-align:center;">${medal}</td>
|
||||
<td style="color:var(--accent);font-weight:600;">${name}</td>
|
||||
<td style="font-weight:700;font-variant-numeric:tabular-nums;">${elo.toFixed(1)}</td>
|
||||
<td style="width:30%;">
|
||||
<div class="acc-bar-bg" style="height:6px;background:var(--surface-3);border-radius:3px;overflow:hidden;">
|
||||
<div style="width:${barPct}%;height:100%;background:${barColor};border-radius:3px;"></div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>`;
|
||||
});
|
||||
|
||||
html += '</tbody></table>';
|
||||
container.innerHTML = html;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════
|
||||
// Window Resize Handler (redraw charts)
|
||||
// ═══════════════════════════════════════════
|
||||
@@ -639,5 +920,8 @@ window.addEventListener('resize', () => {
|
||||
renderWinRateChart(generationsData);
|
||||
renderCompositionChart(generationsData);
|
||||
}
|
||||
if (eloHistoryData && eloHistoryData.length > 0) {
|
||||
renderEloChart(eloHistoryData);
|
||||
}
|
||||
}, 200);
|
||||
});
|
||||
Reference in new issue
Block a user