feat(eval): add run_benchmark.ps1 — full benchmark pipeline for issue #153
This commit is contained in:
KoalaTrapLord committed
1 parent
6e71ba88fd
commit
00374c1f03
1 file changed
+310
@@ -0,0 +1,310 @@
|
||||
#!/usr/bin/env pwsh
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Commander AI Lab — full benchmark pipeline (Issue #153)
|
||||
|
||||
.DESCRIPTION
|
||||
Chains all four steps of the Forge-vs-Synthetic eval benchmark:
|
||||
1. Train baseline model (synthetic self-play)
|
||||
2. Train Forge model (--data-source forge)
|
||||
3. Eval both checkpoints vs Forge built-in (200 games each)
|
||||
4. Compare results with Fisher's exact test
|
||||
|
||||
Writes all artefacts to:
|
||||
ml/models/checkpoints/baseline-synthetic/
|
||||
ml/models/checkpoints/forge-trained/
|
||||
results/eval-baseline.json
|
||||
results/eval-forge.json
|
||||
results/compare-result.json ← pass/fail verdict
|
||||
|
||||
.PARAMETER Iterations
|
||||
PPO training iterations for each model. Default: 100
|
||||
|
||||
.PARAMETER EpisodesPerIter
|
||||
Self-play / Forge episodes per iteration. Default: 64
|
||||
|
||||
.PARAMETER EvalGames
|
||||
Evaluation games per model. Default: 200
|
||||
|
||||
.PARAMETER Threads
|
||||
Parallel Forge workers for eval. Default: 4
|
||||
|
||||
.PARAMETER Seed
|
||||
Optional RNG seed (passed to both eval runs for reproducibility).
|
||||
|
||||
.PARAMETER ForgeResultsDir
|
||||
Directory where Forge writes ml-decisions-forge-*.jsonl files.
|
||||
Default: results
|
||||
|
||||
.PARAMETER ForgeBatchId
|
||||
Batch ID prefix used when the Forge producer writes JSONL files.
|
||||
Default: forge-batch
|
||||
|
||||
.PARAMETER ForgeNumGames
|
||||
Total Forge games the producer will consume during Forge training.
|
||||
Default: 256
|
||||
|
||||
.PARAMETER MinDelta
|
||||
Minimum absolute win-rate improvement to pass the benchmark. Default: 0.05
|
||||
|
||||
.PARAMETER Alpha
|
||||
Significance level for Fisher's exact test. Default: 0.05
|
||||
|
||||
.PARAMETER SkipTraining
|
||||
Skip both training steps (use existing checkpoints). Useful for re-running eval only.
|
||||
|
||||
.PARAMETER SkipEval
|
||||
Skip both eval steps (use existing result JSONs). Useful for re-running compare only.
|
||||
|
||||
.EXAMPLE
|
||||
# Full overnight run
|
||||
.\run_benchmark.ps1
|
||||
|
||||
.EXAMPLE
|
||||
# Quick smoke-test with fewer iterations
|
||||
.\run_benchmark.ps1 -Iterations 10 -EpisodePerIter 16 -EvalGames 50
|
||||
|
||||
.EXAMPLE
|
||||
# Re-run only the comparison from existing JSONs
|
||||
.\run_benchmark.ps1 -SkipTraining -SkipEval
|
||||
#>
|
||||
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[int] $Iterations = 100,
|
||||
[int] $EpisodesPerIter = 64,
|
||||
[int] $EvalGames = 200,
|
||||
[int] $Threads = 4,
|
||||
[int] $Seed = -1,
|
||||
[string] $ForgeResultsDir = "results",
|
||||
[string] $ForgeBatchId = "forge-batch",
|
||||
[int] $ForgeNumGames = 256,
|
||||
[double] $MinDelta = 0.05,
|
||||
[double] $Alpha = 0.05,
|
||||
[switch] $SkipTraining,
|
||||
[switch] $SkipEval
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
# ── Paths ──────────────────────────────────────────────────────────────
|
||||
$BaselineDir = "ml/models/checkpoints/baseline-synthetic"
|
||||
$ForgeDir = "ml/models/checkpoints/forge-trained"
|
||||
$BaselineJson = "results/eval-baseline.json"
|
||||
$ForgeJson = "results/eval-forge.json"
|
||||
$CompareJson = "results/compare-result.json"
|
||||
$LogDir = "logs/benchmark"
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────
|
||||
function Banner([string]$msg) {
|
||||
$line = "=" * 62
|
||||
Write-Host ""
|
||||
Write-Host $line -ForegroundColor Cyan
|
||||
Write-Host " $msg" -ForegroundColor Cyan
|
||||
Write-Host $line -ForegroundColor Cyan
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
function Step([string]$msg) {
|
||||
Write-Host "[$(Get-Date -Format 'HH:mm:ss')] $msg" -ForegroundColor Yellow
|
||||
}
|
||||
|
||||
function Ok([string]$msg) {
|
||||
Write-Host " ✓ $msg" -ForegroundColor Green
|
||||
}
|
||||
|
||||
function Fail([string]$msg) {
|
||||
Write-Host " ✗ $msg" -ForegroundColor Red
|
||||
}
|
||||
|
||||
function Run([string[]]$cmd) {
|
||||
$display = $cmd -join " "
|
||||
Step "Running: $display"
|
||||
& $cmd[0] $cmd[1..($cmd.Length-1)]
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Fail "Command failed (exit $LASTEXITCODE): $display"
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
}
|
||||
|
||||
# ── Setup ──────────────────────────────────────────────────────────────
|
||||
$null = New-Item -ItemType Directory -Force -Path $LogDir, "results" | Out-Null
|
||||
|
||||
$SeedArgs = @()
|
||||
if ($Seed -ge 0) { $SeedArgs = @("--seed", "$Seed") }
|
||||
|
||||
$StartTime = Get-Date
|
||||
Banner "Commander AI Lab — Benchmark Pipeline (Issue #153)"
|
||||
|
||||
Write-Host " Iterations : $Iterations"
|
||||
Write-Host " Episodes/iter : $EpisodesPerIter"
|
||||
Write-Host " Eval games : $EvalGames"
|
||||
Write-Host " Eval threads : $Threads"
|
||||
Write-Host " Seed : $(if ($Seed -ge 0) { $Seed } else { '(random)' })"
|
||||
Write-Host " Min delta : $($MinDelta * 100)%"
|
||||
Write-Host " Alpha : $Alpha"
|
||||
Write-Host " Skip training : $SkipTraining"
|
||||
Write-Host " Skip eval : $SkipEval"
|
||||
Write-Host ""
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# STEP 1 — Train baseline (synthetic self-play)
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
if (-not $SkipTraining) {
|
||||
Banner "Step 1/4 — Training baseline (synthetic)"
|
||||
$t0 = Get-Date
|
||||
|
||||
Run @(
|
||||
"python", "-m", "ml.training.ppo_trainer",
|
||||
"--data-source", "synthetic",
|
||||
"--iterations", "$Iterations",
|
||||
"--episodes-per-iter","$EpisodesPerIter",
|
||||
"--checkpoint-dir", $BaselineDir,
|
||||
"--save-every", "10",
|
||||
"--eval-every", "5"
|
||||
)
|
||||
|
||||
$elapsed = ((Get-Date) - $t0).TotalMinutes
|
||||
Ok "Baseline training complete in $([math]::Round($elapsed,1)) min"
|
||||
Ok "Checkpoints in: $BaselineDir"
|
||||
} else {
|
||||
Step "Skipping training (--SkipTraining)"
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# STEP 2 — Train Forge model
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
if (-not $SkipTraining) {
|
||||
Banner "Step 2/4 — Training Forge model"
|
||||
$t0 = Get-Date
|
||||
|
||||
Run @(
|
||||
"python", "-m", "ml.training.ppo_trainer",
|
||||
"--data-source", "forge",
|
||||
"--iterations", "$Iterations",
|
||||
"--episodes-per-iter", "$EpisodesPerIter",
|
||||
"--checkpoint-dir", $ForgeDir,
|
||||
"--forge-results-dir", $ForgeResultsDir,
|
||||
"--forge-batch-id", $ForgeBatchId,
|
||||
"--forge-num-games", "$ForgeNumGames",
|
||||
"--save-every", "10",
|
||||
"--eval-every", "5"
|
||||
)
|
||||
|
||||
$elapsed = ((Get-Date) - $t0).TotalMinutes
|
||||
Ok "Forge training complete in $([math]::Round($elapsed,1)) min"
|
||||
Ok "Checkpoints in: $ForgeDir"
|
||||
} else {
|
||||
Step "Skipping training (--SkipTraining)"
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# STEP 3 — Eval baseline checkpoint
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
if (-not $SkipEval) {
|
||||
Banner "Step 3a/4 — Evaluating baseline checkpoint"
|
||||
|
||||
# Prefer best_ppo.pt, fall back to latest numbered checkpoint
|
||||
$BaselinePt = "$BaselineDir/best_ppo.pt"
|
||||
if (-not (Test-Path $BaselinePt)) {
|
||||
$BaselinePt = Get-ChildItem "$BaselineDir/ppo_iter_*.pt" |
|
||||
Sort-Object Name -Descending |
|
||||
Select-Object -First 1 -ExpandProperty FullName
|
||||
}
|
||||
if (-not $BaselinePt -or -not (Test-Path $BaselinePt)) {
|
||||
Fail "No checkpoint found in $BaselineDir — run training first or remove --SkipTraining"
|
||||
exit 1
|
||||
}
|
||||
Step "Using checkpoint: $BaselinePt"
|
||||
$t0 = Get-Date
|
||||
|
||||
Run (@(
|
||||
"python", "scripts/eval_policy.py",
|
||||
"--model", $BaselinePt,
|
||||
"--games", "$EvalGames",
|
||||
"--run-id", "baseline-synthetic",
|
||||
"--out", $BaselineJson,
|
||||
"--threads", "$Threads"
|
||||
) + $SeedArgs)
|
||||
|
||||
$elapsed = ((Get-Date) - $t0).TotalMinutes
|
||||
Ok "Baseline eval complete in $([math]::Round($elapsed,1)) min -> $BaselineJson"
|
||||
|
||||
# ── Eval Forge checkpoint ──────────────────────────────────────────
|
||||
Banner "Step 3b/4 — Evaluating Forge checkpoint"
|
||||
|
||||
$ForgePt = "$ForgeDir/best_ppo.pt"
|
||||
if (-not (Test-Path $ForgePt)) {
|
||||
$ForgePt = Get-ChildItem "$ForgeDir/ppo_iter_*.pt" |
|
||||
Sort-Object Name -Descending |
|
||||
Select-Object -First 1 -ExpandProperty FullName
|
||||
}
|
||||
if (-not $ForgePt -or -not (Test-Path $ForgePt)) {
|
||||
Fail "No checkpoint found in $ForgeDir — run training first or remove --SkipTraining"
|
||||
exit 1
|
||||
}
|
||||
Step "Using checkpoint: $ForgePt"
|
||||
$t0 = Get-Date
|
||||
|
||||
Run (@(
|
||||
"python", "scripts/eval_policy.py",
|
||||
"--model", $ForgePt,
|
||||
"--games", "$EvalGames",
|
||||
"--run-id", "forge-trained",
|
||||
"--out", $ForgeJson,
|
||||
"--threads", "$Threads"
|
||||
) + $SeedArgs)
|
||||
|
||||
$elapsed = ((Get-Date) - $t0).TotalMinutes
|
||||
Ok "Forge eval complete in $([math]::Round($elapsed,1)) min -> $ForgeJson"
|
||||
} else {
|
||||
Step "Skipping eval (--SkipEval)"
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# STEP 4 — Compare results
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
Banner "Step 4/4 — Comparing results"
|
||||
|
||||
if (-not (Test-Path $BaselineJson)) {
|
||||
Fail "Baseline result not found: $BaselineJson"
|
||||
exit 1
|
||||
}
|
||||
if (-not (Test-Path $ForgeJson)) {
|
||||
Fail "Forge result not found: $ForgeJson"
|
||||
exit 1
|
||||
}
|
||||
|
||||
python scripts/compare_eval.py `
|
||||
--baseline $BaselineJson `
|
||||
--forge $ForgeJson `
|
||||
--min-delta $MinDelta `
|
||||
--alpha $Alpha `
|
||||
--json-out $CompareJson
|
||||
|
||||
$exitCode = $LASTEXITCODE
|
||||
|
||||
# ── Final summary ──────────────────────────────────────────────────────
|
||||
$totalMin = [math]::Round(((Get-Date) - $StartTime).TotalMinutes, 1)
|
||||
Banner "Benchmark Complete"
|
||||
Write-Host " Total wall time : $totalMin min"
|
||||
Write-Host " Baseline JSON : $BaselineJson"
|
||||
Write-Host " Forge JSON : $ForgeJson"
|
||||
Write-Host " Compare JSON : $CompareJson"
|
||||
Write-Host ""
|
||||
|
||||
if ($exitCode -eq 0) {
|
||||
Ok "PASSED — Forge model meets the benchmark threshold."
|
||||
Write-Host ""
|
||||
Write-Host " Next steps:" -ForegroundColor Cyan
|
||||
Write-Host " git add results/eval-baseline.json results/eval-forge.json results/compare-result.json"
|
||||
Write-Host " git commit -m 'results: forge-vs-synthetic benchmark run'"
|
||||
Write-Host " git push"
|
||||
} else {
|
||||
Fail "FAILED — Forge model did not meet the benchmark threshold."
|
||||
Write-Host " Check $CompareJson for details." -ForegroundColor Yellow
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
exit $exitCode
|
||||
Reference in new issue
Block a user