feat(eval): add run_benchmark.ps1 — full benchmark pipeline for issue #153
This commit is contained in:
KoalaTrapLord committed
1 parent
6e71ba88fd
commit
00374c1f03
1 file changed
+310
@@ -0,0 +1,310 @@
|
|||||||
|
#!/usr/bin/env pwsh
|
||||||
|
<#
|
||||||
|
.SYNOPSIS
|
||||||
|
Commander AI Lab — full benchmark pipeline (Issue #153)
|
||||||
|
|
||||||
|
.DESCRIPTION
|
||||||
|
Chains all four steps of the Forge-vs-Synthetic eval benchmark:
|
||||||
|
1. Train baseline model (synthetic self-play)
|
||||||
|
2. Train Forge model (--data-source forge)
|
||||||
|
3. Eval both checkpoints vs Forge built-in (200 games each)
|
||||||
|
4. Compare results with Fisher's exact test
|
||||||
|
|
||||||
|
Writes all artefacts to:
|
||||||
|
ml/models/checkpoints/baseline-synthetic/
|
||||||
|
ml/models/checkpoints/forge-trained/
|
||||||
|
results/eval-baseline.json
|
||||||
|
results/eval-forge.json
|
||||||
|
results/compare-result.json ← pass/fail verdict
|
||||||
|
|
||||||
|
.PARAMETER Iterations
|
||||||
|
PPO training iterations for each model. Default: 100
|
||||||
|
|
||||||
|
.PARAMETER EpisodesPerIter
|
||||||
|
Self-play / Forge episodes per iteration. Default: 64
|
||||||
|
|
||||||
|
.PARAMETER EvalGames
|
||||||
|
Evaluation games per model. Default: 200
|
||||||
|
|
||||||
|
.PARAMETER Threads
|
||||||
|
Parallel Forge workers for eval. Default: 4
|
||||||
|
|
||||||
|
.PARAMETER Seed
|
||||||
|
Optional RNG seed (passed to both eval runs for reproducibility).
|
||||||
|
|
||||||
|
.PARAMETER ForgeResultsDir
|
||||||
|
Directory where Forge writes ml-decisions-forge-*.jsonl files.
|
||||||
|
Default: results
|
||||||
|
|
||||||
|
.PARAMETER ForgeBatchId
|
||||||
|
Batch ID prefix used when the Forge producer writes JSONL files.
|
||||||
|
Default: forge-batch
|
||||||
|
|
||||||
|
.PARAMETER ForgeNumGames
|
||||||
|
Total Forge games the producer will consume during Forge training.
|
||||||
|
Default: 256
|
||||||
|
|
||||||
|
.PARAMETER MinDelta
|
||||||
|
Minimum absolute win-rate improvement to pass the benchmark. Default: 0.05
|
||||||
|
|
||||||
|
.PARAMETER Alpha
|
||||||
|
Significance level for Fisher's exact test. Default: 0.05
|
||||||
|
|
||||||
|
.PARAMETER SkipTraining
|
||||||
|
Skip both training steps (use existing checkpoints). Useful for re-running eval only.
|
||||||
|
|
||||||
|
.PARAMETER SkipEval
|
||||||
|
Skip both eval steps (use existing result JSONs). Useful for re-running compare only.
|
||||||
|
|
||||||
|
.EXAMPLE
|
||||||
|
# Full overnight run
|
||||||
|
.\run_benchmark.ps1
|
||||||
|
|
||||||
|
.EXAMPLE
|
||||||
|
# Quick smoke-test with fewer iterations
|
||||||
|
.\run_benchmark.ps1 -Iterations 10 -EpisodePerIter 16 -EvalGames 50
|
||||||
|
|
||||||
|
.EXAMPLE
|
||||||
|
# Re-run only the comparison from existing JSONs
|
||||||
|
.\run_benchmark.ps1 -SkipTraining -SkipEval
|
||||||
|
#>
|
||||||
|
|
||||||
|
[CmdletBinding()]
|
||||||
|
param(
|
||||||
|
[int] $Iterations = 100,
|
||||||
|
[int] $EpisodesPerIter = 64,
|
||||||
|
[int] $EvalGames = 200,
|
||||||
|
[int] $Threads = 4,
|
||||||
|
[int] $Seed = -1,
|
||||||
|
[string] $ForgeResultsDir = "results",
|
||||||
|
[string] $ForgeBatchId = "forge-batch",
|
||||||
|
[int] $ForgeNumGames = 256,
|
||||||
|
[double] $MinDelta = 0.05,
|
||||||
|
[double] $Alpha = 0.05,
|
||||||
|
[switch] $SkipTraining,
|
||||||
|
[switch] $SkipEval
|
||||||
|
)
|
||||||
|
|
||||||
|
Set-StrictMode -Version Latest
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
|
||||||
|
# ── Paths ──────────────────────────────────────────────────────────────
|
||||||
|
$BaselineDir = "ml/models/checkpoints/baseline-synthetic"
|
||||||
|
$ForgeDir = "ml/models/checkpoints/forge-trained"
|
||||||
|
$BaselineJson = "results/eval-baseline.json"
|
||||||
|
$ForgeJson = "results/eval-forge.json"
|
||||||
|
$CompareJson = "results/compare-result.json"
|
||||||
|
$LogDir = "logs/benchmark"
|
||||||
|
|
||||||
|
# ── Helpers ────────────────────────────────────────────────────────────
|
||||||
|
function Banner([string]$msg) {
|
||||||
|
$line = "=" * 62
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host $line -ForegroundColor Cyan
|
||||||
|
Write-Host " $msg" -ForegroundColor Cyan
|
||||||
|
Write-Host $line -ForegroundColor Cyan
|
||||||
|
Write-Host ""
|
||||||
|
}
|
||||||
|
|
||||||
|
function Step([string]$msg) {
|
||||||
|
Write-Host "[$(Get-Date -Format 'HH:mm:ss')] $msg" -ForegroundColor Yellow
|
||||||
|
}
|
||||||
|
|
||||||
|
function Ok([string]$msg) {
|
||||||
|
Write-Host " ✓ $msg" -ForegroundColor Green
|
||||||
|
}
|
||||||
|
|
||||||
|
function Fail([string]$msg) {
|
||||||
|
Write-Host " ✗ $msg" -ForegroundColor Red
|
||||||
|
}
|
||||||
|
|
||||||
|
function Run([string[]]$cmd) {
|
||||||
|
$display = $cmd -join " "
|
||||||
|
Step "Running: $display"
|
||||||
|
& $cmd[0] $cmd[1..($cmd.Length-1)]
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
Fail "Command failed (exit $LASTEXITCODE): $display"
|
||||||
|
exit $LASTEXITCODE
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Setup ──────────────────────────────────────────────────────────────
|
||||||
|
$null = New-Item -ItemType Directory -Force -Path $LogDir, "results" | Out-Null
|
||||||
|
|
||||||
|
$SeedArgs = @()
|
||||||
|
if ($Seed -ge 0) { $SeedArgs = @("--seed", "$Seed") }
|
||||||
|
|
||||||
|
$StartTime = Get-Date
|
||||||
|
Banner "Commander AI Lab — Benchmark Pipeline (Issue #153)"
|
||||||
|
|
||||||
|
Write-Host " Iterations : $Iterations"
|
||||||
|
Write-Host " Episodes/iter : $EpisodesPerIter"
|
||||||
|
Write-Host " Eval games : $EvalGames"
|
||||||
|
Write-Host " Eval threads : $Threads"
|
||||||
|
Write-Host " Seed : $(if ($Seed -ge 0) { $Seed } else { '(random)' })"
|
||||||
|
Write-Host " Min delta : $($MinDelta * 100)%"
|
||||||
|
Write-Host " Alpha : $Alpha"
|
||||||
|
Write-Host " Skip training : $SkipTraining"
|
||||||
|
Write-Host " Skip eval : $SkipEval"
|
||||||
|
Write-Host ""
|
||||||
|
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
# STEP 1 — Train baseline (synthetic self-play)
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
if (-not $SkipTraining) {
|
||||||
|
Banner "Step 1/4 — Training baseline (synthetic)"
|
||||||
|
$t0 = Get-Date
|
||||||
|
|
||||||
|
Run @(
|
||||||
|
"python", "-m", "ml.training.ppo_trainer",
|
||||||
|
"--data-source", "synthetic",
|
||||||
|
"--iterations", "$Iterations",
|
||||||
|
"--episodes-per-iter","$EpisodesPerIter",
|
||||||
|
"--checkpoint-dir", $BaselineDir,
|
||||||
|
"--save-every", "10",
|
||||||
|
"--eval-every", "5"
|
||||||
|
)
|
||||||
|
|
||||||
|
$elapsed = ((Get-Date) - $t0).TotalMinutes
|
||||||
|
Ok "Baseline training complete in $([math]::Round($elapsed,1)) min"
|
||||||
|
Ok "Checkpoints in: $BaselineDir"
|
||||||
|
} else {
|
||||||
|
Step "Skipping training (--SkipTraining)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
# STEP 2 — Train Forge model
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
if (-not $SkipTraining) {
|
||||||
|
Banner "Step 2/4 — Training Forge model"
|
||||||
|
$t0 = Get-Date
|
||||||
|
|
||||||
|
Run @(
|
||||||
|
"python", "-m", "ml.training.ppo_trainer",
|
||||||
|
"--data-source", "forge",
|
||||||
|
"--iterations", "$Iterations",
|
||||||
|
"--episodes-per-iter", "$EpisodesPerIter",
|
||||||
|
"--checkpoint-dir", $ForgeDir,
|
||||||
|
"--forge-results-dir", $ForgeResultsDir,
|
||||||
|
"--forge-batch-id", $ForgeBatchId,
|
||||||
|
"--forge-num-games", "$ForgeNumGames",
|
||||||
|
"--save-every", "10",
|
||||||
|
"--eval-every", "5"
|
||||||
|
)
|
||||||
|
|
||||||
|
$elapsed = ((Get-Date) - $t0).TotalMinutes
|
||||||
|
Ok "Forge training complete in $([math]::Round($elapsed,1)) min"
|
||||||
|
Ok "Checkpoints in: $ForgeDir"
|
||||||
|
} else {
|
||||||
|
Step "Skipping training (--SkipTraining)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
# STEP 3 — Eval baseline checkpoint
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
if (-not $SkipEval) {
|
||||||
|
Banner "Step 3a/4 — Evaluating baseline checkpoint"
|
||||||
|
|
||||||
|
# Prefer best_ppo.pt, fall back to latest numbered checkpoint
|
||||||
|
$BaselinePt = "$BaselineDir/best_ppo.pt"
|
||||||
|
if (-not (Test-Path $BaselinePt)) {
|
||||||
|
$BaselinePt = Get-ChildItem "$BaselineDir/ppo_iter_*.pt" |
|
||||||
|
Sort-Object Name -Descending |
|
||||||
|
Select-Object -First 1 -ExpandProperty FullName
|
||||||
|
}
|
||||||
|
if (-not $BaselinePt -or -not (Test-Path $BaselinePt)) {
|
||||||
|
Fail "No checkpoint found in $BaselineDir — run training first or remove --SkipTraining"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
Step "Using checkpoint: $BaselinePt"
|
||||||
|
$t0 = Get-Date
|
||||||
|
|
||||||
|
Run (@(
|
||||||
|
"python", "scripts/eval_policy.py",
|
||||||
|
"--model", $BaselinePt,
|
||||||
|
"--games", "$EvalGames",
|
||||||
|
"--run-id", "baseline-synthetic",
|
||||||
|
"--out", $BaselineJson,
|
||||||
|
"--threads", "$Threads"
|
||||||
|
) + $SeedArgs)
|
||||||
|
|
||||||
|
$elapsed = ((Get-Date) - $t0).TotalMinutes
|
||||||
|
Ok "Baseline eval complete in $([math]::Round($elapsed,1)) min -> $BaselineJson"
|
||||||
|
|
||||||
|
# ── Eval Forge checkpoint ──────────────────────────────────────────
|
||||||
|
Banner "Step 3b/4 — Evaluating Forge checkpoint"
|
||||||
|
|
||||||
|
$ForgePt = "$ForgeDir/best_ppo.pt"
|
||||||
|
if (-not (Test-Path $ForgePt)) {
|
||||||
|
$ForgePt = Get-ChildItem "$ForgeDir/ppo_iter_*.pt" |
|
||||||
|
Sort-Object Name -Descending |
|
||||||
|
Select-Object -First 1 -ExpandProperty FullName
|
||||||
|
}
|
||||||
|
if (-not $ForgePt -or -not (Test-Path $ForgePt)) {
|
||||||
|
Fail "No checkpoint found in $ForgeDir — run training first or remove --SkipTraining"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
Step "Using checkpoint: $ForgePt"
|
||||||
|
$t0 = Get-Date
|
||||||
|
|
||||||
|
Run (@(
|
||||||
|
"python", "scripts/eval_policy.py",
|
||||||
|
"--model", $ForgePt,
|
||||||
|
"--games", "$EvalGames",
|
||||||
|
"--run-id", "forge-trained",
|
||||||
|
"--out", $ForgeJson,
|
||||||
|
"--threads", "$Threads"
|
||||||
|
) + $SeedArgs)
|
||||||
|
|
||||||
|
$elapsed = ((Get-Date) - $t0).TotalMinutes
|
||||||
|
Ok "Forge eval complete in $([math]::Round($elapsed,1)) min -> $ForgeJson"
|
||||||
|
} else {
|
||||||
|
Step "Skipping eval (--SkipEval)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
# STEP 4 — Compare results
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
Banner "Step 4/4 — Comparing results"
|
||||||
|
|
||||||
|
if (-not (Test-Path $BaselineJson)) {
|
||||||
|
Fail "Baseline result not found: $BaselineJson"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
if (-not (Test-Path $ForgeJson)) {
|
||||||
|
Fail "Forge result not found: $ForgeJson"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
python scripts/compare_eval.py `
|
||||||
|
--baseline $BaselineJson `
|
||||||
|
--forge $ForgeJson `
|
||||||
|
--min-delta $MinDelta `
|
||||||
|
--alpha $Alpha `
|
||||||
|
--json-out $CompareJson
|
||||||
|
|
||||||
|
$exitCode = $LASTEXITCODE
|
||||||
|
|
||||||
|
# ── Final summary ──────────────────────────────────────────────────────
|
||||||
|
$totalMin = [math]::Round(((Get-Date) - $StartTime).TotalMinutes, 1)
|
||||||
|
Banner "Benchmark Complete"
|
||||||
|
Write-Host " Total wall time : $totalMin min"
|
||||||
|
Write-Host " Baseline JSON : $BaselineJson"
|
||||||
|
Write-Host " Forge JSON : $ForgeJson"
|
||||||
|
Write-Host " Compare JSON : $CompareJson"
|
||||||
|
Write-Host ""
|
||||||
|
|
||||||
|
if ($exitCode -eq 0) {
|
||||||
|
Ok "PASSED — Forge model meets the benchmark threshold."
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host " Next steps:" -ForegroundColor Cyan
|
||||||
|
Write-Host " git add results/eval-baseline.json results/eval-forge.json results/compare-result.json"
|
||||||
|
Write-Host " git commit -m 'results: forge-vs-synthetic benchmark run'"
|
||||||
|
Write-Host " git push"
|
||||||
|
} else {
|
||||||
|
Fail "FAILED — Forge model did not meet the benchmark threshold."
|
||||||
|
Write-Host " Check $CompareJson for details." -ForegroundColor Yellow
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Host ""
|
||||||
|
exit $exitCode
|
||||||
Reference in new issue
Block a user