fez5587/commander-ai-lab
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(eval): remove Set-StrictMode, fix curly-quotes and -- in strings (issue #153)

Browse Source
This commit is contained in:
KoalaTrapLord committed 2026-03-29 05:31:58 -04:00
1 parent 00374c1f03
commit 915c745911
1 file changed
+56 -72
Download Patch File Download Diff File Expand all files Collapse all files
run_benchmark.ps1
+56 -72
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env pwsh #!/usr/bin/env pwsh
<# <#
.SYNOPSIS .SYNOPSIS
Commander AI Lab — full benchmark pipeline (Issue #153) Commander AI Lab - full benchmark pipeline (Issue #153)
.DESCRIPTION .DESCRIPTION
Chains all four steps of the Forge-vs-Synthetic eval benchmark: Chains all four steps of the Forge-vs-Synthetic eval benchmark:
@@ -15,7 +15,7 @@
ml/models/checkpoints/forge-trained/ ml/models/checkpoints/forge-trained/
results/eval-baseline.json results/eval-baseline.json
results/eval-forge.json results/eval-forge.json
results/compare-result.json ← pass/fail verdict results/compare-result.json <- pass/fail verdict
.PARAMETER Iterations .PARAMETER Iterations
PPO training iterations for each model. Default: 100 PPO training iterations for each model. Default: 100
@@ -51,21 +51,18 @@
Significance level for Fisher's exact test. Default: 0.05 Significance level for Fisher's exact test. Default: 0.05
.PARAMETER SkipTraining .PARAMETER SkipTraining
Skip both training steps (use existing checkpoints). Useful for re-running eval only. Skip both training steps (use existing checkpoints).
.PARAMETER SkipEval .PARAMETER SkipEval
Skip both eval steps (use existing result JSONs). Useful for re-running compare only. Skip both eval steps (use existing result JSONs).
.EXAMPLE .EXAMPLE
# Full overnight run
.\run_benchmark.ps1 .\run_benchmark.ps1
.EXAMPLE .EXAMPLE
# Quick smoke-test with fewer iterations .\run_benchmark.ps1 -Iterations 10 -EpisodesPerIter 16 -EvalGames 50
.\run_benchmark.ps1 -Iterations 10 -EpisodePerIter 16 -EvalGames 50
.EXAMPLE .EXAMPLE
# Re-run only the comparison from existing JSONs
.\run_benchmark.ps1 -SkipTraining -SkipEval .\run_benchmark.ps1 -SkipTraining -SkipEval
#> #>
@@ -85,18 +82,16 @@ param(
[switch] $SkipEval [switch] $SkipEval
) )
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop" $ErrorActionPreference = "Stop"
# ── Paths ────────────────────────────────────────────────────────────── # Paths
$BaselineDir = "ml/models/checkpoints/baseline-synthetic" $BaselineDir = "ml/models/checkpoints/baseline-synthetic"
$ForgeDir = "ml/models/checkpoints/forge-trained" $ForgeDir = "ml/models/checkpoints/forge-trained"
$BaselineJson = "results/eval-baseline.json" $BaselineJson = "results/eval-baseline.json"
$ForgeJson = "results/eval-forge.json" $ForgeJson = "results/eval-forge.json"
$CompareJson = "results/compare-result.json" $CompareJson = "results/compare-result.json"
$LogDir = "logs/benchmark"
# ── Helpers ──────────────────────────────────────────────────────────── # Helpers
function Banner([string]$msg) { function Banner([string]$msg) {
$line = "=" * 62 $line = "=" * 62
Write-Host "" Write-Host ""
@@ -111,11 +106,11 @@ function Step([string]$msg) {
} }
function Ok([string]$msg) { function Ok([string]$msg) {
Write-Host " ✓ $msg" -ForegroundColor Green Write-Host " OK $msg" -ForegroundColor Green
} }
function Fail([string]$msg) { function Err([string]$msg) {
Write-Host " ✗ $msg" -ForegroundColor Red Write-Host " FAIL $msg" -ForegroundColor Red
} }
function Run([string[]]$cmd) { function Run([string[]]$cmd) {
@@ -123,19 +118,19 @@ function Run([string[]]$cmd) {
Step "Running: $display" Step "Running: $display"
& $cmd[0] $cmd[1..($cmd.Length-1)] & $cmd[0] $cmd[1..($cmd.Length-1)]
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Fail "Command failed (exit $LASTEXITCODE): $display" Err "Command failed (exit $LASTEXITCODE): $display"
exit $LASTEXITCODE exit $LASTEXITCODE
} }
} }
# ── Setup ────────────────────────────────────────────────────────────── # Setup
$null = New-Item -ItemType Directory -Force -Path $LogDir, "results" | Out-Null $null = New-Item -ItemType Directory -Force -Path "logs/benchmark", "results" | Out-Null
$SeedArgs = @() $SeedArgs = @()
if ($Seed -ge 0) { $SeedArgs = @("--seed", "$Seed") } if ($Seed -ge 0) { $SeedArgs = @("--seed", "$Seed") }
$StartTime = Get-Date $StartTime = Get-Date
Banner "Commander AI Lab — Benchmark Pipeline (Issue #153)" Banner "Commander AI Lab - Benchmark Pipeline (Issue #153)"
Write-Host " Iterations : $Iterations" Write-Host " Iterations : $Iterations"
Write-Host " Episodes/iter : $EpisodesPerIter" Write-Host " Episodes/iter : $EpisodesPerIter"
@@ -148,35 +143,31 @@ Write-Host " Skip training : $SkipTraining"
Write-Host " Skip eval : $SkipEval" Write-Host " Skip eval : $SkipEval"
Write-Host "" Write-Host ""
# ══════════════════════════════════════════════════════════════════════ # STEP 1 - Train baseline
# STEP 1 — Train baseline (synthetic self-play)
# ══════════════════════════════════════════════════════════════════════
if (-not $SkipTraining) { if (-not $SkipTraining) {
Banner "Step 1/4 — Training baseline (synthetic)" Banner "Step 1/4 - Training baseline (synthetic)"
$t0 = Get-Date $t0 = Get-Date
Run @( Run @(
"python", "-m", "ml.training.ppo_trainer", "python", "-m", "ml.training.ppo_trainer",
"--data-source", "synthetic", "--data-source", "synthetic",
"--iterations", "$Iterations", "--iterations", "$Iterations",
"--episodes-per-iter","$EpisodesPerIter", "--episodes-per-iter", "$EpisodesPerIter",
"--checkpoint-dir", $BaselineDir, "--checkpoint-dir", $BaselineDir,
"--save-every", "10", "--save-every", "10",
"--eval-every", "5" "--eval-every", "5"
) )
$elapsed = ((Get-Date) - $t0).TotalMinutes $elapsed = [math]::Round(((Get-Date) - $t0).TotalMinutes, 1)
Ok "Baseline training complete in $([math]::Round($elapsed,1)) min" Ok "Baseline training complete in $elapsed min"
Ok "Checkpoints in: $BaselineDir" Ok "Checkpoints in: $BaselineDir"
} else { } else {
Step "Skipping training (--SkipTraining)" Step "Skipping training (-SkipTraining flag set)"
} }
# ══════════════════════════════════════════════════════════════════════ # STEP 2 - Train Forge model
# STEP 2 — Train Forge model
# ══════════════════════════════════════════════════════════════════════
if (-not $SkipTraining) { if (-not $SkipTraining) {
Banner "Step 2/4 — Training Forge model" Banner "Step 2/4 - Training Forge model"
$t0 = Get-Date $t0 = Get-Date
Run @( Run @(
@@ -192,20 +183,17 @@ if (-not $SkipTraining) {
"--eval-every", "5" "--eval-every", "5"
) )
$elapsed = ((Get-Date) - $t0).TotalMinutes $elapsed = [math]::Round(((Get-Date) - $t0).TotalMinutes, 1)
Ok "Forge training complete in $([math]::Round($elapsed,1)) min" Ok "Forge training complete in $elapsed min"
Ok "Checkpoints in: $ForgeDir" Ok "Checkpoints in: $ForgeDir"
} else { } else {
Step "Skipping training (--SkipTraining)" Step "Skipping training (-SkipTraining flag set)"
} }
# ══════════════════════════════════════════════════════════════════════ # STEP 3 - Eval
# STEP 3 — Eval baseline checkpoint
# ══════════════════════════════════════════════════════════════════════
if (-not $SkipEval) { if (-not $SkipEval) {
Banner "Step 3a/4 — Evaluating baseline checkpoint" Banner "Step 3a/4 - Evaluating baseline checkpoint"
# Prefer best_ppo.pt, fall back to latest numbered checkpoint
$BaselinePt = "$BaselineDir/best_ppo.pt" $BaselinePt = "$BaselineDir/best_ppo.pt"
if (-not (Test-Path $BaselinePt)) { if (-not (Test-Path $BaselinePt)) {
$BaselinePt = Get-ChildItem "$BaselineDir/ppo_iter_*.pt" | $BaselinePt = Get-ChildItem "$BaselineDir/ppo_iter_*.pt" |
@@ -213,7 +201,7 @@ if (-not $SkipEval) {
Select-Object -First 1 -ExpandProperty FullName Select-Object -First 1 -ExpandProperty FullName
} }
if (-not $BaselinePt -or -not (Test-Path $BaselinePt)) { if (-not $BaselinePt -or -not (Test-Path $BaselinePt)) {
Fail "No checkpoint found in $BaselineDir — run training first or remove --SkipTraining" Err "No checkpoint found in $BaselineDir - run training first"
exit 1 exit 1
} }
Step "Using checkpoint: $BaselinePt" Step "Using checkpoint: $BaselinePt"
@@ -221,18 +209,17 @@ if (-not $SkipEval) {
Run (@( Run (@(
"python", "scripts/eval_policy.py", "python", "scripts/eval_policy.py",
"--model", $BaselinePt, "--model", $BaselinePt,
"--games", "$EvalGames", "--games", "$EvalGames",
"--run-id", "baseline-synthetic", "--run-id", "baseline-synthetic",
"--out", $BaselineJson, "--out", $BaselineJson,
"--threads", "$Threads" "--threads", "$Threads"
) + $SeedArgs) ) + $SeedArgs)
$elapsed = ((Get-Date) - $t0).TotalMinutes $elapsed = [math]::Round(((Get-Date) - $t0).TotalMinutes, 1)
Ok "Baseline eval complete in $([math]::Round($elapsed,1)) min -> $BaselineJson" Ok "Baseline eval complete in $elapsed min -> $BaselineJson"
# ── Eval Forge checkpoint ────────────────────────────────────────── Banner "Step 3b/4 - Evaluating Forge checkpoint"
Banner "Step 3b/4 — Evaluating Forge checkpoint"
$ForgePt = "$ForgeDir/best_ppo.pt" $ForgePt = "$ForgeDir/best_ppo.pt"
if (-not (Test-Path $ForgePt)) { if (-not (Test-Path $ForgePt)) {
@@ -241,7 +228,7 @@ if (-not $SkipEval) {
Select-Object -First 1 -ExpandProperty FullName Select-Object -First 1 -ExpandProperty FullName
} }
if (-not $ForgePt -or -not (Test-Path $ForgePt)) { if (-not $ForgePt -or -not (Test-Path $ForgePt)) {
Fail "No checkpoint found in $ForgeDir — run training first or remove --SkipTraining" Err "No checkpoint found in $ForgeDir - run training first"
exit 1 exit 1
} }
Step "Using checkpoint: $ForgePt" Step "Using checkpoint: $ForgePt"
@@ -256,23 +243,21 @@ if (-not $SkipEval) {
"--threads", "$Threads" "--threads", "$Threads"
) + $SeedArgs) ) + $SeedArgs)
$elapsed = ((Get-Date) - $t0).TotalMinutes $elapsed = [math]::Round(((Get-Date) - $t0).TotalMinutes, 1)
Ok "Forge eval complete in $([math]::Round($elapsed,1)) min -> $ForgeJson" Ok "Forge eval complete in $elapsed min -> $ForgeJson"
} else { } else {
Step "Skipping eval (--SkipEval)" Step "Skipping eval (-SkipEval flag set)"
} }
# ══════════════════════════════════════════════════════════════════════ # STEP 4 - Compare
# STEP 4 — Compare results Banner "Step 4/4 - Comparing results"
# ══════════════════════════════════════════════════════════════════════
Banner "Step 4/4 — Comparing results"
if (-not (Test-Path $BaselineJson)) { if (-not (Test-Path $BaselineJson)) {
Fail "Baseline result not found: $BaselineJson" Err "Baseline result not found: $BaselineJson"
exit 1 exit 1
} }
if (-not (Test-Path $ForgeJson)) { if (-not (Test-Path $ForgeJson)) {
Fail "Forge result not found: $ForgeJson" Err "Forge result not found: $ForgeJson"
exit 1 exit 1
} }
@@ -285,7 +270,6 @@ python scripts/compare_eval.py `
$exitCode = $LASTEXITCODE $exitCode = $LASTEXITCODE
# ── Final summary ──────────────────────────────────────────────────────
$totalMin = [math]::Round(((Get-Date) - $StartTime).TotalMinutes, 1) $totalMin = [math]::Round(((Get-Date) - $StartTime).TotalMinutes, 1)
Banner "Benchmark Complete" Banner "Benchmark Complete"
Write-Host " Total wall time : $totalMin min" Write-Host " Total wall time : $totalMin min"
@@ -295,14 +279,14 @@ Write-Host " Compare JSON : $CompareJson"
Write-Host "" Write-Host ""
if ($exitCode -eq 0) { if ($exitCode -eq 0) {
Ok "PASSED — Forge model meets the benchmark threshold." Ok "PASSED - Forge model meets the benchmark threshold."
Write-Host "" Write-Host ""
Write-Host " Next steps:" -ForegroundColor Cyan Write-Host " Next steps:" -ForegroundColor Cyan
Write-Host " git add results/eval-baseline.json results/eval-forge.json results/compare-result.json" Write-Host " git add results/eval-baseline.json results/eval-forge.json results/compare-result.json"
Write-Host " git commit -m 'results: forge-vs-synthetic benchmark run'" Write-Host " git commit -m 'results: forge-vs-synthetic benchmark run'"
Write-Host " git push" Write-Host " git push"
} else { } else {
Fail "FAILED — Forge model did not meet the benchmark threshold." Err "FAILED - Forge model did not meet the benchmark threshold."
Write-Host " Check $CompareJson for details." -ForegroundColor Yellow Write-Host " Check $CompareJson for details." -ForegroundColor Yellow
} }