rgriffithandClaude Opus 4.6
9b79d18201
Left action panel revised to show triggers list + Create Token/Auto-Arrange. CMD DMG badges added to center life-panel score areas (click opens matrix). Removed renderEnhancedPanels calls that split the battlefield on damage adjust. Added encoding='utf-8' to all bare open() calls across 17 Python files to prevent cp1252 crashes on MTG card text with smart quotes/em-dashes. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
302 lines
10 KiB
Python
302 lines
10 KiB
Python
#!/usr/bin/env python
|
|
"""
|
|
Commander AI Lab — Compare Eval Results (Issue #153)
|
|
═════════════════════════════════════════════════════
|
|
Loads two eval result JSONs produced by eval_policy.py, computes
|
|
win-rate delta, 95% CI, and Fisher's exact test for significance.
|
|
|
|
Usage:
|
|
python scripts/compare_eval.py \
|
|
--baseline results/eval-baseline.json \
|
|
--forge results/eval-forge.json
|
|
|
|
# Custom threshold
|
|
python scripts/compare_eval.py \
|
|
--baseline results/eval-baseline.json \
|
|
--forge results/eval-forge.json \
|
|
--min-delta 0.05 \
|
|
--alpha 0.05
|
|
|
|
Exit codes:
|
|
0 — Forge model passes (delta >= min-delta AND p < alpha)
|
|
1 — Forge model fails (delta below threshold OR not significant)
|
|
2 — Input error (missing files, bad JSON)
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Statistics helpers
|
|
# ─────────────────────────────────────────────
|
|
|
|
def wilson_ci(wins: int, n: int, z: float = 1.96):
|
|
"""Wilson score interval for a proportion."""
|
|
if n == 0:
|
|
return 0.0, 0.0
|
|
p = wins / n
|
|
denom = 1 + z ** 2 / n
|
|
centre = (p + z ** 2 / (2 * n)) / denom
|
|
margin = (z * math.sqrt(p * (1 - p) / n + z ** 2 / (4 * n ** 2))) / denom
|
|
return max(0.0, centre - margin), min(1.0, centre + margin)
|
|
|
|
|
|
def fishers_exact(a_wins, a_n, b_wins, b_n):
|
|
"""
|
|
Two-tailed Fisher's exact test.
|
|
Contingency table:
|
|
[[a_wins, a_n - a_wins],
|
|
[b_wins, b_n - b_wins]]
|
|
|
|
Returns (odds_ratio, p_value).
|
|
Uses scipy if available, otherwise a pure-Python hypergeometric
|
|
approximation suitable for n >= 50.
|
|
"""
|
|
try:
|
|
from scipy.stats import fisher_exact
|
|
table = [
|
|
[a_wins, a_n - a_wins],
|
|
[b_wins, b_n - b_wins],
|
|
]
|
|
oddsratio, pvalue = fisher_exact(table, alternative="two-sided")
|
|
return float(oddsratio), float(pvalue)
|
|
except ImportError:
|
|
pass
|
|
|
|
# Pure-Python fallback: hypergeometric tail sum (two-tailed)
|
|
# Works well when all cell counts >= 5
|
|
def _log_comb(n, k):
|
|
if k < 0 or k > n:
|
|
return -math.inf
|
|
return (
|
|
sum(math.log(n - i) for i in range(k))
|
|
- sum(math.log(i + 1) for i in range(k))
|
|
)
|
|
|
|
N = a_n + b_n
|
|
K = a_wins + b_wins # total wins
|
|
n = a_n # first group size
|
|
k_obs = a_wins
|
|
|
|
log_denom = _log_comb(N, K)
|
|
p_obs = math.exp(_log_comb(n, k_obs) + _log_comb(N - n, K - k_obs) - log_denom)
|
|
|
|
p_val = 0.0
|
|
k_lo = max(0, K - b_n)
|
|
k_hi = min(K, n)
|
|
for k in range(k_lo, k_hi + 1):
|
|
p_k = math.exp(_log_comb(n, k) + _log_comb(N - n, K - k) - log_denom)
|
|
if p_k <= p_obs + 1e-10:
|
|
p_val += p_k
|
|
|
|
a_loss = a_n - a_wins
|
|
b_loss = b_n - b_wins
|
|
or_val = (
|
|
(a_wins * b_loss / (a_loss * b_wins))
|
|
if a_loss > 0 and b_wins > 0
|
|
else float("inf")
|
|
)
|
|
return or_val, min(p_val, 1.0)
|
|
|
|
|
|
# ─────────────────────────────────────────────
|
|
# JSON loader
|
|
# ─────────────────────────────────────────────
|
|
|
|
def load_result(path: str) -> dict:
|
|
p = Path(path)
|
|
if not p.exists():
|
|
print(f"ERROR: File not found: {path}", file=sys.stderr)
|
|
sys.exit(2)
|
|
try:
|
|
with open(p, encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
except json.JSONDecodeError as e:
|
|
print(f"ERROR: Invalid JSON in {path}: {e}", file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
# Accept both {summary: {...}} and flat {win_rate: ...} shapes
|
|
if "summary" in data:
|
|
return data["summary"]
|
|
return data
|
|
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Comparison table printer
|
|
# ─────────────────────────────────────────────
|
|
|
|
def fmt_pct(v) -> str:
|
|
try:
|
|
return f"{float(v):.1%}"
|
|
except (TypeError, ValueError):
|
|
return str(v)
|
|
|
|
|
|
def fmt_f(v, dp=2) -> str:
|
|
try:
|
|
return f"{float(v):.{dp}f}"
|
|
except (TypeError, ValueError):
|
|
return str(v)
|
|
|
|
|
|
def print_comparison(b: dict, f: dict, delta: float, ci_lo: float, ci_hi: float,
|
|
odds: float, pval: float, min_delta: float, alpha: float):
|
|
W = 62
|
|
SEP = "─" * W
|
|
|
|
def row(label, bval, fval, delta_str=""):
|
|
print(f" {label:<28} {bval:>10} {fval:>10} {delta_str:>8}")
|
|
|
|
print()
|
|
print("=" * W)
|
|
print(" COMMANDER AI LAB — EVAL COMPARISON")
|
|
print("=" * W)
|
|
print(f" Baseline : {b.get('checkpoint_path', b.get('run_id', '?'))}")
|
|
print(f" Forge : {f.get('checkpoint_path', f.get('run_id', '?'))}")
|
|
print(SEP)
|
|
print(f" {'Metric':<28} {'Baseline':>10} {'Forge':>10} {'Delta':>8}")
|
|
print(SEP)
|
|
|
|
b_n, f_n = int(b.get("num_games", 0)), int(f.get("num_games", 0))
|
|
b_w, f_w = int(b.get("wins", 0)), int(f.get("wins", 0))
|
|
|
|
row("Games", str(b_n), str(f_n))
|
|
row("Wins", str(b_w), str(f_w), f"{f_w - b_w:+d}")
|
|
row("Win rate", fmt_pct(b.get("win_rate")), fmt_pct(f.get("win_rate")),
|
|
f"{delta:+.1%}")
|
|
|
|
b_lo, b_hi = wilson_ci(b_w, b_n)
|
|
f_lo, f_hi = wilson_ci(f_w, f_n)
|
|
row(" 95% CI",
|
|
f"[{fmt_pct(b_lo)}, {fmt_pct(b_hi)}]",
|
|
f"[{fmt_pct(f_lo)}, {fmt_pct(f_hi)}]",
|
|
)
|
|
row(" Δ 95% CI", "", "", f"[{ci_lo:+.1%}, {ci_hi:+.1%}]")
|
|
print(SEP)
|
|
row("Avg turns",
|
|
fmt_f(b.get("avg_turns")),
|
|
fmt_f(f.get("avg_turns")),
|
|
fmt_f(float(f.get("avg_turns", 0)) - float(b.get("avg_turns", 0)), 1))
|
|
row("Avg life Δ",
|
|
fmt_f(b.get("avg_life_delta")),
|
|
fmt_f(f.get("avg_life_delta")),
|
|
fmt_f(float(f.get("avg_life_delta", 0)) - float(b.get("avg_life_delta", 0)), 1))
|
|
row("Avg cmd dmg",
|
|
fmt_f(b.get("avg_commander_damage")),
|
|
fmt_f(f.get("avg_commander_damage")),
|
|
fmt_f(float(f.get("avg_commander_damage", 0)) - float(b.get("avg_commander_damage", 0)), 1))
|
|
row("Illegal acts/g",
|
|
fmt_f(b.get("avg_illegal_actions")),
|
|
fmt_f(f.get("avg_illegal_actions")),
|
|
fmt_f(float(f.get("avg_illegal_actions", 0)) - float(b.get("avg_illegal_actions", 0)), 2))
|
|
row("Avg entropy",
|
|
fmt_f(b.get("avg_entropy"), 4),
|
|
fmt_f(f.get("avg_entropy"), 4),
|
|
fmt_f(float(f.get("avg_entropy", 0)) - float(b.get("avg_entropy", 0)), 4))
|
|
row("Avg infer ms",
|
|
fmt_f(b.get("avg_inference_ms"), 1),
|
|
fmt_f(f.get("avg_inference_ms"), 1),
|
|
fmt_f(float(f.get("avg_inference_ms", 0)) - float(b.get("avg_inference_ms", 0)), 1))
|
|
print(SEP)
|
|
|
|
# Significance block
|
|
print(f" Fisher's exact test (two-tailed)")
|
|
print(f" odds ratio : {odds:.4f}")
|
|
print(f" p-value : {pval:.6f}")
|
|
print(f" alpha : {alpha}")
|
|
print(SEP)
|
|
|
|
# Pass / fail verdict
|
|
delta_pass = delta >= min_delta
|
|
sig_pass = pval < alpha
|
|
passed = delta_pass and sig_pass
|
|
|
|
if passed:
|
|
verdict = "PASS ✓ Forge model outperforms baseline"
|
|
elif not delta_pass and not sig_pass:
|
|
verdict = f"FAIL ✗ delta ({delta:+.1%}) < {min_delta:.1%} AND not significant (p={pval:.4f})"
|
|
elif not delta_pass:
|
|
verdict = f"FAIL ✗ delta ({delta:+.1%}) < threshold ({min_delta:.1%})"
|
|
else:
|
|
verdict = f"FAIL ✗ not statistically significant (p={pval:.4f} >= alpha={alpha})"
|
|
|
|
print(f" {verdict}")
|
|
print("=" * W)
|
|
print()
|
|
|
|
|
|
# ─────────────────────────────────────────────
|
|
# CLI
|
|
# ─────────────────────────────────────────────
|
|
|
|
def parse_args():
|
|
p = argparse.ArgumentParser(
|
|
description="Compare two eval result JSONs from eval_policy.py.",
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
)
|
|
p.add_argument("--baseline", required=True, help="Baseline (synthetic) eval JSON")
|
|
p.add_argument("--forge", required=True, help="Forge-trained eval JSON")
|
|
p.add_argument("--min-delta", type=float, default=0.05,
|
|
help="Minimum absolute win-rate improvement to pass")
|
|
p.add_argument("--alpha", type=float, default=0.05,
|
|
help="Significance level for Fisher's exact test")
|
|
p.add_argument("--json-out", default="",
|
|
help="Optional: write comparison result to this JSON file")
|
|
return p.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
|
|
b = load_result(args.baseline)
|
|
f = load_result(args.forge)
|
|
|
|
b_n, f_n = int(b.get("num_games", 0)), int(f.get("num_games", 0))
|
|
b_w, f_w = int(b.get("wins", 0)), int(f.get("wins", 0))
|
|
b_wr = b_w / max(b_n, 1)
|
|
f_wr = f_w / max(f_n, 1)
|
|
|
|
delta = f_wr - b_wr
|
|
|
|
# Delta 95% CI via normal approximation
|
|
se_b = math.sqrt(b_wr * (1 - b_wr) / max(b_n, 1))
|
|
se_f = math.sqrt(f_wr * (1 - f_wr) / max(f_n, 1))
|
|
se_delta = math.sqrt(se_b ** 2 + se_f ** 2)
|
|
ci_lo = delta - 1.96 * se_delta
|
|
ci_hi = delta + 1.96 * se_delta
|
|
|
|
odds, pval = fishers_exact(b_w, b_n, f_w, f_n)
|
|
|
|
print_comparison(b, f, delta, ci_lo, ci_hi, odds, pval, args.min_delta, args.alpha)
|
|
|
|
if args.json_out:
|
|
out = {
|
|
"baseline": args.baseline,
|
|
"forge": args.forge,
|
|
"baseline_win_rate": b_wr,
|
|
"forge_win_rate": f_wr,
|
|
"delta": delta,
|
|
"delta_ci95_low": ci_lo,
|
|
"delta_ci95_high": ci_hi,
|
|
"odds_ratio": odds,
|
|
"p_value": pval,
|
|
"alpha": args.alpha,
|
|
"min_delta": args.min_delta,
|
|
"passed": delta >= args.min_delta and pval < args.alpha,
|
|
}
|
|
Path(args.json_out).parent.mkdir(parents=True, exist_ok=True)
|
|
with open(args.json_out, "w", encoding="utf-8") as jf:
|
|
json.dump(out, jf, indent=2)
|
|
print(f"Comparison JSON written to: {args.json_out}")
|
|
|
|
passed = delta >= args.min_delta and pval < args.alpha
|
|
return 0 if passed else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|