#!/usr/bin/env python """ Commander AI Lab — Compare Eval Results (Issue #153) ═════════════════════════════════════════════════════ Loads two eval result JSONs produced by eval_policy.py, computes win-rate delta, 95% CI, and Fisher's exact test for significance. Usage: python scripts/compare_eval.py \ --baseline results/eval-baseline.json \ --forge results/eval-forge.json # Custom threshold python scripts/compare_eval.py \ --baseline results/eval-baseline.json \ --forge results/eval-forge.json \ --min-delta 0.05 \ --alpha 0.05 Exit codes: 0 — Forge model passes (delta >= min-delta AND p < alpha) 1 — Forge model fails (delta below threshold OR not significant) 2 — Input error (missing files, bad JSON) """ import argparse import json import math import sys from pathlib import Path # ───────────────────────────────────────────── # Statistics helpers # ───────────────────────────────────────────── def wilson_ci(wins: int, n: int, z: float = 1.96): """Wilson score interval for a proportion.""" if n == 0: return 0.0, 0.0 p = wins / n denom = 1 + z ** 2 / n centre = (p + z ** 2 / (2 * n)) / denom margin = (z * math.sqrt(p * (1 - p) / n + z ** 2 / (4 * n ** 2))) / denom return max(0.0, centre - margin), min(1.0, centre + margin) def fishers_exact(a_wins, a_n, b_wins, b_n): """ Two-tailed Fisher's exact test. Contingency table: [[a_wins, a_n - a_wins], [b_wins, b_n - b_wins]] Returns (odds_ratio, p_value). Uses scipy if available, otherwise a pure-Python hypergeometric approximation suitable for n >= 50. """ try: from scipy.stats import fisher_exact table = [ [a_wins, a_n - a_wins], [b_wins, b_n - b_wins], ] oddsratio, pvalue = fisher_exact(table, alternative="two-sided") return float(oddsratio), float(pvalue) except ImportError: pass # Pure-Python fallback: hypergeometric tail sum (two-tailed) # Works well when all cell counts >= 5 def _log_comb(n, k): if k < 0 or k > n: return -math.inf return ( sum(math.log(n - i) for i in range(k)) - sum(math.log(i + 1) for i in range(k)) ) N = a_n + b_n K = a_wins + b_wins # total wins n = a_n # first group size k_obs = a_wins log_denom = _log_comb(N, K) p_obs = math.exp(_log_comb(n, k_obs) + _log_comb(N - n, K - k_obs) - log_denom) p_val = 0.0 k_lo = max(0, K - b_n) k_hi = min(K, n) for k in range(k_lo, k_hi + 1): p_k = math.exp(_log_comb(n, k) + _log_comb(N - n, K - k) - log_denom) if p_k <= p_obs + 1e-10: p_val += p_k a_loss = a_n - a_wins b_loss = b_n - b_wins or_val = ( (a_wins * b_loss / (a_loss * b_wins)) if a_loss > 0 and b_wins > 0 else float("inf") ) return or_val, min(p_val, 1.0) # ───────────────────────────────────────────── # JSON loader # ───────────────────────────────────────────── def load_result(path: str) -> dict: p = Path(path) if not p.exists(): print(f"ERROR: File not found: {path}", file=sys.stderr) sys.exit(2) try: with open(p) as f: data = json.load(f) except json.JSONDecodeError as e: print(f"ERROR: Invalid JSON in {path}: {e}", file=sys.stderr) sys.exit(2) # Accept both {summary: {...}} and flat {win_rate: ...} shapes if "summary" in data: return data["summary"] return data # ───────────────────────────────────────────── # Comparison table printer # ───────────────────────────────────────────── def fmt_pct(v) -> str: try: return f"{float(v):.1%}" except (TypeError, ValueError): return str(v) def fmt_f(v, dp=2) -> str: try: return f"{float(v):.{dp}f}" except (TypeError, ValueError): return str(v) def print_comparison(b: dict, f: dict, delta: float, ci_lo: float, ci_hi: float, odds: float, pval: float, min_delta: float, alpha: float): W = 62 SEP = "─" * W def row(label, bval, fval, delta_str=""): print(f" {label:<28} {bval:>10} {fval:>10} {delta_str:>8}") print() print("=" * W) print(" COMMANDER AI LAB — EVAL COMPARISON") print("=" * W) print(f" Baseline : {b.get('checkpoint_path', b.get('run_id', '?'))}") print(f" Forge : {f.get('checkpoint_path', f.get('run_id', '?'))}") print(SEP) print(f" {'Metric':<28} {'Baseline':>10} {'Forge':>10} {'Delta':>8}") print(SEP) b_n, f_n = int(b.get("num_games", 0)), int(f.get("num_games", 0)) b_w, f_w = int(b.get("wins", 0)), int(f.get("wins", 0)) row("Games", str(b_n), str(f_n)) row("Wins", str(b_w), str(f_w), f"{f_w - b_w:+d}") row("Win rate", fmt_pct(b.get("win_rate")), fmt_pct(f.get("win_rate")), f"{delta:+.1%}") b_lo, b_hi = wilson_ci(b_w, b_n) f_lo, f_hi = wilson_ci(f_w, f_n) row(" 95% CI", f"[{fmt_pct(b_lo)}, {fmt_pct(b_hi)}]", f"[{fmt_pct(f_lo)}, {fmt_pct(f_hi)}]", ) row(" Δ 95% CI", "", "", f"[{ci_lo:+.1%}, {ci_hi:+.1%}]") print(SEP) row("Avg turns", fmt_f(b.get("avg_turns")), fmt_f(f.get("avg_turns")), fmt_f(float(f.get("avg_turns", 0)) - float(b.get("avg_turns", 0)), 1)) row("Avg life Δ", fmt_f(b.get("avg_life_delta")), fmt_f(f.get("avg_life_delta")), fmt_f(float(f.get("avg_life_delta", 0)) - float(b.get("avg_life_delta", 0)), 1)) row("Avg cmd dmg", fmt_f(b.get("avg_commander_damage")), fmt_f(f.get("avg_commander_damage")), fmt_f(float(f.get("avg_commander_damage", 0)) - float(b.get("avg_commander_damage", 0)), 1)) row("Illegal acts/g", fmt_f(b.get("avg_illegal_actions")), fmt_f(f.get("avg_illegal_actions")), fmt_f(float(f.get("avg_illegal_actions", 0)) - float(b.get("avg_illegal_actions", 0)), 2)) row("Avg entropy", fmt_f(b.get("avg_entropy"), 4), fmt_f(f.get("avg_entropy"), 4), fmt_f(float(f.get("avg_entropy", 0)) - float(b.get("avg_entropy", 0)), 4)) row("Avg infer ms", fmt_f(b.get("avg_inference_ms"), 1), fmt_f(f.get("avg_inference_ms"), 1), fmt_f(float(f.get("avg_inference_ms", 0)) - float(b.get("avg_inference_ms", 0)), 1)) print(SEP) # Significance block print(f" Fisher's exact test (two-tailed)") print(f" odds ratio : {odds:.4f}") print(f" p-value : {pval:.6f}") print(f" alpha : {alpha}") print(SEP) # Pass / fail verdict delta_pass = delta >= min_delta sig_pass = pval < alpha passed = delta_pass and sig_pass if passed: verdict = "PASS ✓ Forge model outperforms baseline" elif not delta_pass and not sig_pass: verdict = f"FAIL ✗ delta ({delta:+.1%}) < {min_delta:.1%} AND not significant (p={pval:.4f})" elif not delta_pass: verdict = f"FAIL ✗ delta ({delta:+.1%}) < threshold ({min_delta:.1%})" else: verdict = f"FAIL ✗ not statistically significant (p={pval:.4f} >= alpha={alpha})" print(f" {verdict}") print("=" * W) print() # ───────────────────────────────────────────── # CLI # ───────────────────────────────────────────── def parse_args(): p = argparse.ArgumentParser( description="Compare two eval result JSONs from eval_policy.py.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p.add_argument("--baseline", required=True, help="Baseline (synthetic) eval JSON") p.add_argument("--forge", required=True, help="Forge-trained eval JSON") p.add_argument("--min-delta", type=float, default=0.05, help="Minimum absolute win-rate improvement to pass") p.add_argument("--alpha", type=float, default=0.05, help="Significance level for Fisher's exact test") p.add_argument("--json-out", default="", help="Optional: write comparison result to this JSON file") return p.parse_args() def main() -> int: args = parse_args() b = load_result(args.baseline) f = load_result(args.forge) b_n, f_n = int(b.get("num_games", 0)), int(f.get("num_games", 0)) b_w, f_w = int(b.get("wins", 0)), int(f.get("wins", 0)) b_wr = b_w / max(b_n, 1) f_wr = f_w / max(f_n, 1) delta = f_wr - b_wr # Delta 95% CI via normal approximation se_b = math.sqrt(b_wr * (1 - b_wr) / max(b_n, 1)) se_f = math.sqrt(f_wr * (1 - f_wr) / max(f_n, 1)) se_delta = math.sqrt(se_b ** 2 + se_f ** 2) ci_lo = delta - 1.96 * se_delta ci_hi = delta + 1.96 * se_delta odds, pval = fishers_exact(b_w, b_n, f_w, f_n) print_comparison(b, f, delta, ci_lo, ci_hi, odds, pval, args.min_delta, args.alpha) if args.json_out: out = { "baseline": args.baseline, "forge": args.forge, "baseline_win_rate": b_wr, "forge_win_rate": f_wr, "delta": delta, "delta_ci95_low": ci_lo, "delta_ci95_high": ci_hi, "odds_ratio": odds, "p_value": pval, "alpha": args.alpha, "min_delta": args.min_delta, "passed": delta >= args.min_delta and pval < args.alpha, } Path(args.json_out).parent.mkdir(parents=True, exist_ok=True) with open(args.json_out, "w") as jf: json.dump(out, jf, indent=2) print(f"Comparison JSON written to: {args.json_out}") passed = delta >= args.min_delta and pval < args.alpha return 0 if passed else 1 if __name__ == "__main__": sys.exit(main())