fix(eval): replace Java bridge imports with subprocess CLI launch for Forge eval

This commit is contained in:
rgriffith committed 2026-03-30 14:23:52 -04:00
1 parent 517c17047b
commit 6fadeaa387
1 file changed
+38 -26
+38 -26
View File
@@ -39,6 +39,7 @@ import os
import sys import sys
import time import time
import uuid import uuid
import subprocess
from dataclasses import dataclass, field, asdict from dataclasses import dataclass, field, asdict
from pathlib import Path from pathlib import Path
from typing import Dict, List, Optional from typing import Dict, List, Optional
@@ -231,34 +232,45 @@ class ForgeEvaluator:
# ── Real Forge game runner ─────────────── # ── Real Forge game runner ───────────────
def _run_forge_games(self, service) -> List[GameRecord]: def _run_forge_games(self, service) -> List[GameRecord]:
"""Run games via MultiThreadBatchRunner + Forge subprocess.""" """Launch LabCli Java subprocess with --ml-log, then parse JSONL output."""
try: logger.info("[Eval %s] Launching Forge CLI subprocess for %d games", self.cfg.run_id, self.cfg.num_games)
from commanderailab.batch.MultiThreadBatchRunner import MultiThreadBatchRunner # type: ignore deck_args = []
from commanderailab.schema.BatchResult import DeckInfo # type: ignore for i, df in enumerate(self.cfg.deck_files[:4]):
from commanderailab.ai.AiPolicy import AiPolicy # type: ignore deck_args.extend([f"--deck{i+1}", Path(df).stem])
except ImportError: # Resolve Java 17
logger.warning("[Eval] Java bridge not importable from Python — using JSONL parsing fallback") java_exe = self.cfg.java_path
return self._run_forge_via_subprocess(service) for c in [r"C:\Program Files\Eclipse Adoptium\jdk-17.0.18.8-hotspot\bin\java.exe",
r"C:\Program Files\Java\jdk-17\bin\java.exe"]:
decks = [ if Path(c).exists():
DeckInfo(name=Path(f).stem, file=f) java_exe = c
for f in self.cfg.deck_files[:4] break
lab_jar = os.path.join(project_root, "target", "commander-ai-lab-1.0.0-SNAPSHOT.jar")
if not Path(lab_jar).exists():
logger.error("[Eval] Lab JAR not found at %s -- run mvn package -DskipTests first", lab_jar)
return self._run_synthetic_eval(service)
cmd = [
java_exe, "-jar", lab_jar,
"--forge-jar", self.cfg.forge_jar,
"--forge-dir", self.cfg.forge_work_dir,
*deck_args,
"--games", str(self.cfg.num_games),
"--threads", str(self.cfg.num_threads),
"--clock", str(self.cfg.clock_seconds),
"--ml-log",
"--output", os.path.join(self.cfg.results_dir, f"eval-{self.cfg.run_id}-batch.json"),
] ]
runner = MultiThreadBatchRunner( if self.cfg.seed is not None:
self.cfg.forge_jar, self.cfg.forge_work_dir, cmd.extend(["--seed", str(self.cfg.seed)])
decks, AiPolicy.MIDRANGE, logger.info("[Eval %s] CMD: %s", self.cfg.run_id, " ".join(cmd))
self.cfg.num_threads, True, self.cfg.clock_seconds, self.cfg.java_path
)
runner.enableMlLogging(self.cfg.results_dir, f"eval-{self.cfg.run_id}")
runner.warmPool()
try: try:
results = runner.runBatch(self.cfg.num_games, self.cfg.seed) result = subprocess.run(cmd, capture_output=True, text=True,
finally: timeout=self.cfg.clock_seconds * self.cfg.num_games + 300)
runner.shutdownPool() if result.returncode != 0:
logger.warning("[Eval %s] Forge CLI exit %d: %s", self.cfg.run_id, result.returncode, result.stderr[:500])
return self._java_results_to_records(results, service) except (subprocess.TimeoutExpired, FileNotFoundError) as e:
logger.error("[Eval %s] Forge CLI failed: %s -- falling back to synthetic", self.cfg.run_id, e)
return self._run_synthetic_eval(service)
return self._run_forge_via_subprocess(service)
def _run_forge_via_subprocess(self, service) -> List[GameRecord]: def _run_forge_via_subprocess(self, service) -> List[GameRecord]:
"""Parse JSONL files produced by a separately-launched Forge batch.""" """Parse JSONL files produced by a separately-launched Forge batch."""
records = [] records = []