fix(eval): parse batch JSON output instead of JSONL for Forge game results

This commit is contained in:
KoalaTrapLord authored and GitHub committed 2026-03-30 16:12:20 -04:00
1 parent 9e3cd45936
commit ddfe8e601e
1 file changed
+41 -1
+41 -1
View File
@@ -270,7 +270,47 @@ class ForgeEvaluator:
except (subprocess.TimeoutExpired, FileNotFoundError) as e: except (subprocess.TimeoutExpired, FileNotFoundError) as e:
logger.error("[Eval %s] Forge CLI failed: %s -- falling back to synthetic", self.cfg.run_id, e) logger.error("[Eval %s] Forge CLI failed: %s -- falling back to synthetic", self.cfg.run_id, e)
return self._run_synthetic_eval(service) return self._run_synthetic_eval(service)
return self._run_forge_via_subprocess(service) return self._parse_batch_json(service)
def _parse_batch_json(self, service) -> List[GameRecord]:
"""Parse the batch JSON output from LabCli Java subprocess."""
batch_path = os.path.join(
self.cfg.results_dir, f"eval-{self.cfg.run_id}-batch.json"
)
if not Path(batch_path).exists():
logger.warning("[Eval] Batch JSON not found at %s — falling back to synthetic", batch_path)
return self._run_synthetic_eval(service)
with open(batch_path) as f:
batch = json.load(f)
games_list = batch.get("games", [])
if not games_list:
logger.warning("[Eval] No games in batch JSON — falling back to synthetic")
return self._run_synthetic_eval(service)
records = []
for idx, g in enumerate(games_list):
won = g.get("winningSeat", -1) == 0
turns = g.get("totalTurns", 0)
pr = g.get("playerResults", [])
lp = pr[0].get("finalLife", 40) if len(pr) > 0 else 40
lo = pr[1].get("finalLife", 40) if len(pr) > 1 else 40
records.append(GameRecord(
game_index=idx,
won=won,
turns=turns,
life_final_policy=lp,
life_final_opponent=lo,
life_delta=lp - lo,
commander_damage_dealt=0,
illegal_actions=0,
entropy_mean=0.0,
inference_ms_mean=0.0,
decision_steps=turns,
))
if len(records) >= self.cfg.num_games:
break
logger.info("[Eval %s] Parsed %d games from batch JSON", self.cfg.run_id, len(records))
return records
def _run_forge_via_subprocess(self, service) -> List[GameRecord]: def _run_forge_via_subprocess(self, service) -> List[GameRecord]:
"""Parse JSONL files produced by a separately-launched Forge batch.""" """Parse JSONL files produced by a separately-launched Forge batch."""
records = [] records = []