fix(eval): parse batch JSON output instead of JSONL for Forge game results
This commit is contained in:
1 parent
9e3cd45936
commit
ddfe8e601e
1 file changed
+41
-1
@@ -270,7 +270,47 @@ class ForgeEvaluator:
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError) as e:
|
||||
logger.error("[Eval %s] Forge CLI failed: %s -- falling back to synthetic", self.cfg.run_id, e)
|
||||
return self._run_synthetic_eval(service)
|
||||
return self._run_forge_via_subprocess(service)
|
||||
return self._parse_batch_json(service)
|
||||
|
||||
def _parse_batch_json(self, service) -> List[GameRecord]:
|
||||
"""Parse the batch JSON output from LabCli Java subprocess."""
|
||||
batch_path = os.path.join(
|
||||
self.cfg.results_dir, f"eval-{self.cfg.run_id}-batch.json"
|
||||
)
|
||||
if not Path(batch_path).exists():
|
||||
logger.warning("[Eval] Batch JSON not found at %s — falling back to synthetic", batch_path)
|
||||
return self._run_synthetic_eval(service)
|
||||
with open(batch_path) as f:
|
||||
batch = json.load(f)
|
||||
games_list = batch.get("games", [])
|
||||
if not games_list:
|
||||
logger.warning("[Eval] No games in batch JSON — falling back to synthetic")
|
||||
return self._run_synthetic_eval(service)
|
||||
records = []
|
||||
for idx, g in enumerate(games_list):
|
||||
won = g.get("winningSeat", -1) == 0
|
||||
turns = g.get("totalTurns", 0)
|
||||
pr = g.get("playerResults", [])
|
||||
lp = pr[0].get("finalLife", 40) if len(pr) > 0 else 40
|
||||
lo = pr[1].get("finalLife", 40) if len(pr) > 1 else 40
|
||||
records.append(GameRecord(
|
||||
game_index=idx,
|
||||
won=won,
|
||||
turns=turns,
|
||||
life_final_policy=lp,
|
||||
life_final_opponent=lo,
|
||||
life_delta=lp - lo,
|
||||
commander_damage_dealt=0,
|
||||
illegal_actions=0,
|
||||
entropy_mean=0.0,
|
||||
inference_ms_mean=0.0,
|
||||
decision_steps=turns,
|
||||
))
|
||||
if len(records) >= self.cfg.num_games:
|
||||
break
|
||||
logger.info("[Eval %s] Parsed %d games from batch JSON", self.cfg.run_id, len(records))
|
||||
return records
|
||||
|
||||
def _run_forge_via_subprocess(self, service) -> List[GameRecord]:
|
||||
"""Parse JSONL files produced by a separately-launched Forge batch."""
|
||||
records = []
|
||||
|
||||
Reference in new issue
Block a user