fix(eval): parse batch JSON output instead of JSONL for Forge game results
This commit is contained in:
1 parent
9e3cd45936
commit
ddfe8e601e
1 file changed
+41
-1
@@ -270,7 +270,47 @@ class ForgeEvaluator:
|
|||||||
except (subprocess.TimeoutExpired, FileNotFoundError) as e:
|
except (subprocess.TimeoutExpired, FileNotFoundError) as e:
|
||||||
logger.error("[Eval %s] Forge CLI failed: %s -- falling back to synthetic", self.cfg.run_id, e)
|
logger.error("[Eval %s] Forge CLI failed: %s -- falling back to synthetic", self.cfg.run_id, e)
|
||||||
return self._run_synthetic_eval(service)
|
return self._run_synthetic_eval(service)
|
||||||
return self._run_forge_via_subprocess(service)
|
return self._parse_batch_json(service)
|
||||||
|
|
||||||
|
def _parse_batch_json(self, service) -> List[GameRecord]:
|
||||||
|
"""Parse the batch JSON output from LabCli Java subprocess."""
|
||||||
|
batch_path = os.path.join(
|
||||||
|
self.cfg.results_dir, f"eval-{self.cfg.run_id}-batch.json"
|
||||||
|
)
|
||||||
|
if not Path(batch_path).exists():
|
||||||
|
logger.warning("[Eval] Batch JSON not found at %s — falling back to synthetic", batch_path)
|
||||||
|
return self._run_synthetic_eval(service)
|
||||||
|
with open(batch_path) as f:
|
||||||
|
batch = json.load(f)
|
||||||
|
games_list = batch.get("games", [])
|
||||||
|
if not games_list:
|
||||||
|
logger.warning("[Eval] No games in batch JSON — falling back to synthetic")
|
||||||
|
return self._run_synthetic_eval(service)
|
||||||
|
records = []
|
||||||
|
for idx, g in enumerate(games_list):
|
||||||
|
won = g.get("winningSeat", -1) == 0
|
||||||
|
turns = g.get("totalTurns", 0)
|
||||||
|
pr = g.get("playerResults", [])
|
||||||
|
lp = pr[0].get("finalLife", 40) if len(pr) > 0 else 40
|
||||||
|
lo = pr[1].get("finalLife", 40) if len(pr) > 1 else 40
|
||||||
|
records.append(GameRecord(
|
||||||
|
game_index=idx,
|
||||||
|
won=won,
|
||||||
|
turns=turns,
|
||||||
|
life_final_policy=lp,
|
||||||
|
life_final_opponent=lo,
|
||||||
|
life_delta=lp - lo,
|
||||||
|
commander_damage_dealt=0,
|
||||||
|
illegal_actions=0,
|
||||||
|
entropy_mean=0.0,
|
||||||
|
inference_ms_mean=0.0,
|
||||||
|
decision_steps=turns,
|
||||||
|
))
|
||||||
|
if len(records) >= self.cfg.num_games:
|
||||||
|
break
|
||||||
|
logger.info("[Eval %s] Parsed %d games from batch JSON", self.cfg.run_id, len(records))
|
||||||
|
return records
|
||||||
|
|
||||||
def _run_forge_via_subprocess(self, service) -> List[GameRecord]:
|
def _run_forge_via_subprocess(self, service) -> List[GameRecord]:
|
||||||
"""Parse JSONL files produced by a separately-launched Forge batch."""
|
"""Parse JSONL files produced by a separately-launched Forge batch."""
|
||||||
records = []
|
records = []
|
||||||
|
|||||||
Reference in new issue
Block a user