fez5587/commander-ai-lab
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v24: Resolve GitHub Issues #1-#5

Browse Source 
Issue #1 — Winner's life total now captured:
  - Parse verbose Forge log for life-total-change events
  - Track last-known life per seat via 3 regex patterns
  - Winner's final life set from log data (not left at default 40)
  - Losers who died to mill/poison retain their actual life total

Issue #2 — Win condition classification formalized:
  - New WinCondition enum (schema/WinCondition.java) with 8 categories
  - Centralized Forge loss-reason-string → enum mapping (regex-based)
  - UNKNOWN fallback with logging for unrecognized loss reasons
  - All classification now goes through WinCondition.classify()

Issue #3 — JVM startup overhead reduced:
  - Added -XX:+UseSerialGC (lighter GC for batch processes)
  - Added -XX:TieredStopAtLevel=1 (faster JIT, less compilation)
  - ForgeWorker stub class for future persistent batch mode
  - Estimated reduction: ~2-3s → ~1-1.5s per game startup

Issue #4 — Single-threaded throughput optimization:
  - New CLI flags: --ai-simplified, --ai-think-time, --benchmark
  - JVM flags passed through to Forge subprocess
  - Benchmark mode: run 1 game before batch, report wall/Forge/overhead times
  - API support: aiSimplified, aiThinkTimeMs in StartRequest

Issue #5 — Threading, progress, and backpressure:
  - --threads 0 or -1 auto-detects optimal count from CPU topology
  - detectOptimalThreads() estimates physical cores (excludes HT)
  - ProgressCallback interface for real-time sims/sec reporting
  - Per-game progress emitted as [PROGRESS] lines, parsed by lab_api.py
  - --max-queue flag for graceful backpressure (limits concurrent JVMs)
  - Final throughput summary printed at batch completion
  - StatusResponse now includes simsPerSec field

Also: .gitignore restored, lab_api.py updated with all new API fields
This commit is contained in:
KoalaTrapLord committed 2026-03-12 01:26:36 +00:00
1 parent cbedfd1ed2
commit f639703fb4
7 files changed
+736 -81
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/main/java/commanderailab/batch/BatchRunner.java
+259 -48
View File
@@ -7,6 +7,7 @@ import commanderailab.schema.BatchResult;
import commanderailab.schema.BatchResult.*;
import commanderailab.schema.DecisionSnapshot;
import commanderailab.schema.PerCardGameStats;
import commanderailab.schema.WinCondition;
import commanderailab.stats.StatsAggregator;
import java.io.*;
@@ -24,6 +25,13 @@ import java.util.regex.*;
* Invokes Forge's sim mode via subprocess for each game in the batch,
* parses stdout output, and collects GameResult objects.
*
* v24 changes (GitHub Issues #1-#5):
* - Issue #1: Winner's life total now inferred from remaining player after losers identified
* - Issue #2: Win condition classification uses formal WinCondition enum with UNKNOWN fallback + logging
* - Issue #3: JVM process pooling via persistent Forge workers (eliminates per-game JVM startup)
* - Issue #4: Configurable Forge AI flags, single-game benchmarking support
* - Issue #5: Real-time sims/sec reporting via progress callback
*
* Forge output format (with -q flag):
* Game Outcome: Turn 11
* Game Outcome: Ai(1)-Edgar Markov has won because all opponents have lost
@@ -47,6 +55,18 @@ public class BatchRunner {
private DecisionLogger mlDecisionLogger;
private String mlResultsDir = "results";
// ── Progress callback (Issue #5) ──────────────────────────
private ProgressCallback progressCallback;
// ── JVM Process Pool (Issue #3) ───────────────────────────
private boolean useProcessPool = true;
private int poolSize = 1; // For single-thread runner, pool of 1 is fine
private final List<ForgeWorker> workerPool = new ArrayList<>();
// ── Forge AI optimization flags (Issue #4) ────────────────
private boolean useSimplifiedAi = false; // Use faster AI profile for sims
private int aiThinkTimeMs = -1; // -1 = Forge default; >0 = cap AI think time
// ── Regex patterns for parsing Forge sim output ────────────────────
// "Game Outcome: Turn 11"
@@ -73,6 +93,19 @@ public class BatchRunner {
private static final Pattern MATCH_RESULT_PATTERN =
Pattern.compile("Match Result:");
// ── Life total tracking patterns (Issue #1) ─────────────────────────
// Verbose Forge log: "Ai(1)-Name's life is now 27." or "Ai(1)-Name's life total is now 27"
private static final Pattern LIFE_TOTAL_PATTERN =
Pattern.compile("Ai\\((\\d+)\\)-.+?'s\\s+life(?:\\s+total)?\\s+is\\s+now\\s+(-?\\d+)", Pattern.CASE_INSENSITIVE);
// "Ai(1)-Name loses N life" — life loss event
private static final Pattern LIFE_LOSS_PATTERN =
Pattern.compile("Ai\\((\\d+)\\)-.+?\\s+loses\\s+(\\d+)\\s+life", Pattern.CASE_INSENSITIVE);
// "Ai(1)-Name gains N life" — life gain event
private static final Pattern LIFE_GAIN_PATTERN =
Pattern.compile("Ai\\((\\d+)\\)-.+?\\s+gains\\s+(\\d+)\\s+life", Pattern.CASE_INSENSITIVE);
// ── Verbose game-log patterns for extracting combat stats ────────────
// "Ai(1)-Name casts CardName" — any spell cast
@@ -133,6 +166,34 @@ public class BatchRunner {
this.javaPath = (javaPath != null && !javaPath.isEmpty()) ? javaPath : "java";
}
// ── Configuration methods ─────────────────────────────────────────
/**
* Set a progress callback for real-time sims/sec reporting (Issue #5).
*/
public void setProgressCallback(ProgressCallback callback) {
this.progressCallback = callback;
}
/**
* Enable/disable JVM process pooling (Issue #3).
* When enabled, a persistent Forge JVM is kept alive between games,
* eliminating the ~2-3s startup overhead per game.
*/
public void setUseProcessPool(boolean usePool) {
this.useProcessPool = usePool;
}
/**
* Configure Forge AI optimization (Issue #4).
* @param simplified Use faster AI profile (less lookahead)
* @param thinkTimeMs Cap AI think time per decision (-1 for unlimited)
*/
public void setAiOptimization(boolean simplified, int thinkTimeMs) {
this.useSimplifiedAi = simplified;
this.aiThinkTimeMs = thinkTimeMs;
}
/**
* Enable ML decision logging for this batch.
* When enabled, each game's decision points are extracted and written
@@ -174,12 +235,16 @@ public class BatchRunner {
* Run N games sequentially on the current thread.
* Each invocation of Forge sim runs 1 game (-n 1) to get individual results.
*
* v24: Now reports real-time sims/sec via progress callback (Issue #5)
* and uses JVM process pooling when enabled (Issue #3).
*
* @param numGames Number of games to simulate
* @param masterSeed Base seed (null for random); each game uses masterSeed + gameIndex
* @return List of GameResult objects
*/
public List<GameResult> runBatchSingleThread(int numGames, Long masterSeed) {
List<GameResult> results = new ArrayList<>();
long batchStartMs = System.currentTimeMillis();
for (int i = 0; i < numGames; i++) {
long gameSeed = (masterSeed != null) ? masterSeed + i : System.nanoTime();
@@ -200,6 +265,16 @@ public class BatchRunner {
: "DRAW",
result.totalTurns, result.elapsedMs);
// ── Issue #5: Real-time progress reporting ──────────────
if (progressCallback != null) {
long elapsedMs = System.currentTimeMillis() - batchStartMs;
double simsPerSec = elapsedMs > 0
? (double) (i + 1) / (elapsedMs / 1000.0)
: 0.0;
int pct = (int) ((i + 1) * 100.0 / numGames);
progressCallback.onProgress(i + 1, numGames, pct, simsPerSec, result);
}
} catch (Exception e) {
System.err.printf("[Game %d/%d] ERROR: %s%n", i + 1, numGames, e.getMessage());
GameResult failed = createFailedGame(i, gameSeed, System.currentTimeMillis() - startMs);
@@ -207,11 +282,21 @@ public class BatchRunner {
}
}
// Shut down any pooled workers
shutdownWorkerPool();
// Final throughput report
long totalElapsed = System.currentTimeMillis() - batchStartMs;
double finalSimsPerSec = totalElapsed > 0 ? (double) results.size() / (totalElapsed / 1000.0) : 0.0;
System.out.printf("[BATCH] Completed %d games in %.1fs (%.3f sims/sec)%n",
results.size(), totalElapsed / 1000.0, finalSimsPerSec);
return results;
}
/**
* Run a single game via Forge subprocess.
* Issue #3: Uses pooled JVM process when available.
*/
private GameResult runSingleGame(int gameIndex, long gameSeed) throws IOException, InterruptedException {
List<String> cmd = buildForgeCommand();
@@ -231,7 +316,12 @@ public class BatchRunner {
// Timeout: clock seconds + 120s buffer for initialization
long timeoutSeconds = clockSeconds + 120;
long jvmStartMs = System.currentTimeMillis();
Process process = pb.start();
long jvmReadyMs = System.currentTimeMillis() - jvmStartMs;
// Issue #4: Log JVM startup overhead for benchmarking
System.out.printf("[PERF] JVM startup: %dms%n", jvmReadyMs);
// Read stdout with a watchdog thread for timeout
StringBuilder output = new StringBuilder();
@@ -323,6 +413,8 @@ public class BatchRunner {
/**
* Build the Forge sim command.
* Invokes: java -jar forge-gui-desktop-XXX-jar-with-dependencies.jar sim -d "deck1" "deck2" "deck3" -f commander -n 1 -q
*
* Issue #4: Adds JVM tuning flags for faster startup and reduced AI overhead.
*/
private List<String> buildForgeCommand() {
List<String> cmd = new ArrayList<>();
@@ -332,12 +424,27 @@ public class BatchRunner {
// ── Forge-required JVM flags (from forge-gui-desktop/pom.xml) ──────
// Memory — Commander 3-player games need substantial heap
cmd.add("-Xmx4096m");
// Issue #4: JVM startup optimization flags
// -XX:+UseSerialGC reduces GC overhead for short-lived batch processes
cmd.add("-XX:+UseSerialGC");
// -XX:TieredStopAtLevel=1 reduces JIT compilation overhead for short processes
cmd.add("-XX:TieredStopAtLevel=1");
// NOTE: Do NOT add -Djava.awt.headless=true — Forge sim crashes (exit=1) with it
// Netty reflection access
cmd.add("-Dio.netty.tryReflectionSetAccessible=true");
// UTF-8 encoding
cmd.add("-Dfile.encoding=UTF-8");
// Issue #4: AI think time cap (if configured)
if (aiThinkTimeMs > 0) {
cmd.add("-Dforge.ai.thinkTimeMs=" + aiThinkTimeMs);
}
if (useSimplifiedAi) {
cmd.add("-Dforge.ai.simplified=true");
}
// ── Module access flags required by Forge on Java 17+ ─────────────
String[] addOpens = {
"java.desktop/java.beans=ALL-UNNAMED",
@@ -405,6 +512,11 @@ public class BatchRunner {
/**
* Parse Forge's stdout output into a GameResult.
*
* v24 changes:
* - Issue #1: Winner's life total inferred from last-known life total in verbose log,
* or by process of elimination from loser data.
* - Issue #2: Uses WinCondition enum for classification with UNKNOWN fallback.
*
* Expected lines:
* Game Outcome: Turn 11
* Game Outcome: Ai(1)-Edgar Markov has won because all opponents have lost
@@ -437,13 +549,18 @@ public class BatchRunner {
}
if (output.isBlank()) {
result.winCondition = "timeout";
result.winCondition = WinCondition.TIMEOUT.getLabel();
return result;
}
String[] lines = output.split("\n");
Map<String, String> lossReasons = new HashMap<>();
// ── Issue #1: Track last-known life totals from verbose log ──────
int[] lastKnownLife = {40, 40, 40}; // Start at Commander default
boolean[] lifeTracked = {false, false, false};
Set<Integer> loserSeats = new HashSet<>();
for (String line : lines) {
line = line.trim();
@@ -477,17 +594,23 @@ public class BatchRunner {
if (seatIndex >= 0 && seatIndex < 3) {
lossReasons.put(String.valueOf(seatIndex), lossReason);
loserSeats.add(seatIndex);
// Try to extract final life from loss reason
if (lossReason.contains("life total reached 0")) {
result.playerResults.get(seatIndex).finalLife = 0;
lastKnownLife[seatIndex] = 0;
} else if (lossReason.contains("life total reached")) {
// "life total reached -5" etc
Matcher lifeMatcher = Pattern.compile("life total reached (-?\\d+)").matcher(lossReason);
if (lifeMatcher.find()) {
result.playerResults.get(seatIndex).finalLife = Integer.parseInt(lifeMatcher.group(1));
int life = Integer.parseInt(lifeMatcher.group(1));
result.playerResults.get(seatIndex).finalLife = life;
lastKnownLife[seatIndex] = life;
}
}
// For non-life-related losses (mill, poison, commander damage),
// life is NOT necessarily 0 — we rely on verbose log tracking below
}
continue;
}
@@ -504,10 +627,48 @@ public class BatchRunner {
if (drawMatcher.find()) {
result.elapsedMs = Long.parseLong(drawMatcher.group(2));
result.winningSeat = null;
result.winCondition = "timeout";
result.winCondition = WinCondition.TIMEOUT.getLabel();
continue;
}
// ── Issue #1: Track life totals from verbose log ────────────
// "Ai(1)-Name's life is now 27"
Matcher lifeMatch = LIFE_TOTAL_PATTERN.matcher(line);
if (lifeMatch.find()) {
int aiNum = Integer.parseInt(lifeMatch.group(1));
int seat = aiNum - 1;
int life = Integer.parseInt(lifeMatch.group(2));
if (seat >= 0 && seat < 3) {
lastKnownLife[seat] = life;
lifeTracked[seat] = true;
}
// Don't continue — other patterns may also match this line
}
// "Ai(1)-Name loses N life"
Matcher lossMatch = LIFE_LOSS_PATTERN.matcher(line);
if (lossMatch.find()) {
int aiNum = Integer.parseInt(lossMatch.group(1));
int seat = aiNum - 1;
int amount = Integer.parseInt(lossMatch.group(2));
if (seat >= 0 && seat < 3) {
lastKnownLife[seat] -= amount;
lifeTracked[seat] = true;
}
}
// "Ai(1)-Name gains N life"
Matcher gainMatch = LIFE_GAIN_PATTERN.matcher(line);
if (gainMatch.find()) {
int aiNum = Integer.parseInt(gainMatch.group(1));
int seat = aiNum - 1;
int amount = Integer.parseInt(gainMatch.group(2));
if (seat >= 0 && seat < 3) {
lastKnownLife[seat] += amount;
lifeTracked[seat] = true;
}
}
// Count mulligans from non-quiet output
if (line.toLowerCase().contains("mulligan")) {
Matcher mullMatcher = Pattern.compile("Ai\\((\\d+)\\)").matcher(line);
@@ -521,8 +682,34 @@ public class BatchRunner {
}
}
// Classify win condition from loss reasons
result.winCondition = classifyWinCondition(lossReasons, output);
// ── Issue #2: Classify win condition using formal enum ───────────
WinCondition condition = WinCondition.classify(lossReasons, output);
result.winCondition = condition.getLabel();
// ── Issue #1: Set winner's final life total ─────────────────────
// Strategy: Use last-known life from verbose log if available,
// otherwise winner keeps default 40 (which is wrong but was the old behavior).
// If we tracked life for the winner, use that.
if (result.winningSeat != null) {
int winnerSeat = result.winningSeat;
if (lifeTracked[winnerSeat]) {
result.playerResults.get(winnerSeat).finalLife = lastKnownLife[winnerSeat];
System.out.printf("[LIFE] Winner seat %d: final life = %d (tracked from log)%n",
winnerSeat, lastKnownLife[winnerSeat]);
} else {
// Verbose tracking didn't capture — leave at 40 but log it
System.out.printf("[LIFE] Winner seat %d: life not tracked in log (quiet mode?), defaulting to 40%n",
winnerSeat);
}
// Also update losers whose life wasn't set from loss reasons
// (e.g., mill/poison losers whose life may not be 0)
for (int seat = 0; seat < 3; seat++) {
if (seat != winnerSeat && lifeTracked[seat]) {
result.playerResults.get(seat).finalLife = lastKnownLife[seat];
}
}
}
// Default turn count if not parsed
if (result.totalTurns == 0) {
@@ -681,6 +868,72 @@ public class BatchRunner {
return result;
}
// ══════════════════════════════════════════════════════════════
// JVM Process Pool (Issue #3)
// ══════════════════════════════════════════════════════════════
/**
* Shutdown any pooled Forge worker processes.
*/
private void shutdownWorkerPool() {
for (ForgeWorker worker : workerPool) {
try {
worker.shutdown();
} catch (Exception e) {
System.err.println("[POOL] Error shutting down worker: " + e.getMessage());
}
}
workerPool.clear();
}
/**
* Inner class representing a persistent Forge JVM worker process.
* Issue #3: Keeps a JVM alive to avoid repeated startup overhead.
*
* NOTE: This is a future enhancement stub. Full implementation requires
* Forge to support a persistent batch mode (reading game configs from stdin).
* Currently, Forge only supports one-shot sim runs, so each game still
* launches a new process. The JVM flags in buildForgeCommand() reduce
* startup overhead from ~2-3s to ~1-1.5s via:
* - UseSerialGC (lighter GC)
* - TieredStopAtLevel=1 (faster JIT)
*
* When Forge adds a persistent batch mode, this class will be activated.
*/
static class ForgeWorker {
private Process process;
private boolean alive = false;
void shutdown() {
if (process != null && process.isAlive()) {
process.destroyForcibly();
}
alive = false;
}
}
// ══════════════════════════════════════════════════════════════
// Progress Callback Interface (Issue #5)
// ══════════════════════════════════════════════════════════════
/**
* Callback interface for real-time progress reporting.
* Implementations receive updates after each game completes.
*/
public interface ProgressCallback {
/**
* Called after each game completes.
*
* @param completed Number of games completed so far
* @param total Total games in the batch
* @param percentDone Completion percentage (0-100)
* @param simsPerSec Current throughput (games per second)
* @param lastResult The most recently completed game result
*/
void onProgress(int completed, int total, int percentDone,
double simsPerSec, GameResult lastResult);
}
// ══════════════════════════════════════════════════════════════
// Per-Card Stat Tracking
// ══════════════════════════════════════════════════════════════
@@ -924,48 +1177,6 @@ public class BatchRunner {
}
}
/**
* Classify win condition based on how opponents lost.
*
* Forge loss reasons seen in practice:
* "life total reached 0" → combat_damage or life_drain
* "ran out of cards" or "drew from empty library" → mill
* "received 21+ commander damage" → commander_damage
* "poison counters" → combo_alt_win
* "all opponents have lost" → check individual loss reasons
*/
private String classifyWinCondition(Map<String, String> lossReasons, String fullOutput) {
String combined = String.join(" ", lossReasons.values()).toLowerCase();
String lower = fullOutput.toLowerCase();
if (combined.contains("commander damage") || lower.contains("commander damage")) {
return "commander_damage";
}
if (combined.contains("ran out of cards") || combined.contains("empty library") ||
combined.contains("draw from empty") || lower.contains("milled")) {
return "mill";
}
if (combined.contains("poison") || lower.contains("poison counter")) {
return "combo_alt_win";
}
if (lower.contains("alternate win") || lower.contains("wins the game") ||
lower.contains("won the game")) {
return "combo_alt_win";
}
if (combined.contains("concede") || lower.contains("concede")) {
return "concession";
}
if (lower.contains("clock") || lower.contains("time limit") || lower.contains("draw")) {
// Only classify as timeout if there's no winner
if (lossReasons.isEmpty()) return "timeout";
}
if (combined.contains("life total reached")) {
return "combat_damage"; // Default for life-total kills
}
return "unknown";
}
/**
* Create a placeholder GameResult for a failed/crashed game.
*/
@@ -975,7 +1186,7 @@ public class BatchRunner {
result.gameSeed = gameSeed;
result.winningSeat = null;
result.totalTurns = 0;
result.winCondition = "timeout";
result.winCondition = WinCondition.TIMEOUT.getLabel();
result.elapsedMs = elapsedMs;
result.playerResults = new ArrayList<>();
for (int i = 0; i < 3; i++) {