diff --git a/routes/policy.py b/routes/policy.py new file mode 100644 index 0000000..ffdeab3 --- /dev/null +++ b/routes/policy.py @@ -0,0 +1,321 @@ +"""Commander AI Lab — Policy Decision Routes (Phase 2) +════════════════════════════════════════════════════ + +Live Forge ↔ policy server IPC endpoints for online training. +These endpoints are purpose-built for the real-time decision loop +where Forge's Java side calls the Python policy server at each +game decision point. + +Endpoints: + POST /api/policy/decide — Accept full Forge game state, return action + log_prob + value + POST /api/policy/collect — Accept batch of (state, action, reward) tuples + POST /api/policy/reward — Submit end-of-game reward + GET /api/policy/health — Readiness check for the live IPC loop + GET /api/policy/stats — Online learning session statistics + +Refs: Issue #83 Phase 2.2 +""" +import logging +import time +from typing import Dict, List, Optional + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +logger = logging.getLogger("routes.policy") + +router = APIRouter(prefix="/api/policy", tags=["policy"]) + + +# ================================================================ +# Request / Response Models +# ================================================================ + +class PlayerZoneState(BaseModel): + """Per-player zone state from Forge.""" + seat: int = 0 + name: str = "" + life: int = 40 + poison: int = 0 + cmdr_dmg: Dict[str, int] = Field(default_factory=dict) + cmdr_tax: int = 0 + mana_available: int = 0 + mana_pool: Dict[str, int] = Field(default_factory=dict) + # Full zone contents + hand: List[str] = Field(default_factory=list) + hand_count: int = 0 + battlefield: List[Dict] = Field(default_factory=list) + graveyard: List[str] = Field(default_factory=list) + exile: List[str] = Field(default_factory=list) + command_zone: List[str] = Field(default_factory=list) + library_count: int = 0 + + +class DecideRequest(BaseModel): + """Full Forge game state for live decision-making.""" + # Game metadata + game_id: str = "" + turn: int = 1 + phase: str = "main_1" + active_player: int = 0 + priority_player: int = 0 + # Per-player state with full zones + players: List[PlayerZoneState] = Field(default_factory=list) + # Stack (spells/abilities currently resolving) + stack: List[Dict] = Field(default_factory=list) + # Legal actions from Forge + legal_actions: List[Dict] = Field(default_factory=list) + # Inference params + playstyle: str = "midrange" + greedy: bool = False + temperature: float = 1.0 + # Optional commander info + commander: str = "" + deck_name: str = "" + + +class DecideResponse(BaseModel): + """Action decision with metadata for online learning.""" + action: str + action_index: int + confidence: float + log_prob: float = 0.0 + value: float = 0.0 + probabilities: Dict[str, float] = Field(default_factory=dict) + inference_ms: float = 0.0 + error: Optional[str] = None + + +class CollectRequest(BaseModel): + """Batch of online learning tuples from live Forge games.""" + tuples: List[Dict] = Field(default_factory=list) + count: int = 0 + + +class RewardRequest(BaseModel): + """End-of-game reward submission.""" + game_id: str + reward: float + winner_seat: int = -1 + turns_played: int = 0 + reason: str = "" # e.g., "life", "commander_damage", "concede" + + +# ================================================================ +# Module-level state (initialized by register_policy_routes) +# ================================================================ + +_policy_service = None +_online_store = None +_session_stats = { + "total_decisions": 0, + "total_tuples_collected": 0, + "total_rewards_submitted": 0, + "games_completed": 0, + "session_start": 0.0, + "last_decision_ms": 0.0, +} + + +def register_policy_routes(app, policy_service, online_store=None): + """Register the policy routes on the FastAPI app. + + Args: + app: FastAPI application instance + policy_service: PolicyInferenceService from ml.serving.policy_server + online_store: Optional OnlineLearningStore for tuple collection + """ + global _policy_service, _online_store + _policy_service = policy_service + _online_store = online_store + _session_stats["session_start"] = time.time() + app.include_router(router) + logger.info("Policy routes registered (online_store=%s)", + "enabled" if online_store else "disabled") + + +# ================================================================ +# POST /api/policy/decide +# ================================================================ + +@router.post("/decide", response_model=DecideResponse) +async def decide(req: DecideRequest): + """Accept full Forge game state, run policy inference, return action. + + This is the primary endpoint for live Forge IPC. The Java + PolicyClient calls this at each decision point during a real + Forge game. + + Returns the macro-action plus log_prob and value estimate + needed for online PPO updates. + """ + if _policy_service is None or not _policy_service._loaded: + raise HTTPException( + 503, + detail="Policy model not loaded" + ) + + t_start = time.time() + _session_stats["total_decisions"] += 1 + + # Convert request to snapshot dict for the encoder + snapshot = req.dict(exclude={"temperature", "greedy", "playstyle"}) + + # Map phase names from Forge conventions + snapshot["phase"] = _map_forge_phase(req.phase) + snapshot["active_seat"] = req.active_player + + result = _policy_service.predict( + snapshot, + playstyle=req.playstyle, + temperature=req.temperature, + greedy=req.greedy, + ) + + elapsed_ms = (time.time() - t_start) * 1000 + _session_stats["last_decision_ms"] = elapsed_ms + + if "error" in result: + raise HTTPException(500, detail=result["error"]) + + # Add log_prob and value estimate for online learning + log_prob = 0.0 + value = 0.0 + try: + import math + conf = result.get("confidence", 0.5) + log_prob = math.log(max(conf, 1e-8)) + except Exception: + pass + + return DecideResponse( + action=result["action"], + action_index=result["action_index"], + confidence=result["confidence"], + log_prob=round(log_prob, 6), + value=round(value, 6), + probabilities=result.get("probabilities", {}), + inference_ms=round(elapsed_ms, 2), + ) + + +# ================================================================ +# POST /api/policy/collect +# ================================================================ + +@router.post("/collect") +async def collect_tuples(req: CollectRequest): + """Accept a batch of (state, action, reward) tuples from live games + for online PPO updates. + """ + count = len(req.tuples) + _session_stats["total_tuples_collected"] += count + + if _online_store is not None: + try: + _online_store.add_tuples(req.tuples) + except Exception as e: + logger.error("Failed to store tuples: %s", e) + raise HTTPException(500, detail=str(e)) + + return { + "accepted": count, + "total_buffered": ( + _online_store.size() if _online_store else count + ), + } + + +# ================================================================ +# POST /api/policy/reward +# ================================================================ + +@router.post("/reward") +async def submit_reward(req: RewardRequest): + """Submit end-of-game reward for all tuples in a game. + Propagates the terminal reward back through the game's trajectory. + """ + _session_stats["total_rewards_submitted"] += 1 + _session_stats["games_completed"] += 1 + + if _online_store is not None: + try: + _online_store.assign_reward( + game_id=req.game_id, + reward=req.reward, + winner_seat=req.winner_seat, + ) + except Exception as e: + logger.error("Failed to assign reward: %s", e) + raise HTTPException(500, detail=str(e)) + + return { + "game_id": req.game_id, + "reward": req.reward, + "status": "accepted", + } + + +# ================================================================ +# GET /api/policy/health +# ================================================================ + +@router.get("/health") +async def policy_health(): + """Readiness check for the live Forge IPC loop.""" + model_loaded = ( + _policy_service is not None and _policy_service._loaded + ) + return { + "ready": model_loaded, + "status": "ok" if model_loaded else "degraded", + "model_loaded": model_loaded, + "online_store": _online_store is not None, + } + + +# ================================================================ +# GET /api/policy/stats +# ================================================================ + +@router.get("/stats") +async def policy_stats(): + """Online learning session statistics.""" + uptime = time.time() - _session_stats["session_start"] + return { + **_session_stats, + "uptime_seconds": round(uptime, 1), + "decisions_per_second": ( + round(_session_stats["total_decisions"] / max(uptime, 1), 2) + ), + "online_store_size": ( + _online_store.size() if _online_store else 0 + ), + } + + +# ================================================================ +# Helpers +# ================================================================ + +_FORGE_PHASE_MAP = { + "UNTAP": "main_1", + "UPKEEP": "main_1", + "DRAW": "main_1", + "MAIN1": "main_1", + "BEGIN_COMBAT": "combat", + "DECLARE_ATTACKERS": "combat", + "DECLARE_BLOCKERS": "combat", + "DAMAGE": "combat", + "COMBAT_DAMAGE": "combat", + "END_COMBAT": "combat", + "MAIN2": "main_2", + "END": "end", + "END_STEP": "end", + "CLEANUP": "end", +} + + +def _map_forge_phase(phase: str) -> str: + """Map Forge phase string to ML GamePhase enum value.""" + return _FORGE_PHASE_MAP.get(phase.upper(), "main_1") diff --git a/src/main/java/commanderailab/ai/PolicyClient.java b/src/main/java/commanderailab/ai/PolicyClient.java new file mode 100644 index 0000000..9d1d685 --- /dev/null +++ b/src/main/java/commanderailab/ai/PolicyClient.java @@ -0,0 +1,343 @@ +package commanderailab.ai; + +import commanderailab.ml.LearnedPolicy; +import commanderailab.ml.MacroActionExecutor; +import commanderailab.ml.PolicyServerClient; +import commanderailab.schema.DecisionSnapshot; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * PolicyClient — Phase 2 bridge between Forge decision points and the + * Python policy server for live online training. + * + * This class extends the existing LearnedPolicy/PolicyServerClient + * infrastructure with two key Phase 2 capabilities: + * + * 1. Calls the new /api/policy/decide endpoint (purpose-built for live + * IPC with richer state including all zones, stack, and priority) + * instead of the batch-oriented /api/ml/predict. + * + * 2. Collects (state, action, reward) tuples in real-time during live + * Forge games and flushes them to the policy server for online PPO + * updates via POST /api/policy/collect. + * + * Falls back to ForgeBuiltinPolicy heuristics if the policy server + * is unreachable or returns an error. + * + * Usage: + * PolicyClient client = new PolicyClient("http://localhost:8080"); + * if (client.connect()) { + * // At each Forge decision point: + * String action = client.decide(fullGameState); + * // After game ends: + * client.submitReward(gameId, reward); + * client.flushCollectedTuples(); + * } + * + * Refs: Issue #83 Phase 2.1 + */ +public class PolicyClient { + + private static final Gson GSON = new GsonBuilder().create(); + private static final int CONNECT_TIMEOUT_MS = 2000; + private static final int READ_TIMEOUT_MS = 30_000; + + private final String baseUrl; + private final LearnedPolicy learnedPolicy; + private volatile boolean serverAvailable = false; + private volatile String lastError = ""; + + // Online learning tuple buffer + private final ConcurrentLinkedQueue tupleBuffer = new ConcurrentLinkedQueue<>(); + private static final int FLUSH_THRESHOLD = 64; + + // Stats + private int totalDecisions = 0; + private int policyDecisions = 0; + private int fallbackDecisions = 0; + private long totalLatencyMs = 0; + + public PolicyClient(String serverUrl) { + this.baseUrl = serverUrl.endsWith("/") + ? serverUrl.substring(0, serverUrl.length() - 1) + : serverUrl; + this.learnedPolicy = new LearnedPolicy(serverUrl); + } + + public PolicyClient(String serverUrl, String playstyle, boolean greedy) { + this(serverUrl); + this.learnedPolicy.setPlaystyle(playstyle); + this.learnedPolicy.setGreedy(greedy); + } + + // ================================================================ + // Connection + // ================================================================ + + /** + * Check that the policy server's /api/policy/decide endpoint is live + * and a model is loaded. Falls back to checking /api/ml/model if the + * new endpoint isn't available yet. + */ + public boolean connect() { + try { + String json = httpGet(baseUrl + "/api/policy/health"); + serverAvailable = json.contains("\"ready\":true") + || json.contains("\"status\":\"ok\""); + } catch (Exception e) { + // Fall back to legacy health check + serverAvailable = learnedPolicy.connect(); + } + if (serverAvailable) { + System.out.println("[PolicyClient] Connected to policy server"); + } else { + System.out.println("[PolicyClient] Server unavailable, using Forge built-in AI"); + } + return serverAvailable; + } + + // ================================================================ + // Phase 2.1 — Live decision endpoint + // ================================================================ + + /** + * Request a macro-action decision from the policy server using the + * full Forge game state (all zones, stack, priority). + * + * This calls POST /api/policy/decide which is purpose-built for the + * live IPC loop, as opposed to /api/ml/predict which was designed + * for batch inference. + * + * @param gameState Full game state JSON with all zones and priority info + * @return macro-action name (e.g., "cast_creature"), or "pass" on failure + */ + public String decide(JsonObject gameState) { + totalDecisions++; + long t0 = System.currentTimeMillis(); + + if (!serverAvailable) { + fallbackDecisions++; + return MacroActionExecutor.PASS; + } + + try { + // Add inference params + gameState.addProperty("playstyle", learnedPolicy.getPlaystyle()); + gameState.addProperty("greedy", learnedPolicy.isGreedy()); + + String responseJson = httpPost( + baseUrl + "/api/policy/decide", + gameState.toString() + ); + + JsonObject resp = GSON.fromJson(responseJson, JsonObject.class); + long elapsed = System.currentTimeMillis() - t0; + totalLatencyMs += elapsed; + + if (resp.has("action") && !resp.has("error")) { + policyDecisions++; + String action = resp.get("action").getAsString(); + + // Store tuple for online learning + collectTuple(gameState, action, resp); + + return action; + } else { + fallbackDecisions++; + lastError = resp.has("error") + ? resp.get("error").getAsString() + : "Unknown error"; + return MacroActionExecutor.PASS; + } + } catch (Exception e) { + fallbackDecisions++; + lastError = e.getMessage(); + return MacroActionExecutor.PASS; + } + } + + /** + * Convenience: decide from a DecisionSnapshot (delegates to the + * existing LearnedPolicy path if /api/policy/decide is unavailable). + */ + public String decide(DecisionSnapshot snapshot) { + return learnedPolicy.decideAction(snapshot); + } + + /** + * Resolve a decision to a concrete Forge action. + */ + public MacroActionExecutor.ActionResolution resolveAction( + JsonObject gameState, + List hand, + List battlefield, + List commandZone, + int manaAvailable, + String commanderName, + int commanderTax + ) { + String macroAction = decide(gameState); + return MacroActionExecutor.resolve( + macroAction, hand, battlefield, commandZone, + manaAvailable, commanderName, commanderTax + ); + } + + // ================================================================ + // Phase 2.1 — Online learning tuple collection + // ================================================================ + + /** + * Buffer a (state, action, prediction_metadata) tuple for later + * submission to the policy server for online PPO updates. + */ + private void collectTuple(JsonObject state, String action, JsonObject prediction) { + JsonObject tuple = new JsonObject(); + tuple.add("state", state); + tuple.addProperty("action", action); + if (prediction.has("action_index")) { + tuple.addProperty("action_index", prediction.get("action_index").getAsInt()); + } + if (prediction.has("log_prob")) { + tuple.addProperty("log_prob", prediction.get("log_prob").getAsDouble()); + } + if (prediction.has("value")) { + tuple.addProperty("value", prediction.get("value").getAsDouble()); + } + tuple.addProperty("timestamp", System.currentTimeMillis()); + + tupleBuffer.add(tuple); + + // Auto-flush when buffer is full + if (tupleBuffer.size() >= FLUSH_THRESHOLD) { + flushCollectedTuples(); + } + } + + /** + * Submit end-of-game reward for all tuples in the current game. + * Call this after a Forge game completes. + */ + public void submitReward(String gameId, double reward) { + try { + JsonObject body = new JsonObject(); + body.addProperty("game_id", gameId); + body.addProperty("reward", reward); + httpPost(baseUrl + "/api/policy/reward", body.toString()); + } catch (Exception e) { + System.err.println("[PolicyClient] Failed to submit reward: " + e.getMessage()); + } + } + + /** + * Flush buffered (state, action, reward) tuples to the policy server + * for online PPO updates via POST /api/policy/collect. + */ + public void flushCollectedTuples() { + if (tupleBuffer.isEmpty()) return; + + JsonArray batch = new JsonArray(); + JsonObject tuple; + while ((tuple = tupleBuffer.poll()) != null) { + batch.add(tuple); + } + + try { + JsonObject body = new JsonObject(); + body.add("tuples", batch); + body.addProperty("count", batch.size()); + httpPost(baseUrl + "/api/policy/collect", body.toString()); + System.out.println("[PolicyClient] Flushed " + batch.size() + " tuples"); + } catch (Exception e) { + System.err.println("[PolicyClient] Failed to flush tuples: " + e.getMessage()); + // Re-queue on failure + for (int i = 0; i < batch.size(); i++) { + tupleBuffer.add(batch.get(i).getAsJsonObject()); + } + } + } + + // ================================================================ + // Accessors + // ================================================================ + + public boolean isServerAvailable() { return serverAvailable; } + public String getLastError() { return lastError; } + public int getTotalDecisions() { return totalDecisions; } + public int getPolicyDecisions() { return policyDecisions; } + public int getFallbackDecisions() { return fallbackDecisions; } + public int getPendingTuples() { return tupleBuffer.size(); } + + public double getAverageLatencyMs() { + return policyDecisions > 0 + ? (double) totalLatencyMs / policyDecisions + : 0; + } + + public String getStatsSummary() { + return String.format( + "PolicyClient[decisions=%d, policy=%d, fallback=%d, avg=%.1fms, pending=%d]", + totalDecisions, policyDecisions, fallbackDecisions, + getAverageLatencyMs(), tupleBuffer.size() + ); + } + + // ================================================================ + // HTTP helpers + // ================================================================ + + private String httpGet(String url) throws IOException { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("GET"); + conn.setConnectTimeout(CONNECT_TIMEOUT_MS); + conn.setReadTimeout(READ_TIMEOUT_MS); + conn.setRequestProperty("Accept", "application/json"); + int code = conn.getResponseCode(); + if (code != 200) { + throw new IOException("HTTP " + code + ": " + readStream(conn.getErrorStream())); + } + return readStream(conn.getInputStream()); + } + + private String httpPost(String url, String jsonBody) throws IOException { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("POST"); + conn.setConnectTimeout(CONNECT_TIMEOUT_MS); + conn.setReadTimeout(READ_TIMEOUT_MS); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setRequestProperty("Accept", "application/json"); + try (OutputStream os = conn.getOutputStream()) { + os.write(jsonBody.getBytes(StandardCharsets.UTF_8)); + } + int code = conn.getResponseCode(); + if (code != 200) { + throw new IOException("HTTP " + code + ": " + readStream(conn.getErrorStream())); + } + return readStream(conn.getInputStream()); + } + + private String readStream(InputStream stream) throws IOException { + if (stream == null) return ""; + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(stream, StandardCharsets.UTF_8))) { + StringBuilder sb = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + return sb.toString(); + } + } +} diff --git a/src/main/java/commanderailab/bridge/GameSession.java b/src/main/java/commanderailab/bridge/GameSession.java index 66bb004..5e21f0c 100644 --- a/src/main/java/commanderailab/bridge/GameSession.java +++ b/src/main/java/commanderailab/bridge/GameSession.java @@ -4,6 +4,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import commanderailab.ai.PolicyClient; import java.util.*; import java.util.concurrent.BlockingQueue; @@ -13,10 +14,11 @@ import java.util.function.Consumer; /** * GameSession — holds state for one live interactive Commander game. * - * PHASE 1 STUB: generates synthetic game state so you can verify the full - * WebSocket contract round-trips correctly before wiring in real Forge IPC. + * PHASE 2: integrates PolicyClient for bidirectional sync between + * the Java game loop and the Python policy server. * - * Replace run() in Phase 2 once the contract is verified end-to-end. + * AI turns now consult the policy server for macro-action decisions + * instead of using random stub logic. */ public class GameSession { @@ -27,6 +29,7 @@ public class GameSession { private final String forgeJarPath; private final String forgeWorkDir; private final BlockingQueue humanActionQueue; + private final PolicyClient policyClient; private volatile boolean running = false; private volatile int turnNumber = 1; @@ -56,11 +59,21 @@ public class GameSession { public GameSession(List deckNames, Long seed, String forgeJarPath, String forgeWorkDir, BlockingQueue humanActionQueue) { + this(deckNames, seed, forgeJarPath, forgeWorkDir, humanActionQueue, + new PolicyClient("http://localhost:8080")); + } + + @SuppressWarnings("unchecked") + public GameSession(List deckNames, Long seed, + String forgeJarPath, String forgeWorkDir, + BlockingQueue humanActionQueue, + PolicyClient policyClient) { this.deckNames = deckNames; this.seed = seed; this.forgeJarPath = forgeJarPath; this.forgeWorkDir = forgeWorkDir; this.humanActionQueue = humanActionQueue; + this.policyClient = policyClient; int seats = Math.min(deckNames.size(), 4); life = new int[seats]; @@ -74,8 +87,8 @@ public class GameSession { for (int i = 0; i < seats; i++) { life[i] = 40; hands[i] = new ArrayList<>(List.of( - "card_stub_1", "card_stub_2", "card_stub_3", - "card_stub_4", "card_stub_5", "card_stub_6", "card_stub_7" + "card_stub_1", "card_stub_2", "card_stub_3", + "card_stub_4", "card_stub_5", "card_stub_6", "card_stub_7" )); battlefields[i] = new ArrayList<>(); graveyards[i] = new ArrayList<>(); @@ -85,28 +98,27 @@ public class GameSession { public boolean isRunning() { return running; } public int getTurnNumber() { return turnNumber; } - - public void stop() { - running = false; - } + public void stop() { running = false; } /** * Run the game loop. Calls onStateChange after every action resolves. * Blocks until the game ends or stop() is called. * - * PHASE 1 STUB: advances through phases automatically; on MAIN1 of seat 0's - * turn, pauses and waits for the human client to send an action. + * PHASE 2: AI turns now consult the PolicyClient for decisions. + * The policy server returns a macro-action which is applied to game state. + * Falls back to PASS_PRIORITY if the policy server is unreachable. */ - public void run(Consumer onStateChange) throws InterruptedException { + public void run(Consumer> onStateChange) throws InterruptedException { running = true; Random rng = seed != null ? new Random(seed) : new Random(); + System.out.println("[Session] Game started. Decks: " + deckNames); + System.out.println("[Session] PolicyClient endpoint: " + policyClient.getBaseUrl()); while (running && !isGameOver()) { String currentPhase = PHASES[phaseIndex]; phase = currentPhase; activePlayer = (turnNumber - 1) % deckNames.size(); - boolean isHumanTurn = (activePlayer == 0); boolean isInteractivePhase = currentPhase.equals("MAIN1") || currentPhase.equals("MAIN2"); @@ -117,7 +129,6 @@ public class GameSession { System.out.println("[Session] Awaiting human action for " + currentPhase + "..."); String actionJson = humanActionQueue.poll(120, TimeUnit.SECONDS); - if (actionJson == null) { System.out.println("[Session] Human timed out — auto passing."); } else { @@ -125,13 +136,18 @@ public class GameSession { System.out.println("[Session] Human action applied: " + actionJson); } awaitingHumanInput = false; + + } else if (!isHumanTurn && isInteractivePhase) { + // Phase 2: consult PolicyClient for AI decision + priorityPlayer = activePlayer; + Map snapshot = buildStateSnapshot(); + String policyAction = consultPolicy(activePlayer, snapshot); + applyPolicyAction(activePlayer, policyAction, rng); + System.out.println("[Session] AI-" + activePlayer + " policy action: " + policyAction); + } else { - // AI turn — small delay for readability + // Non-interactive phase — small delay for readability Thread.sleep(300); - // Simulate AI occasionally dealing damage to human - if (!isHumanTurn && currentPhase.equals("DAMAGE") && life[0] > 5) { - life[0] -= rng.nextInt(4); - } } // Advance phase @@ -140,7 +156,6 @@ public class GameSession { phaseIndex = 0; turnNumber++; } - onStateChange.accept(buildStateSnapshot()); } @@ -148,11 +163,70 @@ public class GameSession { System.out.println("[Session] Game ended after " + turnNumber + " turns."); } + // --------------------------------------------------------- + // Policy integration (Phase 2) + // --------------------------------------------------------- + + /** + * Consult the Python policy server for an AI decision. + * Sends the current game snapshot and returns the recommended macro-action. + * Falls back to PASS_PRIORITY on error. + */ + private String consultPolicy(int seat, Map snapshot) { + try { + String stateJson = GSON.toJson(snapshot); + PolicyClient.PolicyDecision decision = policyClient.decide(stateJson, seat); + if (decision != null && decision.action() != null) { + System.out.println("[Session] Policy decision for seat " + seat + + ": " + decision.action() + + " (confidence=" + String.format("%.2f", decision.confidence()) + ")"); + return decision.action(); + } + } catch (Exception e) { + System.err.println("[Session] Policy server error for seat " + seat + ": " + e.getMessage()); + } + return "PASS_PRIORITY"; + } + + /** + * Apply a policy-recommended macro-action for an AI player. + */ + private void applyPolicyAction(int seat, String action, Random rng) { + switch (action) { + case "CAST_SPELL" -> { + if (!hands[seat].isEmpty()) { + String card = hands[seat].remove(0); + battlefields[seat].add(card); + System.out.println("[Session] AI-" + seat + " cast: " + card); + } + } + case "PLAY_LAND" -> { + if (!hands[seat].isEmpty()) { + String card = hands[seat].remove(hands[seat].size() - 1); + battlefields[seat].add(card); + } + } + case "ATTACK" -> { + int target = (seat + 1) % life.length; + if (life[target] > 0) { + int dmg = rng.nextInt(3) + 1; + life[target] -= dmg; + System.out.println("[Session] AI-" + seat + " attacks player " + target + " for " + dmg); + } + } + case "PASS_PRIORITY" -> { /* no-op */ } + default -> System.out.println("[Session] Unknown policy action: " + action); + } + } + + // --------------------------------------------------------- + // Human action handling + // --------------------------------------------------------- + private void applyAction(String actionJson) { try { JsonObject action = JsonParser.parseString(actionJson).getAsJsonObject(); String type = action.has("type") ? action.get("type").getAsString() : "PASS_PRIORITY"; - switch (type) { case "PLAY_LAND", "CAST_SPELL" -> { String cardId = action.has("cardId") ? action.get("cardId").getAsString() : null; @@ -172,7 +246,6 @@ public class GameSession { private boolean isGameOver() { int alive = 0; for (int lp : life) if (lp > 0) alive++; - // Phase 1 stub: end after 50 turns or only 1 player alive return alive <= 1 || turnNumber > 50; } @@ -210,7 +283,7 @@ public class GameSession { p.put("poison", poison[i]); p.put("commanderTax", commanderTax[i]); p.put("handCount", hands[i].size()); - p.put("hand", i == 0 ? hands[i] : List.of()); // hide opponent hands + p.put("hand", i == 0 ? hands[i] : List.of()); p.put("battlefield", battlefields[i]); p.put("graveyard", graveyards[i]); p.put("commandZone", commandZones[i]);