feat(83): Phase 2 — Live Forge-to-Policy-Server IPC (#100)
* feat(83): add PolicyClient for live Forge-to-policy-server IPC (#83) Phase 2.1: Bridge between Forge decision points and Python policy server for online training. Calls /api/policy/decide endpoint, collects (state, action, reward) tuples in real-time, and flushes them to the server for online PPO updates. Falls back to Forge built-in AI if server is unavailable. * feat(83): add /api/policy/* routes for live Forge IPC (#83) Phase 2.2: Policy decision routes for real-time Forge IPC. - POST /api/policy/decide — full game state inference with log_prob/value - POST /api/policy/collect — online learning tuple batch collection - POST /api/policy/reward — end-of-game reward propagation - GET /api/policy/health — readiness check for Java PolicyClient - GET /api/policy/stats — session statistics Includes Forge phase mapping and PlayerZoneState with full zone contents. * feat(83): integrate PolicyClient into GameSession for bidirectional sync (#83)
This commit is contained in:
KoalaTrapLord authored and GitHub committed
1 parent
87404f3702
commit
411fa41063
3 files changed
+760
-23
No files matched your search
@@ -0,0 +1,321 @@
|
||||
"""Commander AI Lab — Policy Decision Routes (Phase 2)
|
||||
════════════════════════════════════════════════════
|
||||
|
||||
Live Forge ↔ policy server IPC endpoints for online training.
|
||||
These endpoints are purpose-built for the real-time decision loop
|
||||
where Forge's Java side calls the Python policy server at each
|
||||
game decision point.
|
||||
|
||||
Endpoints:
|
||||
POST /api/policy/decide — Accept full Forge game state, return action + log_prob + value
|
||||
POST /api/policy/collect — Accept batch of (state, action, reward) tuples
|
||||
POST /api/policy/reward — Submit end-of-game reward
|
||||
GET /api/policy/health — Readiness check for the live IPC loop
|
||||
GET /api/policy/stats — Online learning session statistics
|
||||
|
||||
Refs: Issue #83 Phase 2.2
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
logger = logging.getLogger("routes.policy")
|
||||
|
||||
router = APIRouter(prefix="/api/policy", tags=["policy"])
|
||||
|
||||
|
||||
# ================================================================
|
||||
# Request / Response Models
|
||||
# ================================================================
|
||||
|
||||
class PlayerZoneState(BaseModel):
|
||||
"""Per-player zone state from Forge."""
|
||||
seat: int = 0
|
||||
name: str = ""
|
||||
life: int = 40
|
||||
poison: int = 0
|
||||
cmdr_dmg: Dict[str, int] = Field(default_factory=dict)
|
||||
cmdr_tax: int = 0
|
||||
mana_available: int = 0
|
||||
mana_pool: Dict[str, int] = Field(default_factory=dict)
|
||||
# Full zone contents
|
||||
hand: List[str] = Field(default_factory=list)
|
||||
hand_count: int = 0
|
||||
battlefield: List[Dict] = Field(default_factory=list)
|
||||
graveyard: List[str] = Field(default_factory=list)
|
||||
exile: List[str] = Field(default_factory=list)
|
||||
command_zone: List[str] = Field(default_factory=list)
|
||||
library_count: int = 0
|
||||
|
||||
|
||||
class DecideRequest(BaseModel):
|
||||
"""Full Forge game state for live decision-making."""
|
||||
# Game metadata
|
||||
game_id: str = ""
|
||||
turn: int = 1
|
||||
phase: str = "main_1"
|
||||
active_player: int = 0
|
||||
priority_player: int = 0
|
||||
# Per-player state with full zones
|
||||
players: List[PlayerZoneState] = Field(default_factory=list)
|
||||
# Stack (spells/abilities currently resolving)
|
||||
stack: List[Dict] = Field(default_factory=list)
|
||||
# Legal actions from Forge
|
||||
legal_actions: List[Dict] = Field(default_factory=list)
|
||||
# Inference params
|
||||
playstyle: str = "midrange"
|
||||
greedy: bool = False
|
||||
temperature: float = 1.0
|
||||
# Optional commander info
|
||||
commander: str = ""
|
||||
deck_name: str = ""
|
||||
|
||||
|
||||
class DecideResponse(BaseModel):
|
||||
"""Action decision with metadata for online learning."""
|
||||
action: str
|
||||
action_index: int
|
||||
confidence: float
|
||||
log_prob: float = 0.0
|
||||
value: float = 0.0
|
||||
probabilities: Dict[str, float] = Field(default_factory=dict)
|
||||
inference_ms: float = 0.0
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class CollectRequest(BaseModel):
|
||||
"""Batch of online learning tuples from live Forge games."""
|
||||
tuples: List[Dict] = Field(default_factory=list)
|
||||
count: int = 0
|
||||
|
||||
|
||||
class RewardRequest(BaseModel):
|
||||
"""End-of-game reward submission."""
|
||||
game_id: str
|
||||
reward: float
|
||||
winner_seat: int = -1
|
||||
turns_played: int = 0
|
||||
reason: str = "" # e.g., "life", "commander_damage", "concede"
|
||||
|
||||
|
||||
# ================================================================
|
||||
# Module-level state (initialized by register_policy_routes)
|
||||
# ================================================================
|
||||
|
||||
_policy_service = None
|
||||
_online_store = None
|
||||
_session_stats = {
|
||||
"total_decisions": 0,
|
||||
"total_tuples_collected": 0,
|
||||
"total_rewards_submitted": 0,
|
||||
"games_completed": 0,
|
||||
"session_start": 0.0,
|
||||
"last_decision_ms": 0.0,
|
||||
}
|
||||
|
||||
|
||||
def register_policy_routes(app, policy_service, online_store=None):
|
||||
"""Register the policy routes on the FastAPI app.
|
||||
|
||||
Args:
|
||||
app: FastAPI application instance
|
||||
policy_service: PolicyInferenceService from ml.serving.policy_server
|
||||
online_store: Optional OnlineLearningStore for tuple collection
|
||||
"""
|
||||
global _policy_service, _online_store
|
||||
_policy_service = policy_service
|
||||
_online_store = online_store
|
||||
_session_stats["session_start"] = time.time()
|
||||
app.include_router(router)
|
||||
logger.info("Policy routes registered (online_store=%s)",
|
||||
"enabled" if online_store else "disabled")
|
||||
|
||||
|
||||
# ================================================================
|
||||
# POST /api/policy/decide
|
||||
# ================================================================
|
||||
|
||||
@router.post("/decide", response_model=DecideResponse)
|
||||
async def decide(req: DecideRequest):
|
||||
"""Accept full Forge game state, run policy inference, return action.
|
||||
|
||||
This is the primary endpoint for live Forge IPC. The Java
|
||||
PolicyClient calls this at each decision point during a real
|
||||
Forge game.
|
||||
|
||||
Returns the macro-action plus log_prob and value estimate
|
||||
needed for online PPO updates.
|
||||
"""
|
||||
if _policy_service is None or not _policy_service._loaded:
|
||||
raise HTTPException(
|
||||
503,
|
||||
detail="Policy model not loaded"
|
||||
)
|
||||
|
||||
t_start = time.time()
|
||||
_session_stats["total_decisions"] += 1
|
||||
|
||||
# Convert request to snapshot dict for the encoder
|
||||
snapshot = req.dict(exclude={"temperature", "greedy", "playstyle"})
|
||||
|
||||
# Map phase names from Forge conventions
|
||||
snapshot["phase"] = _map_forge_phase(req.phase)
|
||||
snapshot["active_seat"] = req.active_player
|
||||
|
||||
result = _policy_service.predict(
|
||||
snapshot,
|
||||
playstyle=req.playstyle,
|
||||
temperature=req.temperature,
|
||||
greedy=req.greedy,
|
||||
)
|
||||
|
||||
elapsed_ms = (time.time() - t_start) * 1000
|
||||
_session_stats["last_decision_ms"] = elapsed_ms
|
||||
|
||||
if "error" in result:
|
||||
raise HTTPException(500, detail=result["error"])
|
||||
|
||||
# Add log_prob and value estimate for online learning
|
||||
log_prob = 0.0
|
||||
value = 0.0
|
||||
try:
|
||||
import math
|
||||
conf = result.get("confidence", 0.5)
|
||||
log_prob = math.log(max(conf, 1e-8))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return DecideResponse(
|
||||
action=result["action"],
|
||||
action_index=result["action_index"],
|
||||
confidence=result["confidence"],
|
||||
log_prob=round(log_prob, 6),
|
||||
value=round(value, 6),
|
||||
probabilities=result.get("probabilities", {}),
|
||||
inference_ms=round(elapsed_ms, 2),
|
||||
)
|
||||
|
||||
|
||||
# ================================================================
|
||||
# POST /api/policy/collect
|
||||
# ================================================================
|
||||
|
||||
@router.post("/collect")
|
||||
async def collect_tuples(req: CollectRequest):
|
||||
"""Accept a batch of (state, action, reward) tuples from live games
|
||||
for online PPO updates.
|
||||
"""
|
||||
count = len(req.tuples)
|
||||
_session_stats["total_tuples_collected"] += count
|
||||
|
||||
if _online_store is not None:
|
||||
try:
|
||||
_online_store.add_tuples(req.tuples)
|
||||
except Exception as e:
|
||||
logger.error("Failed to store tuples: %s", e)
|
||||
raise HTTPException(500, detail=str(e))
|
||||
|
||||
return {
|
||||
"accepted": count,
|
||||
"total_buffered": (
|
||||
_online_store.size() if _online_store else count
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ================================================================
|
||||
# POST /api/policy/reward
|
||||
# ================================================================
|
||||
|
||||
@router.post("/reward")
|
||||
async def submit_reward(req: RewardRequest):
|
||||
"""Submit end-of-game reward for all tuples in a game.
|
||||
Propagates the terminal reward back through the game's trajectory.
|
||||
"""
|
||||
_session_stats["total_rewards_submitted"] += 1
|
||||
_session_stats["games_completed"] += 1
|
||||
|
||||
if _online_store is not None:
|
||||
try:
|
||||
_online_store.assign_reward(
|
||||
game_id=req.game_id,
|
||||
reward=req.reward,
|
||||
winner_seat=req.winner_seat,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Failed to assign reward: %s", e)
|
||||
raise HTTPException(500, detail=str(e))
|
||||
|
||||
return {
|
||||
"game_id": req.game_id,
|
||||
"reward": req.reward,
|
||||
"status": "accepted",
|
||||
}
|
||||
|
||||
|
||||
# ================================================================
|
||||
# GET /api/policy/health
|
||||
# ================================================================
|
||||
|
||||
@router.get("/health")
|
||||
async def policy_health():
|
||||
"""Readiness check for the live Forge IPC loop."""
|
||||
model_loaded = (
|
||||
_policy_service is not None and _policy_service._loaded
|
||||
)
|
||||
return {
|
||||
"ready": model_loaded,
|
||||
"status": "ok" if model_loaded else "degraded",
|
||||
"model_loaded": model_loaded,
|
||||
"online_store": _online_store is not None,
|
||||
}
|
||||
|
||||
|
||||
# ================================================================
|
||||
# GET /api/policy/stats
|
||||
# ================================================================
|
||||
|
||||
@router.get("/stats")
|
||||
async def policy_stats():
|
||||
"""Online learning session statistics."""
|
||||
uptime = time.time() - _session_stats["session_start"]
|
||||
return {
|
||||
**_session_stats,
|
||||
"uptime_seconds": round(uptime, 1),
|
||||
"decisions_per_second": (
|
||||
round(_session_stats["total_decisions"] / max(uptime, 1), 2)
|
||||
),
|
||||
"online_store_size": (
|
||||
_online_store.size() if _online_store else 0
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ================================================================
|
||||
# Helpers
|
||||
# ================================================================
|
||||
|
||||
_FORGE_PHASE_MAP = {
|
||||
"UNTAP": "main_1",
|
||||
"UPKEEP": "main_1",
|
||||
"DRAW": "main_1",
|
||||
"MAIN1": "main_1",
|
||||
"BEGIN_COMBAT": "combat",
|
||||
"DECLARE_ATTACKERS": "combat",
|
||||
"DECLARE_BLOCKERS": "combat",
|
||||
"DAMAGE": "combat",
|
||||
"COMBAT_DAMAGE": "combat",
|
||||
"END_COMBAT": "combat",
|
||||
"MAIN2": "main_2",
|
||||
"END": "end",
|
||||
"END_STEP": "end",
|
||||
"CLEANUP": "end",
|
||||
}
|
||||
|
||||
|
||||
def _map_forge_phase(phase: str) -> str:
|
||||
"""Map Forge phase string to ML GamePhase enum value."""
|
||||
return _FORGE_PHASE_MAP.get(phase.upper(), "main_1")
|
||||
@@ -0,0 +1,343 @@
|
||||
package commanderailab.ai;
|
||||
|
||||
import commanderailab.ml.LearnedPolicy;
|
||||
import commanderailab.ml.MacroActionExecutor;
|
||||
import commanderailab.ml.PolicyServerClient;
|
||||
import commanderailab.schema.DecisionSnapshot;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonObject;
|
||||
|
||||
import java.io.*;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
|
||||
/**
|
||||
* PolicyClient — Phase 2 bridge between Forge decision points and the
|
||||
* Python policy server for live online training.
|
||||
*
|
||||
* This class extends the existing LearnedPolicy/PolicyServerClient
|
||||
* infrastructure with two key Phase 2 capabilities:
|
||||
*
|
||||
* 1. Calls the new /api/policy/decide endpoint (purpose-built for live
|
||||
* IPC with richer state including all zones, stack, and priority)
|
||||
* instead of the batch-oriented /api/ml/predict.
|
||||
*
|
||||
* 2. Collects (state, action, reward) tuples in real-time during live
|
||||
* Forge games and flushes them to the policy server for online PPO
|
||||
* updates via POST /api/policy/collect.
|
||||
*
|
||||
* Falls back to ForgeBuiltinPolicy heuristics if the policy server
|
||||
* is unreachable or returns an error.
|
||||
*
|
||||
* Usage:
|
||||
* PolicyClient client = new PolicyClient("http://localhost:8080");
|
||||
* if (client.connect()) {
|
||||
* // At each Forge decision point:
|
||||
* String action = client.decide(fullGameState);
|
||||
* // After game ends:
|
||||
* client.submitReward(gameId, reward);
|
||||
* client.flushCollectedTuples();
|
||||
* }
|
||||
*
|
||||
* Refs: Issue #83 Phase 2.1
|
||||
*/
|
||||
public class PolicyClient {
|
||||
|
||||
private static final Gson GSON = new GsonBuilder().create();
|
||||
private static final int CONNECT_TIMEOUT_MS = 2000;
|
||||
private static final int READ_TIMEOUT_MS = 30_000;
|
||||
|
||||
private final String baseUrl;
|
||||
private final LearnedPolicy learnedPolicy;
|
||||
private volatile boolean serverAvailable = false;
|
||||
private volatile String lastError = "";
|
||||
|
||||
// Online learning tuple buffer
|
||||
private final ConcurrentLinkedQueue<JsonObject> tupleBuffer = new ConcurrentLinkedQueue<>();
|
||||
private static final int FLUSH_THRESHOLD = 64;
|
||||
|
||||
// Stats
|
||||
private int totalDecisions = 0;
|
||||
private int policyDecisions = 0;
|
||||
private int fallbackDecisions = 0;
|
||||
private long totalLatencyMs = 0;
|
||||
|
||||
public PolicyClient(String serverUrl) {
|
||||
this.baseUrl = serverUrl.endsWith("/")
|
||||
? serverUrl.substring(0, serverUrl.length() - 1)
|
||||
: serverUrl;
|
||||
this.learnedPolicy = new LearnedPolicy(serverUrl);
|
||||
}
|
||||
|
||||
public PolicyClient(String serverUrl, String playstyle, boolean greedy) {
|
||||
this(serverUrl);
|
||||
this.learnedPolicy.setPlaystyle(playstyle);
|
||||
this.learnedPolicy.setGreedy(greedy);
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// Connection
|
||||
// ================================================================
|
||||
|
||||
/**
|
||||
* Check that the policy server's /api/policy/decide endpoint is live
|
||||
* and a model is loaded. Falls back to checking /api/ml/model if the
|
||||
* new endpoint isn't available yet.
|
||||
*/
|
||||
public boolean connect() {
|
||||
try {
|
||||
String json = httpGet(baseUrl + "/api/policy/health");
|
||||
serverAvailable = json.contains("\"ready\":true")
|
||||
|| json.contains("\"status\":\"ok\"");
|
||||
} catch (Exception e) {
|
||||
// Fall back to legacy health check
|
||||
serverAvailable = learnedPolicy.connect();
|
||||
}
|
||||
if (serverAvailable) {
|
||||
System.out.println("[PolicyClient] Connected to policy server");
|
||||
} else {
|
||||
System.out.println("[PolicyClient] Server unavailable, using Forge built-in AI");
|
||||
}
|
||||
return serverAvailable;
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// Phase 2.1 — Live decision endpoint
|
||||
// ================================================================
|
||||
|
||||
/**
|
||||
* Request a macro-action decision from the policy server using the
|
||||
* full Forge game state (all zones, stack, priority).
|
||||
*
|
||||
* This calls POST /api/policy/decide which is purpose-built for the
|
||||
* live IPC loop, as opposed to /api/ml/predict which was designed
|
||||
* for batch inference.
|
||||
*
|
||||
* @param gameState Full game state JSON with all zones and priority info
|
||||
* @return macro-action name (e.g., "cast_creature"), or "pass" on failure
|
||||
*/
|
||||
public String decide(JsonObject gameState) {
|
||||
totalDecisions++;
|
||||
long t0 = System.currentTimeMillis();
|
||||
|
||||
if (!serverAvailable) {
|
||||
fallbackDecisions++;
|
||||
return MacroActionExecutor.PASS;
|
||||
}
|
||||
|
||||
try {
|
||||
// Add inference params
|
||||
gameState.addProperty("playstyle", learnedPolicy.getPlaystyle());
|
||||
gameState.addProperty("greedy", learnedPolicy.isGreedy());
|
||||
|
||||
String responseJson = httpPost(
|
||||
baseUrl + "/api/policy/decide",
|
||||
gameState.toString()
|
||||
);
|
||||
|
||||
JsonObject resp = GSON.fromJson(responseJson, JsonObject.class);
|
||||
long elapsed = System.currentTimeMillis() - t0;
|
||||
totalLatencyMs += elapsed;
|
||||
|
||||
if (resp.has("action") && !resp.has("error")) {
|
||||
policyDecisions++;
|
||||
String action = resp.get("action").getAsString();
|
||||
|
||||
// Store tuple for online learning
|
||||
collectTuple(gameState, action, resp);
|
||||
|
||||
return action;
|
||||
} else {
|
||||
fallbackDecisions++;
|
||||
lastError = resp.has("error")
|
||||
? resp.get("error").getAsString()
|
||||
: "Unknown error";
|
||||
return MacroActionExecutor.PASS;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
fallbackDecisions++;
|
||||
lastError = e.getMessage();
|
||||
return MacroActionExecutor.PASS;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience: decide from a DecisionSnapshot (delegates to the
|
||||
* existing LearnedPolicy path if /api/policy/decide is unavailable).
|
||||
*/
|
||||
public String decide(DecisionSnapshot snapshot) {
|
||||
return learnedPolicy.decideAction(snapshot);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a decision to a concrete Forge action.
|
||||
*/
|
||||
public MacroActionExecutor.ActionResolution resolveAction(
|
||||
JsonObject gameState,
|
||||
List<String> hand,
|
||||
List<String> battlefield,
|
||||
List<String> commandZone,
|
||||
int manaAvailable,
|
||||
String commanderName,
|
||||
int commanderTax
|
||||
) {
|
||||
String macroAction = decide(gameState);
|
||||
return MacroActionExecutor.resolve(
|
||||
macroAction, hand, battlefield, commandZone,
|
||||
manaAvailable, commanderName, commanderTax
|
||||
);
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// Phase 2.1 — Online learning tuple collection
|
||||
// ================================================================
|
||||
|
||||
/**
|
||||
* Buffer a (state, action, prediction_metadata) tuple for later
|
||||
* submission to the policy server for online PPO updates.
|
||||
*/
|
||||
private void collectTuple(JsonObject state, String action, JsonObject prediction) {
|
||||
JsonObject tuple = new JsonObject();
|
||||
tuple.add("state", state);
|
||||
tuple.addProperty("action", action);
|
||||
if (prediction.has("action_index")) {
|
||||
tuple.addProperty("action_index", prediction.get("action_index").getAsInt());
|
||||
}
|
||||
if (prediction.has("log_prob")) {
|
||||
tuple.addProperty("log_prob", prediction.get("log_prob").getAsDouble());
|
||||
}
|
||||
if (prediction.has("value")) {
|
||||
tuple.addProperty("value", prediction.get("value").getAsDouble());
|
||||
}
|
||||
tuple.addProperty("timestamp", System.currentTimeMillis());
|
||||
|
||||
tupleBuffer.add(tuple);
|
||||
|
||||
// Auto-flush when buffer is full
|
||||
if (tupleBuffer.size() >= FLUSH_THRESHOLD) {
|
||||
flushCollectedTuples();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Submit end-of-game reward for all tuples in the current game.
|
||||
* Call this after a Forge game completes.
|
||||
*/
|
||||
public void submitReward(String gameId, double reward) {
|
||||
try {
|
||||
JsonObject body = new JsonObject();
|
||||
body.addProperty("game_id", gameId);
|
||||
body.addProperty("reward", reward);
|
||||
httpPost(baseUrl + "/api/policy/reward", body.toString());
|
||||
} catch (Exception e) {
|
||||
System.err.println("[PolicyClient] Failed to submit reward: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush buffered (state, action, reward) tuples to the policy server
|
||||
* for online PPO updates via POST /api/policy/collect.
|
||||
*/
|
||||
public void flushCollectedTuples() {
|
||||
if (tupleBuffer.isEmpty()) return;
|
||||
|
||||
JsonArray batch = new JsonArray();
|
||||
JsonObject tuple;
|
||||
while ((tuple = tupleBuffer.poll()) != null) {
|
||||
batch.add(tuple);
|
||||
}
|
||||
|
||||
try {
|
||||
JsonObject body = new JsonObject();
|
||||
body.add("tuples", batch);
|
||||
body.addProperty("count", batch.size());
|
||||
httpPost(baseUrl + "/api/policy/collect", body.toString());
|
||||
System.out.println("[PolicyClient] Flushed " + batch.size() + " tuples");
|
||||
} catch (Exception e) {
|
||||
System.err.println("[PolicyClient] Failed to flush tuples: " + e.getMessage());
|
||||
// Re-queue on failure
|
||||
for (int i = 0; i < batch.size(); i++) {
|
||||
tupleBuffer.add(batch.get(i).getAsJsonObject());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// Accessors
|
||||
// ================================================================
|
||||
|
||||
public boolean isServerAvailable() { return serverAvailable; }
|
||||
public String getLastError() { return lastError; }
|
||||
public int getTotalDecisions() { return totalDecisions; }
|
||||
public int getPolicyDecisions() { return policyDecisions; }
|
||||
public int getFallbackDecisions() { return fallbackDecisions; }
|
||||
public int getPendingTuples() { return tupleBuffer.size(); }
|
||||
|
||||
public double getAverageLatencyMs() {
|
||||
return policyDecisions > 0
|
||||
? (double) totalLatencyMs / policyDecisions
|
||||
: 0;
|
||||
}
|
||||
|
||||
public String getStatsSummary() {
|
||||
return String.format(
|
||||
"PolicyClient[decisions=%d, policy=%d, fallback=%d, avg=%.1fms, pending=%d]",
|
||||
totalDecisions, policyDecisions, fallbackDecisions,
|
||||
getAverageLatencyMs(), tupleBuffer.size()
|
||||
);
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// HTTP helpers
|
||||
// ================================================================
|
||||
|
||||
private String httpGet(String url) throws IOException {
|
||||
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
|
||||
conn.setRequestMethod("GET");
|
||||
conn.setConnectTimeout(CONNECT_TIMEOUT_MS);
|
||||
conn.setReadTimeout(READ_TIMEOUT_MS);
|
||||
conn.setRequestProperty("Accept", "application/json");
|
||||
int code = conn.getResponseCode();
|
||||
if (code != 200) {
|
||||
throw new IOException("HTTP " + code + ": " + readStream(conn.getErrorStream()));
|
||||
}
|
||||
return readStream(conn.getInputStream());
|
||||
}
|
||||
|
||||
private String httpPost(String url, String jsonBody) throws IOException {
|
||||
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
|
||||
conn.setRequestMethod("POST");
|
||||
conn.setConnectTimeout(CONNECT_TIMEOUT_MS);
|
||||
conn.setReadTimeout(READ_TIMEOUT_MS);
|
||||
conn.setDoOutput(true);
|
||||
conn.setRequestProperty("Content-Type", "application/json");
|
||||
conn.setRequestProperty("Accept", "application/json");
|
||||
try (OutputStream os = conn.getOutputStream()) {
|
||||
os.write(jsonBody.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
int code = conn.getResponseCode();
|
||||
if (code != 200) {
|
||||
throw new IOException("HTTP " + code + ": " + readStream(conn.getErrorStream()));
|
||||
}
|
||||
return readStream(conn.getInputStream());
|
||||
}
|
||||
|
||||
private String readStream(InputStream stream) throws IOException {
|
||||
if (stream == null) return "";
|
||||
try (BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(stream, StandardCharsets.UTF_8))) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
sb.append(line);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
import commanderailab.ai.PolicyClient;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
@@ -13,10 +14,11 @@ import java.util.function.Consumer;
|
||||
/**
|
||||
* GameSession — holds state for one live interactive Commander game.
|
||||
*
|
||||
* PHASE 1 STUB: generates synthetic game state so you can verify the full
|
||||
* WebSocket contract round-trips correctly before wiring in real Forge IPC.
|
||||
* PHASE 2: integrates PolicyClient for bidirectional sync between
|
||||
* the Java game loop and the Python policy server.
|
||||
*
|
||||
* Replace run() in Phase 2 once the contract is verified end-to-end.
|
||||
* AI turns now consult the policy server for macro-action decisions
|
||||
* instead of using random stub logic.
|
||||
*/
|
||||
public class GameSession {
|
||||
|
||||
@@ -27,6 +29,7 @@ public class GameSession {
|
||||
private final String forgeJarPath;
|
||||
private final String forgeWorkDir;
|
||||
private final BlockingQueue<String> humanActionQueue;
|
||||
private final PolicyClient policyClient;
|
||||
|
||||
private volatile boolean running = false;
|
||||
private volatile int turnNumber = 1;
|
||||
@@ -56,11 +59,21 @@ public class GameSession {
|
||||
public GameSession(List<String> deckNames, Long seed,
|
||||
String forgeJarPath, String forgeWorkDir,
|
||||
BlockingQueue<String> humanActionQueue) {
|
||||
this(deckNames, seed, forgeJarPath, forgeWorkDir, humanActionQueue,
|
||||
new PolicyClient("http://localhost:8080"));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public GameSession(List<String> deckNames, Long seed,
|
||||
String forgeJarPath, String forgeWorkDir,
|
||||
BlockingQueue<String> humanActionQueue,
|
||||
PolicyClient policyClient) {
|
||||
this.deckNames = deckNames;
|
||||
this.seed = seed;
|
||||
this.forgeJarPath = forgeJarPath;
|
||||
this.forgeWorkDir = forgeWorkDir;
|
||||
this.humanActionQueue = humanActionQueue;
|
||||
this.policyClient = policyClient;
|
||||
|
||||
int seats = Math.min(deckNames.size(), 4);
|
||||
life = new int[seats];
|
||||
@@ -74,8 +87,8 @@ public class GameSession {
|
||||
for (int i = 0; i < seats; i++) {
|
||||
life[i] = 40;
|
||||
hands[i] = new ArrayList<>(List.of(
|
||||
"card_stub_1", "card_stub_2", "card_stub_3",
|
||||
"card_stub_4", "card_stub_5", "card_stub_6", "card_stub_7"
|
||||
"card_stub_1", "card_stub_2", "card_stub_3",
|
||||
"card_stub_4", "card_stub_5", "card_stub_6", "card_stub_7"
|
||||
));
|
||||
battlefields[i] = new ArrayList<>();
|
||||
graveyards[i] = new ArrayList<>();
|
||||
@@ -85,28 +98,27 @@ public class GameSession {
|
||||
|
||||
public boolean isRunning() { return running; }
|
||||
public int getTurnNumber() { return turnNumber; }
|
||||
|
||||
public void stop() {
|
||||
running = false;
|
||||
}
|
||||
public void stop() { running = false; }
|
||||
|
||||
/**
|
||||
* Run the game loop. Calls onStateChange after every action resolves.
|
||||
* Blocks until the game ends or stop() is called.
|
||||
*
|
||||
* PHASE 1 STUB: advances through phases automatically; on MAIN1 of seat 0's
|
||||
* turn, pauses and waits for the human client to send an action.
|
||||
* PHASE 2: AI turns now consult the PolicyClient for decisions.
|
||||
* The policy server returns a macro-action which is applied to game state.
|
||||
* Falls back to PASS_PRIORITY if the policy server is unreachable.
|
||||
*/
|
||||
public void run(Consumer<Object> onStateChange) throws InterruptedException {
|
||||
public void run(Consumer<Map<String, Object>> onStateChange) throws InterruptedException {
|
||||
running = true;
|
||||
Random rng = seed != null ? new Random(seed) : new Random();
|
||||
|
||||
System.out.println("[Session] Game started. Decks: " + deckNames);
|
||||
System.out.println("[Session] PolicyClient endpoint: " + policyClient.getBaseUrl());
|
||||
|
||||
while (running && !isGameOver()) {
|
||||
String currentPhase = PHASES[phaseIndex];
|
||||
phase = currentPhase;
|
||||
activePlayer = (turnNumber - 1) % deckNames.size();
|
||||
|
||||
boolean isHumanTurn = (activePlayer == 0);
|
||||
boolean isInteractivePhase = currentPhase.equals("MAIN1") || currentPhase.equals("MAIN2");
|
||||
|
||||
@@ -117,7 +129,6 @@ public class GameSession {
|
||||
|
||||
System.out.println("[Session] Awaiting human action for " + currentPhase + "...");
|
||||
String actionJson = humanActionQueue.poll(120, TimeUnit.SECONDS);
|
||||
|
||||
if (actionJson == null) {
|
||||
System.out.println("[Session] Human timed out — auto passing.");
|
||||
} else {
|
||||
@@ -125,13 +136,18 @@ public class GameSession {
|
||||
System.out.println("[Session] Human action applied: " + actionJson);
|
||||
}
|
||||
awaitingHumanInput = false;
|
||||
|
||||
} else if (!isHumanTurn && isInteractivePhase) {
|
||||
// Phase 2: consult PolicyClient for AI decision
|
||||
priorityPlayer = activePlayer;
|
||||
Map<String, Object> snapshot = buildStateSnapshot();
|
||||
String policyAction = consultPolicy(activePlayer, snapshot);
|
||||
applyPolicyAction(activePlayer, policyAction, rng);
|
||||
System.out.println("[Session] AI-" + activePlayer + " policy action: " + policyAction);
|
||||
|
||||
} else {
|
||||
// AI turn — small delay for readability
|
||||
// Non-interactive phase — small delay for readability
|
||||
Thread.sleep(300);
|
||||
// Simulate AI occasionally dealing damage to human
|
||||
if (!isHumanTurn && currentPhase.equals("DAMAGE") && life[0] > 5) {
|
||||
life[0] -= rng.nextInt(4);
|
||||
}
|
||||
}
|
||||
|
||||
// Advance phase
|
||||
@@ -140,7 +156,6 @@ public class GameSession {
|
||||
phaseIndex = 0;
|
||||
turnNumber++;
|
||||
}
|
||||
|
||||
onStateChange.accept(buildStateSnapshot());
|
||||
}
|
||||
|
||||
@@ -148,11 +163,70 @@ public class GameSession {
|
||||
System.out.println("[Session] Game ended after " + turnNumber + " turns.");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// Policy integration (Phase 2)
|
||||
// ---------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Consult the Python policy server for an AI decision.
|
||||
* Sends the current game snapshot and returns the recommended macro-action.
|
||||
* Falls back to PASS_PRIORITY on error.
|
||||
*/
|
||||
private String consultPolicy(int seat, Map<String, Object> snapshot) {
|
||||
try {
|
||||
String stateJson = GSON.toJson(snapshot);
|
||||
PolicyClient.PolicyDecision decision = policyClient.decide(stateJson, seat);
|
||||
if (decision != null && decision.action() != null) {
|
||||
System.out.println("[Session] Policy decision for seat " + seat
|
||||
+ ": " + decision.action()
|
||||
+ " (confidence=" + String.format("%.2f", decision.confidence()) + ")");
|
||||
return decision.action();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.err.println("[Session] Policy server error for seat " + seat + ": " + e.getMessage());
|
||||
}
|
||||
return "PASS_PRIORITY";
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a policy-recommended macro-action for an AI player.
|
||||
*/
|
||||
private void applyPolicyAction(int seat, String action, Random rng) {
|
||||
switch (action) {
|
||||
case "CAST_SPELL" -> {
|
||||
if (!hands[seat].isEmpty()) {
|
||||
String card = hands[seat].remove(0);
|
||||
battlefields[seat].add(card);
|
||||
System.out.println("[Session] AI-" + seat + " cast: " + card);
|
||||
}
|
||||
}
|
||||
case "PLAY_LAND" -> {
|
||||
if (!hands[seat].isEmpty()) {
|
||||
String card = hands[seat].remove(hands[seat].size() - 1);
|
||||
battlefields[seat].add(card);
|
||||
}
|
||||
}
|
||||
case "ATTACK" -> {
|
||||
int target = (seat + 1) % life.length;
|
||||
if (life[target] > 0) {
|
||||
int dmg = rng.nextInt(3) + 1;
|
||||
life[target] -= dmg;
|
||||
System.out.println("[Session] AI-" + seat + " attacks player " + target + " for " + dmg);
|
||||
}
|
||||
}
|
||||
case "PASS_PRIORITY" -> { /* no-op */ }
|
||||
default -> System.out.println("[Session] Unknown policy action: " + action);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// Human action handling
|
||||
// ---------------------------------------------------------
|
||||
|
||||
private void applyAction(String actionJson) {
|
||||
try {
|
||||
JsonObject action = JsonParser.parseString(actionJson).getAsJsonObject();
|
||||
String type = action.has("type") ? action.get("type").getAsString() : "PASS_PRIORITY";
|
||||
|
||||
switch (type) {
|
||||
case "PLAY_LAND", "CAST_SPELL" -> {
|
||||
String cardId = action.has("cardId") ? action.get("cardId").getAsString() : null;
|
||||
@@ -172,7 +246,6 @@ public class GameSession {
|
||||
private boolean isGameOver() {
|
||||
int alive = 0;
|
||||
for (int lp : life) if (lp > 0) alive++;
|
||||
// Phase 1 stub: end after 50 turns or only 1 player alive
|
||||
return alive <= 1 || turnNumber > 50;
|
||||
}
|
||||
|
||||
@@ -210,7 +283,7 @@ public class GameSession {
|
||||
p.put("poison", poison[i]);
|
||||
p.put("commanderTax", commanderTax[i]);
|
||||
p.put("handCount", hands[i].size());
|
||||
p.put("hand", i == 0 ? hands[i] : List.of()); // hide opponent hands
|
||||
p.put("hand", i == 0 ? hands[i] : List.of());
|
||||
p.put("battlefield", battlefields[i]);
|
||||
p.put("graveyard", graveyards[i]);
|
||||
p.put("commandZone", commandZones[i]);
|
||||
|
||||
Reference in new issue
Block a user