From 11940835fc26c46d485b3e57da039b841232861d Mon Sep 17 00:00:00 2001 From: KoalaTrapLord <265133236+KoalaTrapLord@users.noreply.github.com> Date: Mon, 30 Mar 2026 12:33:28 -0400 Subject: [PATCH] fix: validate ollama suggest_cards names against DB + Scryfall before pipeline merge - Add validate_card_names() helper: checks collection_entries DB first, then falls back to Scryfall /cards/named?fuzzy= for unknowns. Drops any name that cannot be resolved (hallucinated cards). - Inject validation in Step 4 immediately after each suggest_cards() call so hallucinated names never reach ratio enforcement or assembly. - Log dropped hallucinations at WARNING level with count + examples. - Resolves the 'Sonic Hedgehog Charger' / 'Sonic Boom' class of LLM-invented card names that were silently entering the deck. --- .../deck_builder/pipeline/build_deck.py | 125 +++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/src/commander_ai_lab/deck_builder/pipeline/build_deck.py b/src/commander_ai_lab/deck_builder/pipeline/build_deck.py index 8ae8979..8a978bf 100644 --- a/src/commander_ai_lab/deck_builder/pipeline/build_deck.py +++ b/src/commander_ai_lab/deck_builder/pipeline/build_deck.py @@ -8,9 +8,12 @@ Orchestrates: EDHrec data -> Scryfall -> Ollama suggestions -> from __future__ import annotations import logging +import sqlite3 import time from typing import Dict, List, Optional +import httpx + from ..api import edhrec, scryfall from ..api import ollama_client as ollama from ..api.sim_insights import get_sim_insights @@ -20,6 +23,101 @@ from ..core.rules_engine import check_ban_list, filter_by_color_identity, valida logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Card-name validation helper +# --------------------------------------------------------------------------- + +_SCRYFALL_NAMED_URL = "https://api.scryfall.com/cards/named" +_SCRYFALL_HEADERS = {"User-Agent": "commander-ai-lab/1.0"} +_SCRYFALL_TIMEOUT = 5.0 # seconds per lookup + + +def validate_card_names( + names: List[str], + db_conn: Optional[sqlite3.Connection] = None, +) -> List[str]: + """Return only names that resolve to real MTG cards. + + Resolution order: + 1. Exact match (case-insensitive) in ``collection_entries`` DB. + 2. Scryfall ``/cards/named?fuzzy=`` HTTP lookup. + + Names that fail both checks are dropped and logged as hallucinations. + The returned list preserves the original order of surviving names. + + Parameters + ---------- + names: + Raw list of card name strings from ``suggest_cards()``. + db_conn: + Optional SQLite connection to the lab database. When provided, + local collection lookups are attempted before hitting Scryfall, + which is faster and avoids unnecessary network calls for cards + the user already owns. + """ + if not names: + return [] + + valid: List[str] = [] + dropped: List[str] = [] + + for raw_name in names: + name = raw_name.strip() + if not name: + continue + + resolved = False + + # -- 1. Local DB check -- + if db_conn is not None: + try: + row = db_conn.execute( + "SELECT name FROM collection_entries " + "WHERE name = ? COLLATE NOCASE LIMIT 1", + (name,), + ).fetchone() + if row: + valid.append(row[0]) # use canonical capitalisation from DB + resolved = True + except Exception as db_err: + logger.debug("DB lookup failed for '%s': %s", name, db_err) + + if resolved: + continue + + # -- 2. Scryfall fuzzy lookup -- + try: + import urllib.parse + url = f"{_SCRYFALL_NAMED_URL}?fuzzy={urllib.parse.quote(name)}" + resp = httpx.get(url, headers=_SCRYFALL_HEADERS, timeout=_SCRYFALL_TIMEOUT) + if resp.status_code == 200: + data = resp.json() + canonical = data.get("name", name) + valid.append(canonical) + resolved = True + # 404 = card not found; any other status = treat as unresolved + except Exception as sf_err: + logger.debug("Scryfall lookup failed for '%s': %s", name, sf_err) + + if not resolved: + dropped.append(name) + + if dropped: + examples = dropped[:5] + logger.warning( + "validate_card_names: dropped %d hallucinated/unresolvable name(s) " + "(showing up to 5): %s", + len(dropped), + examples, + ) + + return valid + + +# --------------------------------------------------------------------------- +# Pipeline +# --------------------------------------------------------------------------- + def build_deck(request: BuildRequest) -> BuildResult: """ @@ -90,6 +188,14 @@ def build_deck(request: BuildRequest) -> BuildResult: all_names.extend(c.name for c in cards) all_names = list(set(all_names)) + # Grab a DB connection once for the validation step (best-effort) + _db_conn: Optional[sqlite3.Connection] = None + if request.collection_path: + try: + _db_conn = sqlite3.connect(request.collection_path) + except Exception as e: + logger.debug("Could not open DB for name validation: %s", e) + # Ask Ollama to suggest cards for categories that need more. # sim_context is forwarded so the LLM factors in what traits have # been statistically successful in our own game simulations. @@ -106,9 +212,26 @@ def build_deck(request: BuildRequest) -> BuildResult: strategy_notes=request.strategy_notes, sim_context=sim_context, ) + + # ── Validation gate: drop hallucinated / unresolvable names ────── + before = len(suggested) + suggested = validate_card_names(suggested, db_conn=_db_conn) + after = len(suggested) + if before != after: + warnings.append( + f"Step 4 [{category}]: {before - after} suggested name(s) could not be " + f"resolved against DB/Scryfall and were dropped." + ) + ollama_suggestions[category] = suggested all_names.extend(suggested) - logger.info(f" Ollama {category}: {len(suggested)} suggestions") + logger.info(f" Ollama {category}: {after} valid suggestions ({before} raw)") + + if _db_conn: + try: + _db_conn.close() + except Exception: + pass sources.append("ollama")