fix: validate ollama suggest_cards names against DB + Scryfall before pipeline merge

- Add validate_card_names() helper: checks collection_entries DB first,
  then falls back to Scryfall /cards/named?fuzzy= for unknowns.
  Drops any name that cannot be resolved (hallucinated cards).
- Inject validation in Step 4 immediately after each suggest_cards()
  call so hallucinated names never reach ratio enforcement or assembly.
- Log dropped hallucinations at WARNING level with count + examples.
- Resolves the 'Sonic Hedgehog Charger' / 'Sonic Boom' class of
  LLM-invented card names that were silently entering the deck.
This commit is contained in:
KoalaTrapLord committed 2026-03-30 12:33:28 -04:00
1 parent d12b47f874
commit 11940835fc
1 file changed
+124 -1
@@ -8,9 +8,12 @@ Orchestrates: EDHrec data -> Scryfall -> Ollama suggestions ->
from __future__ import annotations
import logging
import sqlite3
import time
from typing import Dict, List, Optional
import httpx
from ..api import edhrec, scryfall
from ..api import ollama_client as ollama
from ..api.sim_insights import get_sim_insights
@@ -20,6 +23,101 @@ from ..core.rules_engine import check_ban_list, filter_by_color_identity, valida
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Card-name validation helper
# ---------------------------------------------------------------------------
_SCRYFALL_NAMED_URL = "https://api.scryfall.com/cards/named"
_SCRYFALL_HEADERS = {"User-Agent": "commander-ai-lab/1.0"}
_SCRYFALL_TIMEOUT = 5.0 # seconds per lookup
def validate_card_names(
names: List[str],
db_conn: Optional[sqlite3.Connection] = None,
) -> List[str]:
"""Return only names that resolve to real MTG cards.
Resolution order:
1. Exact match (case-insensitive) in ``collection_entries`` DB.
2. Scryfall ``/cards/named?fuzzy=<name>`` HTTP lookup.
Names that fail both checks are dropped and logged as hallucinations.
The returned list preserves the original order of surviving names.
Parameters
----------
names:
Raw list of card name strings from ``suggest_cards()``.
db_conn:
Optional SQLite connection to the lab database. When provided,
local collection lookups are attempted before hitting Scryfall,
which is faster and avoids unnecessary network calls for cards
the user already owns.
"""
if not names:
return []
valid: List[str] = []
dropped: List[str] = []
for raw_name in names:
name = raw_name.strip()
if not name:
continue
resolved = False
# -- 1. Local DB check --
if db_conn is not None:
try:
row = db_conn.execute(
"SELECT name FROM collection_entries "
"WHERE name = ? COLLATE NOCASE LIMIT 1",
(name,),
).fetchone()
if row:
valid.append(row[0]) # use canonical capitalisation from DB
resolved = True
except Exception as db_err:
logger.debug("DB lookup failed for '%s': %s", name, db_err)
if resolved:
continue
# -- 2. Scryfall fuzzy lookup --
try:
import urllib.parse
url = f"{_SCRYFALL_NAMED_URL}?fuzzy={urllib.parse.quote(name)}"
resp = httpx.get(url, headers=_SCRYFALL_HEADERS, timeout=_SCRYFALL_TIMEOUT)
if resp.status_code == 200:
data = resp.json()
canonical = data.get("name", name)
valid.append(canonical)
resolved = True
# 404 = card not found; any other status = treat as unresolved
except Exception as sf_err:
logger.debug("Scryfall lookup failed for '%s': %s", name, sf_err)
if not resolved:
dropped.append(name)
if dropped:
examples = dropped[:5]
logger.warning(
"validate_card_names: dropped %d hallucinated/unresolvable name(s) "
"(showing up to 5): %s",
len(dropped),
examples,
)
return valid
# ---------------------------------------------------------------------------
# Pipeline
# ---------------------------------------------------------------------------
def build_deck(request: BuildRequest) -> BuildResult:
"""
@@ -90,6 +188,14 @@ def build_deck(request: BuildRequest) -> BuildResult:
all_names.extend(c.name for c in cards)
all_names = list(set(all_names))
# Grab a DB connection once for the validation step (best-effort)
_db_conn: Optional[sqlite3.Connection] = None
if request.collection_path:
try:
_db_conn = sqlite3.connect(request.collection_path)
except Exception as e:
logger.debug("Could not open DB for name validation: %s", e)
# Ask Ollama to suggest cards for categories that need more.
# sim_context is forwarded so the LLM factors in what traits have
# been statistically successful in our own game simulations.
@@ -106,9 +212,26 @@ def build_deck(request: BuildRequest) -> BuildResult:
strategy_notes=request.strategy_notes,
sim_context=sim_context,
)
# ── Validation gate: drop hallucinated / unresolvable names ──────
before = len(suggested)
suggested = validate_card_names(suggested, db_conn=_db_conn)
after = len(suggested)
if before != after:
warnings.append(
f"Step 4 [{category}]: {before - after} suggested name(s) could not be "
f"resolved against DB/Scryfall and were dropped."
)
ollama_suggestions[category] = suggested
all_names.extend(suggested)
logger.info(f" Ollama {category}: {len(suggested)} suggestions")
logger.info(f" Ollama {category}: {after} valid suggestions ({before} raw)")
if _db_conn:
try:
_db_conn.close()
except Exception:
pass
sources.append("ollama")