From 495a3fa5d61a05649597eeedc21d141617fca570 Mon Sep 17 00:00:00 2001 From: momentum dev Date: Wed, 22 Jul 2026 20:31:16 -0500 Subject: [PATCH] =?UTF-8?q?feat:=20strategy=20knobs=20in=20the=20database?= =?UTF-8?q?=20=E2=80=94=20config=5Foverrides=20table,=20admin-managed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner preference: the bulk of TRADING_*/TRIGGER_* logic switches should live in the DB, not the compose file (keys + connection info stay in compose — the DB address cannot live inside the DB, and secrets don't belong in a table the stack can SELECT). Precedence at boot: compose env < overrides.env < config_overrides table; every entrypoint already routes through env_overrides.load(), which now applies the table last. Admin /api/config and profile-apply write the table (secrets whitelist unchanged); the config page shows source=database. DB down = silent fall-through to file/compose, boot never blocks on config. One-shot migration: python -m runtime.env_overrides seed-from-env run in any stack container to copy its current effective knobs into the table, after which the compose lines are dead weight strippable at the next redeploy. Co-Authored-By: Claude Fable 5 --- admin_api.py | 23 ++++--- runtime/env_overrides.py | 94 ++++++++++++++++++++++++++++- tests/unit/test_env_overrides_db.py | 41 +++++++++++++ 3 files changed, 148 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_env_overrides_db.py diff --git a/admin_api.py b/admin_api.py index 243cb51..a31efde 100644 --- a/admin_api.py +++ b/admin_api.py @@ -123,8 +123,9 @@ def _safe_overrides() -> dict: """Overrides for API responses with secret VALUES redacted — a /api/mode live switch writes ALPACA_LIVE_* into overrides.env; without this they'd echo back in every /api/status and /api/config response.""" + merged = {**env_overrides.read_overrides(), **env_overrides.read_db_overrides()} return {k: ("***" if any(m in k for m in SECRET_MARKERS) else v) - for k, v in env_overrides.read_overrides().items()} + for k, v in merged.items()} def _editable(key: str) -> bool: @@ -169,8 +170,10 @@ def effective_config() -> dict: both surface as ``compose``); a documented .env.example key that is absent surfaces as ``missing`` (drift). Secret values are masked.""" overrides = env_overrides.read_overrides() + db_overrides = env_overrides.read_db_overrides() example = _env_example_keys() - keys = set(example) | set(overrides) | {k for k in os.environ if _editable(k)} + keys = (set(example) | set(overrides) | set(db_overrides) + | {k for k in os.environ if _editable(k)}) out: dict[str, dict] = {} for key in sorted(keys): value = os.environ.get(key) @@ -179,7 +182,8 @@ def effective_config() -> dict: "source": "missing" if key in example else "unset", "drift": key in example} continue - source = "overrides.env" if key in overrides else "compose" + source = ("database" if key in db_overrides + else "overrides.env" if key in overrides else "compose") out[key] = {"value_masked": _mask_value(key, str(value)), "source": source} return out @@ -350,9 +354,14 @@ class Handler(BaseHTTPRequestHandler): bad = [k for k in list(updates) + list(deletes) if not _editable(k)] if bad: return self._send(400, {"error": f"not editable: {bad}"}) - merged = env_overrides.write_overrides( - {**{k: str(v) for k, v in updates.items()}, - **{k: None for k in deletes}}) + # knobs go to the DATABASE (owner-facing store; whitelist above + # already refused secrets). overrides.env keeps secrets/bootstrap. + try: + merged = env_overrides.write_db_overrides( + {**{k: str(v) for k, v in updates.items()}, + **{k: None for k in deletes}}) + except Exception as exc: # noqa: BLE001 + return self._send(502, {"error": f"db write failed: {exc}"}) return self._send(200, {"ok": True, "overrides": merged, "note": "restart affected services to apply"}) @@ -362,7 +371,7 @@ class Handler(BaseHTTPRequestHandler): if not prof: return self._send(404, {"error": f"no profile {name!r}"}) knobs = {k: str(v) for k, v in prof.items() if not k.startswith("_")} - merged = env_overrides.write_overrides(knobs) + merged = env_overrides.write_db_overrides(knobs) return self._send(200, {"ok": True, "applied": name, "overrides": merged, "note": "restart the loop to apply"}) diff --git a/runtime/env_overrides.py b/runtime/env_overrides.py index 8e920a9..d698a08 100644 --- a/runtime/env_overrides.py +++ b/runtime/env_overrides.py @@ -12,16 +12,23 @@ OVERRIDES_FILE = os.environ.get("ENV_OVERRIDES_FILE") or os.path.join( def load() -> int: - """Apply overrides (override=True). Returns count of applied keys.""" + """Apply overrides (override=True). Returns count of applied keys. + Order: overrides.env file first, then the config_overrides DB table (DB + wins — it is the owner-facing store; the file keeps secrets + bootstrap).""" + n = 0 try: from dotenv import load_dotenv if os.path.exists(OVERRIDES_FILE): before = dict(os.environ) load_dotenv(OVERRIDES_FILE, override=True) - return sum(1 for k, v in os.environ.items() if before.get(k) != v) + n = sum(1 for k, v in os.environ.items() if before.get(k) != v) except Exception: # noqa: BLE001 — a broken overrides file must not stop boot pass - return 0 + try: + n += load_db() + except Exception: # noqa: BLE001 + pass + return n def read_overrides() -> dict[str, str]: @@ -50,3 +57,84 @@ def write_overrides(updates: dict[str, str | None]) -> dict[str, str]: for k in sorted(cur): f.write(f"{k}={cur[k]}\n") return cur + + +# ---------- database-backed overrides (owner preference 2026-07-22) ---------- +# Strategy knobs (TRADING_*/TRIGGER_*/... — no secrets, admin whitelist enforces) +# live in the config_overrides table and are applied at boot AFTER the file +# pass, so precedence is: compose env < overrides.env < database. Secrets and +# connection info stay in compose/overrides.env by design — the DB address +# cannot live inside the DB, and secret values don't belong in a table the +# whole stack can SELECT. + +def _db_con(): + from storage.db_pg import get_connection + return get_connection("momentum") + + +def read_db_overrides(con=None) -> dict[str, str]: + """Best-effort: {} when the DB or table doesn't exist yet.""" + try: + con = con or _db_con() + rows = con.execute( + "SELECT key, value FROM config_overrides ORDER BY key").fetchall() + return {str(k): str(v) for k, v in rows} + except Exception: # noqa: BLE001 — missing table / DB down = no overrides + return {} + + +def write_db_overrides(updates: dict[str, str | None], con=None) -> dict[str, str]: + """Merge updates into config_overrides (None deletes). Creates the table on + first use. Raises on DB failure — a config write must not silently vanish.""" + con = con or _db_con() + con.execute( + "CREATE TABLE IF NOT EXISTS config_overrides (" + " key TEXT PRIMARY KEY, value TEXT NOT NULL," + " updated_at TIMESTAMPTZ NOT NULL DEFAULT now())") + for k, v in updates.items(): + if v is None: + con.execute("DELETE FROM config_overrides WHERE key=?", [k]) + else: + con.execute( + "INSERT INTO config_overrides (key, value) VALUES (?, ?) " + "ON CONFLICT (key) DO UPDATE SET value=EXCLUDED.value, " + "updated_at=now()", [k, str(v)]) + return read_db_overrides(con) + + +def load_db() -> int: + """Apply DB overrides onto os.environ (they win over compose + file). + Silent no-op when the DB is unreachable — boot must never block on config.""" + if os.environ.get("ENV_OVERRIDES_DB", "1") != "1": + return 0 + n = 0 + for k, v in read_db_overrides().items(): + if os.environ.get(k) != v: + os.environ[k] = v + n += 1 + return n + + +def _seed_from_env() -> dict[str, str]: + """One-shot migration: copy the current process env's strategy knobs into + config_overrides so the compose lines become strippable. Same whitelist as + the admin panel; secret-marked keys never enter the table.""" + prefixes = ("TRADING_", "TRIGGER_", "WATCHER_", "LIVE_BARS_", "NEWS_", + "OLLAMA_", "DISCOVER_", "SYMBOL_WEIGHTS") + secret = ("KEY", "SECRET", "TOKEN", "PASSWORD", "WEBHOOK", "URL") + knobs = {k: v for k, v in os.environ.items() + if (k.startswith(prefixes) or k == "ALPACA_DATA_FEED") + and not any(m in k.upper() for m in secret)} + return write_db_overrides(knobs) + + +if __name__ == "__main__": + import sys + if len(sys.argv) > 1 and sys.argv[1] == "seed-from-env": + from dotenv import load_dotenv + load_dotenv() + load() # file overrides first, so the seed captures the effective values + merged = _seed_from_env() + print(f"seeded {len(merged)} knobs into config_overrides") + else: + print("usage: python -m runtime.env_overrides seed-from-env") diff --git a/tests/unit/test_env_overrides_db.py b/tests/unit/test_env_overrides_db.py new file mode 100644 index 0000000..3aa8257 --- /dev/null +++ b/tests/unit/test_env_overrides_db.py @@ -0,0 +1,41 @@ +"""DB-backed config overrides: table CRUD + boot precedence.""" + +import os + +import pytest + +from runtime import env_overrides +from storage.event_store import EventStore + + +@pytest.fixture +def con(): + s = EventStore(":memory:") + yield s.con + s.close() + + +def test_write_read_roundtrip_and_delete(con): + merged = env_overrides.write_db_overrides( + {"TRADING_MAX_ORDERS_PER_TICK": "3", "TRIGGER_RVOL_MIN": "1.5"}, con=con) + assert merged == {"TRADING_MAX_ORDERS_PER_TICK": "3", "TRIGGER_RVOL_MIN": "1.5"} + merged = env_overrides.write_db_overrides( + {"TRIGGER_RVOL_MIN": None, "TRADING_MAX_ORDERS_PER_TICK": "5"}, con=con) + assert merged == {"TRADING_MAX_ORDERS_PER_TICK": "5"} + + +def test_read_missing_table_is_empty(con): + assert env_overrides.read_db_overrides(con=con) == {} + + +def test_load_db_applies_onto_environ(con, monkeypatch): + env_overrides.write_db_overrides({"TRADING_TEST_KNOB_XYZ": "42"}, con=con) + monkeypatch.setattr(env_overrides, "_db_con", lambda: con) + monkeypatch.setenv("TRADING_TEST_KNOB_XYZ", "1") # compose value loses + n = env_overrides.load_db() + assert n == 1 and os.environ["TRADING_TEST_KNOB_XYZ"] == "42" + + +def test_load_db_kill_switch(monkeypatch): + monkeypatch.setenv("ENV_OVERRIDES_DB", "0") + assert env_overrides.load_db() == 0