operator-plane: guarded command bus (flatten / pause kill switch)
Adds OperatorCommandEvent + a closed command vocabulary (flatten_all, flatten_symbol, pause_entries, resume_entries). POST /api/command emits it (token-required: 503 when DASHBOARD_TOKEN unset — commands never run unauthenticated); the live loop's new 5s "command" step consumes them via OperatorCommandConsumer with three replay-safety layers (in-session nonce set, persisted operator_command_done reload, 10-min staleness window). flatten_all routes through close_session DIRECTLY (never step_eod_flatten) so it cannot trip plan 003's daily-restart exit. Pause is checked at the top of BOTH request_approvals_for_ready_signals AND submit_breakout_now (missing either leaks pause on that path). qabot gains an authorized-chat-only /flatten slash command (prose "flatten" still refuses via _ACTION_WORDS). Snapshot exposes broker truth + command_bus + a defensive experiment ledger. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fbb84ae166
commit
e7c34f298f
8 files changed
+619
No files matched your search
@@ -0,0 +1,109 @@
|
||||
"""POST /api/command — token gate, enum validation, event emission (plan 007).
|
||||
|
||||
Drives DashboardHandler.do_POST directly (no loopback socket), against a real
|
||||
DashboardState pinned to an isolated Postgres schema so emit_command actually
|
||||
lands an event."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import psycopg2
|
||||
import pytest
|
||||
|
||||
from api.main import DashboardHandler, DashboardState
|
||||
from storage.event_schema import EventType
|
||||
from storage.event_store import EventStore
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolated_pg_schema(monkeypatch):
|
||||
schema = f"mem_{uuid.uuid4().hex[:12]}"
|
||||
monkeypatch.setenv("MOMENTUM_PG_SCHEMA", schema)
|
||||
yield
|
||||
url = os.environ.get("DATABASE_URL")
|
||||
if url:
|
||||
conn = psycopg2.connect(url)
|
||||
conn.autocommit = True
|
||||
conn.cursor().execute(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE')
|
||||
conn.close()
|
||||
|
||||
|
||||
class CaptureHandler(DashboardHandler):
|
||||
def __init__(self, state, path, body=b"", headers=None):
|
||||
self.state = state
|
||||
self.path = path
|
||||
self.rfile = io.BytesIO(body)
|
||||
self.headers = headers or {}
|
||||
self.captured = []
|
||||
|
||||
def _send_json(self, payload, status=200):
|
||||
self.captured.append((status, payload))
|
||||
|
||||
|
||||
def _post(state, body, headers=None):
|
||||
raw = json.dumps(body).encode()
|
||||
hdrs = {"Content-Length": str(len(raw))}
|
||||
if headers:
|
||||
hdrs.update(headers)
|
||||
h = CaptureHandler(state, "/api/command", raw, hdrs)
|
||||
h.do_POST()
|
||||
return h.captured[-1]
|
||||
|
||||
|
||||
def test_command_503_when_token_unset(monkeypatch):
|
||||
monkeypatch.delenv("DASHBOARD_TOKEN", raising=False)
|
||||
status, payload = _post(DashboardState("momentum"), {"command": "flatten_all"})
|
||||
assert status == 503
|
||||
assert payload["ok"] is False
|
||||
|
||||
|
||||
def test_command_unknown_rejected(monkeypatch):
|
||||
monkeypatch.setenv("DASHBOARD_TOKEN", "s3cret")
|
||||
status, payload = _post(DashboardState("momentum"), {"command": "sell_everything"},
|
||||
headers={"X-Dashboard-Token": "s3cret"})
|
||||
assert status == 400
|
||||
assert "unknown command" in payload["error"]
|
||||
|
||||
|
||||
def test_command_flatten_symbol_requires_symbol(monkeypatch):
|
||||
monkeypatch.setenv("DASHBOARD_TOKEN", "s3cret")
|
||||
status, payload = _post(DashboardState("momentum"), {"command": "flatten_symbol"},
|
||||
headers={"X-Dashboard-Token": "s3cret"})
|
||||
assert status == 400
|
||||
|
||||
|
||||
def test_command_bad_token_401(monkeypatch):
|
||||
monkeypatch.setenv("DASHBOARD_TOKEN", "s3cret")
|
||||
status, _ = _post(DashboardState("momentum"), {"command": "flatten_all"},
|
||||
headers={"X-Dashboard-Token": "wrong"})
|
||||
assert status == 401
|
||||
|
||||
|
||||
def test_command_valid_emits_event(monkeypatch):
|
||||
monkeypatch.setenv("DASHBOARD_TOKEN", "s3cret")
|
||||
state = DashboardState("momentum")
|
||||
status, payload = _post(state, {"command": "flatten_all"},
|
||||
headers={"X-Dashboard-Token": "s3cret"})
|
||||
assert status == 200 and payload["ok"] is True and payload["nonce"]
|
||||
store = EventStore("momentum")
|
||||
try:
|
||||
events = store.query_events(event_type=EventType.OPERATOR_COMMAND.value, limit=None)
|
||||
assert len(events) == 1
|
||||
assert json.loads(events[0]["payload_json"])["command"] == "flatten_all"
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def test_command_emits_even_without_execution_service(monkeypatch):
|
||||
# the standalone dashboard (execution_service=None) is a thin emitter
|
||||
monkeypatch.setenv("DASHBOARD_TOKEN", "s3cret")
|
||||
state = DashboardState("momentum", execution_service=None)
|
||||
status, payload = _post(state, {"command": "pause_entries"},
|
||||
headers={"X-Dashboard-Token": "s3cret"})
|
||||
assert status == 200 and payload["ok"] is True
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
pytest.main([__file__, "-q"])
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Operator command bus: consumer dedup/staleness, pause on BOTH entry paths,
|
||||
flatten-does-not-self-restart, and the qabot slash-command carve-out (plan 007)."""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from runtime.operator_commands import OperatorCommandConsumer
|
||||
from storage.event_schema import EventMode, EventType, OperatorCommandEvent
|
||||
from storage.event_store import EventStore
|
||||
from tests.unit.test_trading_execution import FakeExecutor, emit_signal, make_service
|
||||
|
||||
|
||||
class FakeExecution:
|
||||
def __init__(self):
|
||||
self.closed = []
|
||||
self.flattened = []
|
||||
self.paused = None
|
||||
|
||||
def close_session(self, reason="eod_flatten"):
|
||||
self.closed.append(reason)
|
||||
return {"closed_positions": ["AAA"], "cancelled_entries": 0, "errors": []}
|
||||
|
||||
def flatten_symbol(self, symbol, reason="operator_flatten_symbol"):
|
||||
self.flattened.append(symbol)
|
||||
return {"ok": True, "symbol": symbol}
|
||||
|
||||
def set_operator_paused(self, paused):
|
||||
self.paused = bool(paused)
|
||||
|
||||
|
||||
def _emit_command(store, command, symbol=None, when=None, nonce=None):
|
||||
store.emit(OperatorCommandEvent(
|
||||
timestamp=when or datetime.now(), mode=EventMode.PAPER,
|
||||
message=f"cmd {command}", command=command, symbol=symbol,
|
||||
requested_by="test", nonce=nonce or uuid.uuid4().hex))
|
||||
|
||||
|
||||
def test_command_consumed_once_nonce_dedup():
|
||||
store = EventStore(":memory:")
|
||||
ex = FakeExecution()
|
||||
_emit_command(store, "flatten_all")
|
||||
consumer = OperatorCommandConsumer(store, ex)
|
||||
assert len(consumer.poll()) == 1
|
||||
assert ex.closed == ["operator_flatten"]
|
||||
# second poll: already-consumed nonce -> no re-execution
|
||||
assert consumer.poll() == []
|
||||
assert ex.closed == ["operator_flatten"]
|
||||
store.close()
|
||||
|
||||
|
||||
def test_done_event_prevents_reexecution_after_restart():
|
||||
store = EventStore(":memory:")
|
||||
_emit_command(store, "flatten_all")
|
||||
OperatorCommandConsumer(store, FakeExecution()).poll() # process A executes + echoes _done
|
||||
ex2 = FakeExecution()
|
||||
# process B boots fresh, reloads the _done nonce, must NOT re-run
|
||||
assert OperatorCommandConsumer(store, ex2).poll() == []
|
||||
assert ex2.closed == []
|
||||
store.close()
|
||||
|
||||
|
||||
def test_stale_command_ignored_on_boot():
|
||||
store = EventStore(":memory:")
|
||||
ex = FakeExecution()
|
||||
_emit_command(store, "flatten_all", when=datetime.now() - timedelta(minutes=20))
|
||||
_emit_command(store, "flatten_symbol", symbol="AAA") # fresh
|
||||
consumer = OperatorCommandConsumer(store, ex, stale_seconds=600)
|
||||
consumer.poll()
|
||||
assert ex.closed == [] # the 20-min-old flatten_all was NOT run
|
||||
assert ex.flattened == ["AAA"] # the fresh one was
|
||||
store.close()
|
||||
|
||||
|
||||
def test_flatten_all_routes_through_close_session_not_self_restart():
|
||||
import run_live_paper
|
||||
run_live_paper._clean_exit_requested.clear()
|
||||
store = EventStore(":memory:")
|
||||
ex = FakeExecution()
|
||||
_emit_command(store, "flatten_all")
|
||||
OperatorCommandConsumer(store, ex).poll()
|
||||
assert ex.closed == ["operator_flatten"]
|
||||
# the operator flatten must NOT trip plan 003's daily-restart exit
|
||||
assert not run_live_paper._clean_exit_requested.is_set()
|
||||
store.close()
|
||||
|
||||
|
||||
def test_pause_resume_dispatch():
|
||||
store = EventStore(":memory:")
|
||||
ex = FakeExecution()
|
||||
_emit_command(store, "pause_entries")
|
||||
OperatorCommandConsumer(store, ex).poll()
|
||||
assert ex.paused is True
|
||||
_emit_command(store, "resume_entries")
|
||||
OperatorCommandConsumer(store, ex).poll()
|
||||
assert ex.paused is False
|
||||
store.close()
|
||||
|
||||
|
||||
def test_done_echo_records_result():
|
||||
store = EventStore(":memory:")
|
||||
_emit_command(store, "flatten_symbol", symbol="BBB")
|
||||
OperatorCommandConsumer(store, FakeExecution()).poll()
|
||||
done = store.query_events(event_type=EventType.OPERATOR_COMMAND_DONE.value, limit=None)
|
||||
assert len(done) == 1
|
||||
|
||||
|
||||
# ---- pause blocks BOTH entry paths (the drift class plan 005 warns about) ----
|
||||
def test_pause_blocks_both_entry_paths():
|
||||
store = EventStore(":memory:")
|
||||
emit_signal(store) # a would-otherwise-pass ready signal
|
||||
svc = make_service(store, FakeExecutor())
|
||||
svc.set_operator_paused(True)
|
||||
assert svc.request_approvals_for_ready_signals() == [] # SLOW path blocked
|
||||
fast = svc.submit_breakout_now("GOOD", 14.0, 13.45, last_price=14.1)
|
||||
assert fast.get("skipped") == "paused" # FAST path blocked
|
||||
# resume re-enables the slow path (the same signal now creates a request)
|
||||
svc.set_operator_paused(False)
|
||||
assert len(svc.request_approvals_for_ready_signals()) == 1
|
||||
store.close()
|
||||
|
||||
|
||||
# ---- qabot slash-command carve-out + auth ----
|
||||
def test_qabot_parse_flatten_command():
|
||||
from telegram_bot import parse_flatten_command
|
||||
assert parse_flatten_command("/flatten") == ("flatten_all", None)
|
||||
assert parse_flatten_command("/flatten nvda") == ("flatten_symbol", "NVDA")
|
||||
assert parse_flatten_command("please flatten everything") is None # prose still refuses
|
||||
assert parse_flatten_command("how did we do?") is None
|
||||
|
||||
|
||||
def test_qabot_unauthorized_chat_refused():
|
||||
from telegram_bot import is_authorized
|
||||
assert is_authorized("999", "123") is False
|
||||
assert is_authorized("123", "123") is True
|
||||
assert is_authorized("123", None) is False # fail closed when unconfigured
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
import pytest
|
||||
pytest.main([__file__, "-q"])
|
||||
Reference in new issue
Block a user