Local-data cleanup: archive dead DuckDB relics, rotate logs, fix labels

Everything hot already lives in Postgres — local data/ was 25 MB of
pre-migration relics and unbounded logs. Moved the two dead .duckdb
files (untouched since 2026-06-16) to archive/, added nightly log
rotation (copy-truncate >20 MB, safe with systemd append mode), and
updated the last vestigial "market.duckdb" docstrings to say Postgres,
closing the CONSOLIDATION_PLAN cleanup items.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

research/ingestion/market_data.py

@@ -1,4 +1,4 @@
-"""Market data ingestion: Alpaca bars -> research market.duckdb.
+"""Market data ingestion: Alpaca bars -> the research market schema (Postgres).
 
 This module fixes the root cause of "live paper trading never worked":
 the watcher reads *minute* bars from the research database, but the old

research/ingestion/watcher_task.py

@@ -3,7 +3,7 @@
 Two providers implement the WatchlistProvider protocol:
 
 * ResearchWatchlistProvider — reads candidates and bars from the research
-  market.duckdb (populated by research.ingestion.market_data). The price
+  market schema (populated by research.ingestion.market_data). The price
   band is configurable via WATCHER_PRICE_MIN / WATCHER_PRICE_MAX, fixing
   the old hardcoded ``close BETWEEN 1 AND 20`` filter that silently
   excluded every ingested symbol.

research/premarket_eval.py

@@ -103,6 +103,19 @@ def main() -> None:
         print(f"events retention: purged {getattr(cur, 'rowcount', '?')} spam rows >14d")
     except Exception as exc:  # noqa: BLE001 — housekeeping must never break the eval
         print(f"events retention skipped: {exc}")
+    # log rotation: systemd units append to data/*.log with no bound (live_run.log hit 12 MB
+    # in 3 weeks). Copy-then-truncate is safe with O_APPEND writers; keep one generation.
+    try:
+        import glob
+        import shutil
+        for lp in glob.glob(os.path.join(os.path.dirname(__file__), "..", "data", "*.log")):
+            if os.path.getsize(lp) > 20 * 1024 * 1024:
+                shutil.copy2(lp, lp + ".1")
+                with open(lp, "r+") as f:
+                    f.truncate(0)
+                print(f"rotated {os.path.basename(lp)}")
+    except Exception as exc:  # noqa: BLE001
+        print(f"log rotation skipped: {exc}")
     labeler.build_premarket(con, rebuild=True)          # rebuild from the grown DB
     m = _metrics(con)
     ts = datetime.now(timezone.utc).isoformat(timespec="seconds")

research/schema.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 from storage.db import get_connection
 
-DEFAULT_RESEARCH_DB = "data/research/market.duckdb"
+DEFAULT_RESEARCH_DB = "data/research/market.duckdb"  # vestigial label: the PG facade ignores file paths
 
 
 def get_research_connection(db_path: str | Path = DEFAULT_RESEARCH_DB):

research_cli.py

@@ -90,7 +90,7 @@ def main(argv: list[str] | None = None) -> int:
     p_gap.add_argument("--min-gap", type=float, default=5.0)
     p_gap.add_argument("--min-rvol", type=float, default=2.0)
 
-    sub.add_parser("news", help="ingest RSS feeds into news.duckdb")
+    sub.add_parser("news", help="ingest RSS feeds into the news store (Postgres)")
 
     args = parser.parse_args(argv)
     if hasattr(args, "date"):

run_live_paper.py

@@ -2,7 +2,7 @@
 
 This is the piece that was missing: a single loop tying together
 
-  1. ingest    — latest 1-minute Alpaca IEX bars into research market.duckdb
+  1. ingest    — latest 1-minute Alpaca bars into the market schema (Postgres)
                  (plus a daily-bar backfill on startup so gap %% / RVOL work)
   2. watch     — Watcher.tick over the ingested bars, emitting signal_ready
   3. sync      — Alpaca paper account / positions / orders into the event DB