From 857c64ea4851e7e0b87324ac61932e16659bd398 Mon Sep 17 00:00:00 2001 From: momentum dev Date: Sat, 4 Jul 2026 15:26:40 -0400 Subject: [PATCH] Local-data cleanup: archive dead DuckDB relics, rotate logs, fix labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything hot already lives in Postgres — local data/ was 25 MB of pre-migration relics and unbounded logs. Moved the two dead .duckdb files (untouched since 2026-06-16) to archive/, added nightly log rotation (copy-truncate >20 MB, safe with systemd append mode), and updated the last vestigial "market.duckdb" docstrings to say Postgres, closing the CONSOLIDATION_PLAN cleanup items. Co-Authored-By: Claude Opus 4.8 --- research/ingestion/market_data.py | 2 +- research/ingestion/watcher_task.py | 2 +- research/premarket_eval.py | 13 +++++++++++++ research/schema.py | 2 +- research_cli.py | 2 +- run_live_paper.py | 2 +- 6 files changed, 18 insertions(+), 5 deletions(-) diff --git a/research/ingestion/market_data.py b/research/ingestion/market_data.py index 3db8d7a..64e087f 100644 --- a/research/ingestion/market_data.py +++ b/research/ingestion/market_data.py @@ -1,4 +1,4 @@ -"""Market data ingestion: Alpaca bars -> research market.duckdb. +"""Market data ingestion: Alpaca bars -> the research market schema (Postgres). This module fixes the root cause of "live paper trading never worked": the watcher reads *minute* bars from the research database, but the old diff --git a/research/ingestion/watcher_task.py b/research/ingestion/watcher_task.py index 4921f0b..47d179f 100644 --- a/research/ingestion/watcher_task.py +++ b/research/ingestion/watcher_task.py @@ -3,7 +3,7 @@ Two providers implement the WatchlistProvider protocol: * ResearchWatchlistProvider — reads candidates and bars from the research - market.duckdb (populated by research.ingestion.market_data). The price + market schema (populated by research.ingestion.market_data). The price band is configurable via WATCHER_PRICE_MIN / WATCHER_PRICE_MAX, fixing the old hardcoded ``close BETWEEN 1 AND 20`` filter that silently excluded every ingested symbol. diff --git a/research/premarket_eval.py b/research/premarket_eval.py index 08850bb..b9ad521 100644 --- a/research/premarket_eval.py +++ b/research/premarket_eval.py @@ -103,6 +103,19 @@ def main() -> None: print(f"events retention: purged {getattr(cur, 'rowcount', '?')} spam rows >14d") except Exception as exc: # noqa: BLE001 — housekeeping must never break the eval print(f"events retention skipped: {exc}") + # log rotation: systemd units append to data/*.log with no bound (live_run.log hit 12 MB + # in 3 weeks). Copy-then-truncate is safe with O_APPEND writers; keep one generation. + try: + import glob + import shutil + for lp in glob.glob(os.path.join(os.path.dirname(__file__), "..", "data", "*.log")): + if os.path.getsize(lp) > 20 * 1024 * 1024: + shutil.copy2(lp, lp + ".1") + with open(lp, "r+") as f: + f.truncate(0) + print(f"rotated {os.path.basename(lp)}") + except Exception as exc: # noqa: BLE001 + print(f"log rotation skipped: {exc}") labeler.build_premarket(con, rebuild=True) # rebuild from the grown DB m = _metrics(con) ts = datetime.now(timezone.utc).isoformat(timespec="seconds") diff --git a/research/schema.py b/research/schema.py index 4d56fd0..82e0270 100644 --- a/research/schema.py +++ b/research/schema.py @@ -6,7 +6,7 @@ from pathlib import Path from storage.db import get_connection -DEFAULT_RESEARCH_DB = "data/research/market.duckdb" +DEFAULT_RESEARCH_DB = "data/research/market.duckdb" # vestigial label: the PG facade ignores file paths def get_research_connection(db_path: str | Path = DEFAULT_RESEARCH_DB): diff --git a/research_cli.py b/research_cli.py index ddde484..450d354 100644 --- a/research_cli.py +++ b/research_cli.py @@ -90,7 +90,7 @@ def main(argv: list[str] | None = None) -> int: p_gap.add_argument("--min-gap", type=float, default=5.0) p_gap.add_argument("--min-rvol", type=float, default=2.0) - sub.add_parser("news", help="ingest RSS feeds into news.duckdb") + sub.add_parser("news", help="ingest RSS feeds into the news store (Postgres)") args = parser.parse_args(argv) if hasattr(args, "date"): diff --git a/run_live_paper.py b/run_live_paper.py index 51f9c4a..edaa782 100644 --- a/run_live_paper.py +++ b/run_live_paper.py @@ -2,7 +2,7 @@ This is the piece that was missing: a single loop tying together - 1. ingest — latest 1-minute Alpaca IEX bars into research market.duckdb + 1. ingest — latest 1-minute Alpaca bars into the market schema (Postgres) (plus a daily-bar backfill on startup so gap %% / RVOL work) 2. watch — Watcher.tick over the ingested bars, emitting signal_ready 3. sync — Alpaca paper account / positions / orders into the event DB