From 5ed038feb41feabe739068ca435def99cae2526d Mon Sep 17 00:00:00 2001 From: momentum dev Date: Thu, 18 Jun 2026 23:24:36 -0400 Subject: [PATCH] Batch bulk DB writes (execute_batch): unfreeze the discovery step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live loop froze ~40s every discovery pass. py-spy showed the main thread stuck in upsert_daily_bars -> PgConnection.executemany, which used psycopg2's cur.executemany — ONE INSERT round-trip per row. A discovery pass upserts thousands of daily bars across the universe, so over the LAN Postgres that's thousands of serial round-trips. While frozen the loop emits nothing and can't service the trigger/exit cadence. Switch executemany to psycopg2.extras.execute_batch (page_size=200): many statements per round-trip, same SQL, same ON CONFLICT upsert semantics. Benchmarked on the live DB: 771-row upsert 1.87s -> 0.06s (~31x); a real discovery pass now finishes in one cycle (ingest 6803 rows, no stall). Helps every bulk writer (daily/minute bars, scanner snapshots). Co-Authored-By: Claude Opus 4.8 --- storage/db_pg.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/storage/db_pg.py b/storage/db_pg.py index 2997717..f139a05 100644 --- a/storage/db_pg.py +++ b/storage/db_pg.py @@ -37,6 +37,7 @@ from typing import Any import pandas as pd import psycopg2 +from psycopg2.extras import execute_batch SCHEMA_SQL_PATH = Path(__file__).with_name("pg_schema.sql") @@ -138,7 +139,11 @@ class PgConnection: def go(): cur = self._conn.cursor() - cur.executemany(_translate(sql), seq) + # execute_batch sends page_size statements per round-trip; plain + # psycopg2 executemany sends ONE per row. Over the remote LAN + # Postgres that was ~771 round-trips for a daily-bar upsert and + # froze the live loop's discovery step for ~40s each pass. + execute_batch(cur, _translate(sql), seq, page_size=200) cur.close() return self return self._run(go)