From 9db1ed112fdcac762fda87b94be58855ee2701aa Mon Sep 17 00:00:00 2001 From: KoalaTrapLord <265133236+KoalaTrapLord@users.noreply.github.com> Date: Thu, 19 Mar 2026 06:30:03 -0400 Subject: [PATCH] Increase _STALL_LIMIT from 300s to 600s (10 min) to prevent premature batch kills --- services/forge_runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/forge_runner.py b/services/forge_runner.py index 1b7eb4e..ee1a358 100644 --- a/services/forge_runner.py +++ b/services/forge_runner.py @@ -26,7 +26,7 @@ _deepseek_lock = threading.Lock() _watchdog_registry: dict[str, dict] = {} # batch_id -> {state, proc, last_activity} _watchdog_lock = threading.Lock() _watchdog_started = False -_STALL_LIMIT = 300 # 5 min +_STALL_LIMIT = 600 # 10 min def _global_watchdog_loop(): """Single daemon thread that checks all active batches for stalls.""" @@ -44,7 +44,7 @@ def _global_watchdog_loop(): entry["state"].log_lines.append("[WATCHDOG] Process stalled. Killed.") proc.kill() entry["state"].running = False - entry["state"].error = "Stall detected (5 min no output)" + entry["state"].error = "Stall detected (10 min no output)" finished.append(bid) for bid in finished: del _watchdog_registry[bid] @@ -518,4 +518,4 @@ def _run_deepseek_batch_thread( state.error = str(e) state.running = False state.log_lines.append(f'[DeepSeek Batch] FATAL: {e}') - traceback.print_exc() \ No newline at end of file + traceback.print_exc()