Increase _STALL_LIMIT from 300s to 600s (10 min) to prevent premature batch kills
This commit is contained in:
1 parent
75568bfdb1
commit
9db1ed112f
1 file changed
+3
-3
@@ -26,7 +26,7 @@ _deepseek_lock = threading.Lock()
|
||||
_watchdog_registry: dict[str, dict] = {} # batch_id -> {state, proc, last_activity}
|
||||
_watchdog_lock = threading.Lock()
|
||||
_watchdog_started = False
|
||||
_STALL_LIMIT = 300 # 5 min
|
||||
_STALL_LIMIT = 600 # 10 min
|
||||
|
||||
def _global_watchdog_loop():
|
||||
"""Single daemon thread that checks all active batches for stalls."""
|
||||
@@ -44,7 +44,7 @@ def _global_watchdog_loop():
|
||||
entry["state"].log_lines.append("[WATCHDOG] Process stalled. Killed.")
|
||||
proc.kill()
|
||||
entry["state"].running = False
|
||||
entry["state"].error = "Stall detected (5 min no output)"
|
||||
entry["state"].error = "Stall detected (10 min no output)"
|
||||
finished.append(bid)
|
||||
for bid in finished:
|
||||
del _watchdog_registry[bid]
|
||||
@@ -518,4 +518,4 @@ def _run_deepseek_batch_thread(
|
||||
state.error = str(e)
|
||||
state.running = False
|
||||
state.log_lines.append(f'[DeepSeek Batch] FATAL: {e}')
|
||||
traceback.print_exc()
|
||||
traceback.print_exc()
|
||||
Reference in new issue
Block a user