HealthCheck Improved
This commit is contained in:
Luis Fernando Molina committed
1 parent
8d613699ef
commit
47f4ab28b9
4 files changed
+308
-211
No files matched your search
@@ -12,7 +12,9 @@
|
||||
|
||||
const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog');
|
||||
let logger; // We need this here in order for the loopHealth function to work and be able to rescue the loop when it gets in trouble.
|
||||
let timeoutHandle;
|
||||
|
||||
let nextLoopTimeoutHandle;
|
||||
let checkLoopHealthHandle;
|
||||
|
||||
let thisObject = {
|
||||
initialize: initialize,
|
||||
@@ -128,11 +130,11 @@
|
||||
try {
|
||||
if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] run -> Entering function."); }
|
||||
|
||||
let intervalHandle;
|
||||
let fixedTimeLoopIntervalHandle;
|
||||
|
||||
if (bot.runAtFixedInterval === true) {
|
||||
|
||||
intervalHandle = setInterval(loop, bot.fixedInterval);
|
||||
fixedTimeLoopIntervalHandle = setInterval(loop, bot.fixedInterval);
|
||||
loop(); // First run.
|
||||
|
||||
} else {
|
||||
@@ -214,8 +216,9 @@
|
||||
case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle.
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Operation Failed. Aborting the process.");
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
@@ -224,8 +227,9 @@
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Unhandled err.result received. -> err.message = " + err.message);
|
||||
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
return;
|
||||
}
|
||||
@@ -234,8 +238,9 @@
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> err = " + err.message);
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
}
|
||||
}
|
||||
@@ -243,8 +248,9 @@
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> err = " + err.message);
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
}
|
||||
}
|
||||
@@ -280,8 +286,9 @@
|
||||
case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle.
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized > Operation Failed. Aborting the process.");
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
@@ -291,24 +298,27 @@
|
||||
case "Too far in the future.": {
|
||||
logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeUserBot -> onInizialized > Too far in the future. This Loop will enter in coma.");
|
||||
nextWaitTime = 'Coma';
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
loopControl(nextWaitTime);
|
||||
return;
|
||||
}
|
||||
case "Not needed now, but soon.": {
|
||||
logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeUserBot -> onInizialized > Not needed now, but soon. This Loop will continue with Normal wait time.");
|
||||
nextWaitTime = 'Normal';
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
loopControl(nextWaitTime);
|
||||
return;
|
||||
}
|
||||
default: {
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized > Unhandled custom response received. -> err.message = " + err.message);
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
@@ -319,8 +329,9 @@
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized > Unhandled err.result received. -> err.message = " + err.message);
|
||||
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
return;
|
||||
}
|
||||
@@ -329,8 +340,9 @@
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized -> err = " + err.message);
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
}
|
||||
}
|
||||
@@ -338,8 +350,9 @@
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> err = " + err.message);
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
}
|
||||
}
|
||||
@@ -374,8 +387,9 @@
|
||||
case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle.
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished > Operation Failed. Aborting the process.");
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
@@ -397,32 +411,36 @@
|
||||
case "Month before it is needed.": {
|
||||
logger.write(MODULE_NAME, "[WARN] run -> loop -> startUserBot -> onFinished > Month before it is needed. This Loop will be terminated.");
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
return;
|
||||
}
|
||||
case "Month fully processed.": {
|
||||
logger.write(MODULE_NAME, "[WARN] run -> loop -> startUserBot -> onFinished > Month fully processed. This Loop will be terminated.");
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
return;
|
||||
}
|
||||
case "End of the month reached.": {
|
||||
logger.write(MODULE_NAME, "[WARN] run -> loop -> startUserBot -> onFinished > End of the month reached. This Loop will be terminated.");
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
return;
|
||||
}
|
||||
default: {
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished > Unhandled custom response received. -> err.message = " + err.message);
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
@@ -433,8 +451,9 @@
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished > Unhandled err.result received. -> err.message = " + err.message);
|
||||
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
return;
|
||||
}
|
||||
@@ -443,8 +462,9 @@
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished -> err = " + err.message);
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
}
|
||||
}
|
||||
@@ -452,8 +472,9 @@
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> err = " + err.message);
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
}
|
||||
}
|
||||
@@ -473,8 +494,9 @@
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> onStop -> Stopping the Loop Gracefully. See you next time!"); }
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
return;
|
||||
|
||||
@@ -490,21 +512,21 @@
|
||||
case 'Normal': {
|
||||
if (bot.runAtFixedInterval === true) {
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Fixed Interval Normal exit point reached."); }
|
||||
setTimeout(checkLoopHealth, bot.fixedInterval * 5, bot.loopCounter);
|
||||
checkLoopHealthHandle = setTimeout(checkLoopHealth, bot.fixedInterval * 5, bot.loopCounter);
|
||||
logger.persist();
|
||||
return;
|
||||
} else {
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.normalWaitTime / 1000) + " seconds."); }
|
||||
setTimeout(checkLoopHealth, processConfig.normalWaitTime * 5, bot.loopCounter);
|
||||
timeoutHandle = setTimeout(loop, processConfig.normalWaitTime);
|
||||
checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.normalWaitTime * 5, bot.loopCounter);
|
||||
nextLoopTimeoutHandle = setTimeout(loop, processConfig.normalWaitTime);
|
||||
logger.persist();
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'Retry': {
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.retryWaitTime / 1000) + " seconds."); }
|
||||
setTimeout(checkLoopHealth, processConfig.retryWaitTime * 5, bot.loopCounter);
|
||||
timeoutHandle = setTimeout(loop, processConfig.retryWaitTime);
|
||||
checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.retryWaitTime * 5, bot.loopCounter);
|
||||
nextLoopTimeoutHandle = setTimeout(loop, processConfig.retryWaitTime);
|
||||
logger.persist();
|
||||
}
|
||||
break;
|
||||
@@ -515,7 +537,7 @@
|
||||
return;
|
||||
} else {
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.sleepWaitTime / 60000) + " minutes."); }
|
||||
timeoutHandle = setTimeout(loop, processConfig.sleepWaitTime);
|
||||
nextLoopTimeoutHandle = setTimeout(loop, processConfig.sleepWaitTime);
|
||||
logger.persist();
|
||||
}
|
||||
}
|
||||
@@ -527,7 +549,7 @@
|
||||
return;
|
||||
} else {
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.comaWaitTime / 3600000) + " hours."); }
|
||||
timeoutHandle = setTimeout(loop, processConfig.comaWaitTime);
|
||||
nextLoopTimeoutHandle = setTimeout(loop, processConfig.comaWaitTime);
|
||||
logger.persist();
|
||||
}
|
||||
}
|
||||
@@ -548,7 +570,7 @@
|
||||
console.log((new Date().toISOString() + " [ERROR] run -> loop -> checkLoopHealth -> " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " Loop # " + pad(Number(bot.loopCounter), 5) + " found dead. Resurrecting it now."));
|
||||
|
||||
logger.persist(); // We persist the logs of the failed execution.
|
||||
clearTimeout(timeoutHandle); // We cancel the timeout in case the original loop was still running and schedulled to reexecute.
|
||||
clearTimeout(nextLoopTimeoutHandle); // We cancel the timeout in case the original loop was still running and schedulled to reexecute.
|
||||
loop(); // We restart the loop so that the processing can continue.
|
||||
|
||||
} else {
|
||||
@@ -596,8 +618,9 @@
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onInizialized -> CURRENT_EXECUTION_AT must be either 'Cloud' or 'Browser'.");
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onInizialized -> global.CURRENT_EXECUTION_AT = " + global.CURRENT_EXECUTION_AT);
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
return;
|
||||
}
|
||||
@@ -637,8 +660,9 @@
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> err.message = " + err.message);
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
return;
|
||||
}
|
||||
@@ -649,8 +673,9 @@
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> err.message = " + err.message);
|
||||
logger.persist();
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
return;
|
||||
}
|
||||
@@ -658,8 +683,9 @@
|
||||
|
||||
} catch (err) {
|
||||
parentLogger.write(MODULE_NAME, "[ERROR] run -> loop -> err = " + err.message);
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(err);
|
||||
}
|
||||
}
|
||||
@@ -667,8 +693,9 @@
|
||||
|
||||
catch (err) {
|
||||
parentLogger.write(MODULE_NAME, "[ERROR] run -> err = " + err.message);
|
||||
clearInterval(intervalHandle);
|
||||
clearTimeout(timeoutHandle);
|
||||
clearInterval(fixedTimeLoopIntervalHandle);
|
||||
clearTimeout(nextLoopTimeoutHandle);
|
||||
clearTimeout(checkLoopHealthHandle);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user