From 1f84be75f39cfa85d24b1ba1b4b60bc20eb6dd51 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 26 Aug 2019 17:18:12 +0200 Subject: [PATCH] Multi Period fully tested --- Integrations/FileStorage.js | 4 +- MultiPeriod.js | 422 ++++++++++++++++++++++++++++-------- 2 files changed, 332 insertions(+), 94 deletions(-) diff --git a/Integrations/FileStorage.js b/Integrations/FileStorage.js index a2071deb7..e9c7011cd 100644 --- a/Integrations/FileStorage.js +++ b/Integrations/FileStorage.js @@ -64,7 +64,7 @@ exports.newFileStorage = function newFileStorage(logger) { } logInfo('getTextFile: ' + fileLocation) - console.log('getTextFile: ' + fileLocation) + //console.log('getTextFile: ' + fileLocation) let fileContent = await readFileAsync(fileLocation) callBackFunction(global.DEFAULT_OK_RESPONSE, fileContent.toString()) @@ -130,7 +130,7 @@ exports.newFileStorage = function newFileStorage(logger) { } logInfo('createTextFile: ' + fileLocation) - console.log('createTextFile: ' + fileLocation) + //console.log('createTextFile: ' + fileLocation) let directoryPath = fileLocation.substring(0, fileLocation.lastIndexOf('/') + 1); mkDirByPathSync(directoryPath) diff --git a/MultiPeriod.js b/MultiPeriod.js index d92808f4e..fa414ff94 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -60,7 +60,7 @@ } usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, fileStorage); - usertBot.initialize(dataDependencies, callBackFunction, pAssistant); + usertBot.initialize(callBackFunction, pAssistant); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); @@ -78,6 +78,7 @@ let currentOutputPeriodLabel let market = global.MARKET; + let botNeverRan = true; /* Context Variables */ @@ -89,7 +90,7 @@ let previousDay; // Holds the date of the previous day relative to the processing date. - let interExecutionMemoryArray; + let interExecutionMemoryArray = []; getContextVariables(); @@ -235,7 +236,7 @@ contextVariables.lastFile = new Date(thisReport.lastFile); interExecutionMemoryArray = thisReport.interExecutionMemoryArray; - processTimePeriods(); + processTimePeriodsMarketFiles(); return; } else { @@ -266,7 +267,7 @@ interExecutionMemoryArray.push(emptyObject); } - processTimePeriods(); + processTimePeriodsMarketFiles(); } @@ -280,62 +281,21 @@ } } - function processTimePeriods() { + function processTimePeriodsMarketFiles() { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> Entering function."); } try { let n; - let botNeverRan = true; - bot.multiPeriodDailyProcessDatetime = new Date(contextVariables.lastFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. - - advanceTime(); - - function advanceTime() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> Entering function."); } - - bot.multiPeriodDailyProcessDatetime = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() + ONE_DAY_IN_MILISECONDS); - previousDay = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() - ONE_DAY_IN_MILISECONDS); - - console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " " + " bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime.toISOString()); - - if (global.FULL_LOG === 'true') { - logger.newInternalLoop(bot.codeName, bot.process); - } - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> previousDay = " + previousDay); } - - /* Validation that we are not going past the head of the market. */ - - if (bot.multiPeriodDailyProcessDatetime.valueOf() > contextVariables.dateEndOfMarket.valueOf()) { - - const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> " + logText); } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - - } - - periodsLoop(); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> advanceTime -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } + periodsLoop(); function periodsLoop() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoop -> Entering function."); } /* @@ -349,7 +309,7 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -358,7 +318,239 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> Entering function."); } + + const timePeriod = global.marketFilesPeriods[n][0]; + const outputPeriodLabel = global.marketFilesPeriods[n][1]; + + let dependencyIndex = 0; + dataFiles = []; + + let timePeriodFilter = global.EXECUTION_CONFIG.timePeriodFilter + if (timePeriodFilter !== undefined) { + if (timePeriodFilter.indexOf(outputPeriodLabel) > -1) { + currentTimePeriod = global.marketFilesPeriods[n][0]; + currentOutputPeriodLabel = global.marketFilesPeriods[n][1]; + } + } + + dependencyLoopBody(); + + function dependencyLoopBody() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } + + let dependency = dataDependencies.config[dependencyIndex]; + let storage = storages[dependencyIndex]; + + getFile(); + + function getFile() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> Entering function."); } + + let fileName = market.assetA + '_' + market.assetB + ".json"; + + if (dependency.dataSet !== "Multi-Period-Market") { + + dependencyControlLoop(); + return + } + + let filePath = dependency.product + '/' + dependency.dataSet + "/" + outputPeriodLabel; + + storage.getTextFile(filePath, fileName, onFileReceived); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); + callBackFunction(err); + return; + } + + let dataFile = JSON.parse(text); + dataFiles.push(dataFile); + + dependencyControlLoop(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoop -> dependencyLoopBody -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function dependencyControlLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> dependencyControlLoop -> Entering function."); } + + dependencyIndex++; + + if (dependencyIndex < dataDependencies.config.length) { + + dependencyLoopBody(); + + } else { + + let mapKey = global.marketFilesPeriods[n][1]; + multiPeriodDataFiles.set(mapKey, dataFiles) + + periodsControlLoop(); + + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> dependencyControlLoop -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsControlLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsControlLoop -> Entering function."); } + + n++; + + if (n < global.marketFilesPeriods.length) { + + periodsLoopBody(); + + } else { + + processTimePeriodsDailyFiles(); + + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsControlLoop -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function processTimePeriodsDailyFiles() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> Entering function."); } + + try { + + let n = 0; + + if (currentTimePeriod) { + callTheBot() + return + } + + bot.multiPeriodDailyProcessDatetime = new Date(contextVariables.lastFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. + + advanceTime(); + + function advanceTime() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> advanceTime -> Entering function."); } + + bot.multiPeriodDailyProcessDatetime = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() + ONE_DAY_IN_MILISECONDS); + previousDay = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() - ONE_DAY_IN_MILISECONDS); + + console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " " + " bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime.toISOString()); + + if (global.FULL_LOG === 'true') { + logger.newInternalLoop(bot.codeName, bot.process); + } + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> advanceTime -> bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> advanceTime -> previousDay = " + previousDay); } + + /* Validation that we are not going past the head of the market. */ + + if (bot.multiPeriodDailyProcessDatetime.valueOf() > contextVariables.dateEndOfMarket.valueOf()) { + + const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> advanceTime -> " + logText); } + + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + + } + + periodsLoop(); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> advanceTime -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoop -> Entering function."); } + + /* + + We will iterate through all posible periods. + + */ + + n = 0 // loop Variable representing each possible period as defined at the periods array. + + periodsLoopBody(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoop -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsLoopBody() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> Entering function."); } const timePeriod = global.dailyFilePeriods[n][0]; const outputPeriodLabel = global.dailyFilePeriods[n][1]; @@ -370,7 +562,7 @@ if (period === outputPeriodLabel) { validPeriod = true } } if (validPeriod === false) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Discarding period for not being listed as a valid period. -> outputPeriodLabel = " + outputPeriodLabel); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> Discarding period for not being listed as a valid period. -> outputPeriodLabel = " + outputPeriodLabel); } periodsControlLoop(); return; } @@ -378,7 +570,7 @@ let timePeriodFilter = global.EXECUTION_CONFIG.timePeriodFilter if (timePeriodFilter !== undefined) { - if (timePeriodFilter.indexOf(outputPeriodLabel) > 0 -1) { + if (timePeriodFilter.indexOf(outputPeriodLabel) > -1) { currentTimePeriod = global.dailyFilePeriods[n][0]; currentOutputPeriodLabel = global.dailyFilePeriods[n][1]; } @@ -393,7 +585,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } let dependency = dataDependencies.config[dependencyIndex]; let storage = storages[dependencyIndex]; @@ -412,10 +604,17 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } + + if (dependency.dataSet === "Multi-Period-Market") { + + dependencyControlLoop(); + return + } let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2); let filePath + if (dependency.dataSet === "Multi-Period-Daily") { filePath = dependency.product + '/' + dependency.dataSet + "/" + outputPeriodLabel + "/" + dateForPath; } else { @@ -429,8 +628,8 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); } if ((err.message === "File does not exist." && botNeverRan === true) || err.code === 'The specified key does not exist.') { @@ -441,7 +640,7 @@ After that, we will not accept more missing files on any of the dependencies, and if any is missing we will abort the processing. */ - logger.write(MODULE_NAME, "[WARN] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found."); + logger.write(MODULE_NAME, "[WARN] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found."); advanceTime(); return; @@ -449,14 +648,14 @@ if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_RETRY_RESPONSE); return; } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); callBackFunction(err); return; } @@ -467,13 +666,13 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -482,7 +681,13 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } + + if (dependency.dataSet === "Multi-Period-Market") { + + dependencyControlLoop(); + return + } let dateForPath = bot.multiPeriodDailyProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCDate(), 2); let filePath @@ -499,21 +704,21 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); } if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> filePath = " + filePath); - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> fileName = " + fileName); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> filePath = " + filePath); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> fileName = " + fileName); callBackFunction(global.DEFAULT_RETRY_RESPONSE); return; } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); callBackFunction(err); return; } @@ -528,19 +733,19 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoop -> dependencyLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoop -> dependencyLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -549,7 +754,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> dependencyControlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> dependencyControlLoop -> Entering function."); } dependencyIndex++; @@ -567,14 +772,14 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> dependencyControlLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> dependencyControlLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -583,7 +788,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsControlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsControlLoop -> Entering function."); } n++; @@ -600,7 +805,7 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsControlLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsControlLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -609,7 +814,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> callTheBot -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> callTheBot -> Entering function."); } usertBot.start( multiPeriodDataFiles, @@ -625,52 +830,65 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> callTheBot -> onBotFinished -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> Entering function."); } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - callBackFunction(err); return; } botNeverRan = false; - writeDataRanges(onWritten); + if (currentTimePeriod > global.dailyFilePeriods[0][0]) { + writeMarketStatusReport(onMarketStatusReport); + } else { + writeDataRanges(onWritten); + } function onWritten(err) { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> callTheBot -> onBotFinished -> onWritten -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onWritten -> Entering function."); } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); callBackFunction(err); return; } - writeStatusReport(bot.multiPeriodDailyProcessDatetime, advanceTime); + writeDailyStatusReport(bot.multiPeriodDailyProcessDatetime, advanceTime); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function onMarketStatusReport() { + try { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> Entering function."); } + callBackFunction(global.DEFAULT_OK_RESPONSE); + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> onBotFinished -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> callTheBot -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -750,10 +968,10 @@ } } - function writeStatusReport(lastFileDate, callBack) { + function writeDailyStatusReport(lastFileDate, callBack) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> Entering function."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> lastFileDate = " + lastFileDate); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDailyStatusReport -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDailyStatusReport -> lastFileDate = " + lastFileDate); } try { @@ -768,10 +986,30 @@ bot.hasTheBotJustStarted = false; } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> writeDailyStatusReport -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } + + function writeMarketStatusReport(callBack) { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeMarketStatusReport -> Entering function."); } + + try { + + let reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period" + "-" + "dataSet.V1"; + let thisReport = statusDependencies.statusReports.get(reportKey); + + thisReport.file.lastExecution = bot.processDatetime; + thisReport.save(callBack); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> writeMarketStatusReport -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } catch (err) {