diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 45e6cf024..514f6d635 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -296,8 +296,12 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> onInizialized -> Execution finished well."); } /* If the process is configured to run inside a framework, we continue there, otherwise we run the bot directly. */ + if (processConfig.framework === undefined) { + initializeUserBot(); + return; + } - switch (processConfig.framework) { + switch (processConfig.framework.name) { case 'Multi-Period-Market': { processFramework = MULTI_PERIOD_MARKET.newMultiPeriodMarket(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE); intitializeProcessFramework(); @@ -308,7 +312,17 @@ intitializeProcessFramework(); break; } - default: { initializeUserBot();} + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Process Framework not Supported."); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Process Framework Name = " + processConfig.framework.name); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + return; + } } return; } diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index c9c8fe1cf..508590812 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -23,7 +23,7 @@ let usertBot; let currentBotStorage = BLOB_STORAGE.newBlobStorage(bot, logger); - let pProcessConfig; + let processConfig; return thisObject; @@ -91,12 +91,14 @@ let contextVariables = { lastFile: undefined, // Datetime of the last file files sucessfully produced by this process. - dateBeginingOfMarket: undefined, // Datetime of the first trade file in the whole market history. + dateBeginOfMarket: undefined, // Datetime of the first trade file in the whole market history. dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process. }; let previousDay; // Holds the date of the previous day relative to the processing date. - let currentDay; // Holds the processing date. + let currentDay; // Holds the processing date. + + let interExecutionMemoryArray; getContextVariables(); @@ -110,84 +112,103 @@ let reportKey; let statusReport; - /* - We look first for the bot who knows the begining of the marke in order to get when the market starts. - */ + if (processConfig.framework.startDate.fixedDate !== undefined) { - let botWhoKnowsTheBeginingOfTheMarket = statusDependencies.config[processConfig.statusDependenciesIndexBegingOfMarket]; - let botWhoKnowsTheEndOfTheMarket = statusDependencies.config[processConfig.statusDependenciesIndexEndOfMarket]; + /* The starting date is fixed, we will start from there. */ - reportKey = botWhoKnowsTheBeginingOfTheMarket.devTeam + "-" + botWhoKnowsTheBeginingOfTheMarket.bot + "-" + botWhoKnowsTheBeginingOfTheMarket.process + "-" + "dataSet.V1"; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } + contextVariables.dateBeginOfMarket = new Date(processConfig.framework.startDate.fixedDate); - statusReport = statusDependencies.statusReports.get(reportKey); + } else { - if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } + /* + We look first for the bot who knows the begining of the marke in order to get when the market starts. + */ - if (statusReport.status === "Status Report is corrupt.") { - logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } + let botWhoKnowsTheBeginingOfTheMarket = statusDependencies.config[processConfig.framework.startDate.takeItFromStatusDependency]; - thisReport = statusDependencies.statusReports.get(reportKey).file; + reportKey = botWhoKnowsTheBeginingOfTheMarket.devTeam + "-" + botWhoKnowsTheBeginingOfTheMarket.bot + "-" + botWhoKnowsTheBeginingOfTheMarket.process + "-" + "dataSet.V1"; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } - if (thisReport.lastFile === undefined) { - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); - logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet."); + statusReport = statusDependencies.statusReports.get(reportKey); - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Dependency does not exist." + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; } - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); - callBackFunction(customOK); - return; - } - contextVariables.dateBeginingOfMarket = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS); - //contextVariables.dateBeginingOfMarket = new Date("2018-01-01"); // TODO: Remove this temporaty patch. - - /* - Here we get the status report from the bot who knows which is the end of the market. - */ - - reportKey = botWhoKnowsTheEndOfTheMarket.devTeam + "-" + botWhoKnowsTheEndOfTheMarket.bot + "-" + botWhoKnowsTheEndOfTheMarket.process + "-" + "dataSet.V1"; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } - - statusReport = statusDependencies.statusReports.get(reportKey); - - if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - if (statusReport.status === "Status Report is corrupt.") { - logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - thisReport = statusDependencies.statusReports.get(reportKey).file; - - if (thisReport.lastFile === undefined) { - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); - - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Dependency not ready." + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; } - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); - callBackFunction(customOK); - return; + + thisReport = statusDependencies.statusReports.get(reportKey).file; + + if (thisReport.lastFile === undefined) { + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); + logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet."); + + let customOK = { + result: global.CUSTOM_OK_RESPONSE.result, + message: "Dependency does not exist." + } + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); + callBackFunction(customOK); + return; + } + + contextVariables.dateBeginOfMarket = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS); + } - contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf()); + if (processConfig.framework.endDate.fixedDate !== undefined) { + + /* The ending date is fixed, we will end there. */ + + contextVariables.dateEndOfMarket = new Date(processConfig.framework.endDate.fixedDate); + + } else { + + /* + Here we get the status report from the bot who knows which is the end of the market. + */ + + let botWhoKnowsTheEndOfTheMarket = statusDependencies.config[processConfig.framework.endDate.takeItFromStatusDependency]; + + reportKey = botWhoKnowsTheEndOfTheMarket.devTeam + "-" + botWhoKnowsTheEndOfTheMarket.bot + "-" + botWhoKnowsTheEndOfTheMarket.process + "-" + "dataSet.V1"; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } + + statusReport = statusDependencies.statusReports.get(reportKey); + + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + thisReport = statusDependencies.statusReports.get(reportKey).file; + + if (thisReport.lastFile === undefined) { + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); + + let customOK = { + result: global.CUSTOM_OK_RESPONSE.result, + message: "Dependency not ready." + } + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); + callBackFunction(customOK); + return; + } + + contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf()); + } /* Finally we get our own Status Report. */ @@ -214,7 +235,7 @@ contextVariables.lastFile = new Date(thisReport.lastFile); - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> thisReport.lastFile !== undefined"); } + interExecutionMemoryArray = thisReport.interExecutionMemoryArray; processTimePeriods(); return; @@ -222,16 +243,28 @@ } else { /* - In the case when there is no status report, we take the date of the file with the first trades as the begining of the market. Then we will - go one day further in time, so that the previous day does fine a file at the begining of the market. + In the case when there is no status report, we assume like the last processed file is the one on the date of Begining of Market. */ - contextVariables.lastFile = new Date(contextVariables.dateBeginingOfMarket.getUTCFullYear() + "-" + (contextVariables.dateBeginingOfMarket.getUTCMonth() + 1) + "-" + contextVariables.dateBeginingOfMarket.getUTCDate() + " " + "00:00" + GMT_SECONDS); - contextVariables.lastFile = new Date(contextVariables.lastFile.valueOf() + ONE_DAY_IN_MILISECONDS); + contextVariables.lastFile = new Date(contextVariables.dateBeginOfMarket.getUTCFullYear() + "-" + (contextVariables.dateBeginOfMarket.getUTCMonth() + 1) + "-" + contextVariables.dateBeginOfMarket.getUTCDate() + " " + "00:00" + GMT_SECONDS); + //contextVariables.lastFile = new Date(contextVariables.lastFile.valueOf() + ONE_DAY_IN_MILISECONDS); if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> thisReport.lastFile === undefined"); } if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> contextVariables.lastFile = " + contextVariables.lastFile); } + /* + The first time the process is running is the right time to create the data structure that is going to be shared across different executions. + This data structure has one object per each timePeriod. + */ + + interExecutionMemoryArray = []; + + for (let i = 0; i < global.dailyFilePeriods.length; i++) + { + let emptyObject = {}; + interExecutionMemoryArray.push(emptyObject); + } + processTimePeriods(); return; } @@ -342,7 +375,12 @@ let previousFile; let currentFile; - getPreviousFile(); + if (currentDay.valueOf() > contextVariables.dateBeginOfMarket.valueOf()) { + getPreviousFile(); + } else { + previousFile = []; + getCurrentFile() + } function getPreviousFile() { @@ -378,6 +416,14 @@ return; } + if (err.result === "Fail Because" && err.message === "File does not exist.") { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> filePath = " + filePath); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> fileName = " + fileName); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { @@ -424,6 +470,15 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); } if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); } + if (err.result === "Fail Because" && err.message === "File does not exist.") { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> filePath = " + filePath); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> fileName = " + fileName); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.message); @@ -492,7 +547,15 @@ const timePeriod = global.dailyFilePeriods[n][0]; const outputPeriodLabel = global.dailyFilePeriods[n][1]; - usertBot.start(dataFiles, timePeriod, outputPeriodLabel, currentDay, onBotFinished); + usertBot.start( + dataFiles, + timePeriod, + outputPeriodLabel, + currentDay, + contextVariables.dateBeginOfMarket, + contextVariables.dateEndOfMarket, + interExecutionMemoryArray[n], + onBotFinished); function onBotFinished(err) { @@ -592,7 +655,7 @@ let folderName = bot.products[productIndex].codeName; - writeDataRange(contextVariables.dateBeginingOfMarket, currentDay, folderName, controlLoop); + writeDataRange(contextVariables.dateBeginOfMarket, currentDay, folderName, controlLoop); } function controlLoop() { @@ -669,6 +732,7 @@ thisReport.file.lastExecution = bot.currentDaytime; thisReport.file.lastFile = lastFileDate; + thisReport.file.interExecutionMemoryArray = interExecutionMemoryArray; thisReport.save(callBack); } diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index f0e3ffccb..20ade8176 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -23,6 +23,8 @@ let usertBot; + let processConfig; + return thisObject; function initialize(pProcessConfig, pStatusDependencies, pDataDependencies, callBackFunction) { @@ -36,6 +38,7 @@ statusDependencies = pStatusDependencies; dataDependencies = pDataDependencies; + processConfig = pProcessConfig; for (let i = 0; i < dataDependencies.config.length; i++) { @@ -72,6 +75,29 @@ let market = global.MARKET; + /* Context Variables */ + + let contextVariables = { + dateBeginOfMarket: undefined, // Datetime of the first trade file in the whole market history. + dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process. + }; + + if (processConfig.framework.startDate.fixedDate !== undefined) { + + /* The starting date is fixed, we will start from there. */ + + contextVariables.dateBeginOfMarket = new Date(processConfig.framework.startDate.fixedDate); + + } + + if (processConfig.framework.endDate.fixedDate !== undefined) { + + /* The ending date is fixed, we will end there. */ + + contextVariables.dateEndOfMarket = new Date(processConfig.framework.endDate.fixedDate); + + } + processTimePeriods(); function processTimePeriods() { @@ -218,7 +244,13 @@ const timePeriod = global.marketFilesPeriods[n][0]; const outputPeriodLabel = global.marketFilesPeriods[n][1]; - usertBot.start(dataFiles, timePeriod, outputPeriodLabel, onBotFinished); + usertBot.start( + dataFiles, + timePeriod, + outputPeriodLabel, + contextVariables.dateBeginOfMarket, + contextVariables.dateEndOfMarket, + onBotFinished); function onBotFinished(err) {