From 4ab1b9d86cd18b60c285d19e04e2a5a83641cd61 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 21 May 2021 14:46:18 +0200 Subject: [PATCH] Data Aggregation Functions Library fully implemented and running --- .../API-Data-Fetcher-Bot/FetchingProcess.js | 8 +- .../FromOneMinToMultiPeriodMarket.js | 50 ++++--- .../DataAggregationFunctions.js | 130 +++++++++--------- 3 files changed, 102 insertions(+), 86 deletions(-) diff --git a/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js b/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js index a1f86ea4c..3e6293634 100644 --- a/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js +++ b/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js @@ -650,8 +650,12 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { /* We will need to save this at the Status Report */ contextVariables.lastFile = file.date - - await readDatasetFile("/" + dateForPath) + /* + For One-Min type of datasets, since they are saved as Daily Files, there is no need + to read the already existing content to append to it. In this case is enought to + set the existing content to an empty array. + */ + existingFileContent = "[]" appendToExistingDataset() await saveDatasetFile("/" + dateForPath) diff --git a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodMarket.js b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodMarket.js index 6c9470416..8e89b3480 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodMarket.js +++ b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodMarket.js @@ -24,7 +24,6 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process of the last processed day, and then it will process that full day again adding all the elements found at the current run. */ const MODULE_NAME = "From One Min To Multi Period Market" - const ONE_MIN_DATASET_TYPE = "One-Min" let thisObject = { initialize: initialize, @@ -36,8 +35,7 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process let statusDependenciesModule let dataDependenciesModule - let dataDependencyNode - let outputDatasetNode + let node = {} // Usefull nodes for this module will be stored here. return thisObject; @@ -45,8 +43,9 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process statusDependenciesModule = pStatusDependencies dataDependenciesModule = pDataDependencies - TS.projects.superalgos.functionLibraries.TS.projects.superalgos.functionLibraries.dataAggregationFunctions.checkForKnownConstraints( + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.checkForKnownConstraints( dataDependenciesModule, + node, processIndex, callBackFunction ) @@ -57,8 +56,7 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process statusDependenciesModule = undefined dataDependenciesModule = undefined thisObject = undefined - dataDependencyNode = undefined - outputDatasetNode = undefined + node = undefined } function start(callBackFunction) { @@ -72,7 +70,8 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process beginingOfMarket: undefined // Datetime of the begining of the market. } - TS.projects.superalgos.functionLibraries.TS.projects.superalgos.functionLibraries.dataAggregationFunctions.getContextVariables( + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.getContextVariables( + statusDependenciesModule, contextVariables, loadExistingFiles, buildOutput, @@ -104,7 +103,7 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + - outputDatasetNode.referenceParent.parentNode.config.codeName + "/" + // Product Name + node.outputDataset.referenceParent.parentNode.config.codeName + "/" + // Product Name TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.codeName + "/" + TIME_FRAME_LABEL; filePath += '/' + fileName @@ -176,7 +175,9 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process moveToNextDay() function moveToNextDay() { - TS.projects.superalgos.functionLibraries.TS.projects.superalgos.functionLibraries.dataAggregationFunctions.advanceTime( + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.advanceTime( + fromDate, + lastDate, contextVariables, timeframesLoop, processIndex, @@ -219,8 +220,8 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process for (let i = 0; i < previousElements.length; i++) { let element = {} - for (let j = 0; j < outputDatasetNode.referenceParent.parentNode.record.properties.length; j++) { - let property = outputDatasetNode.referenceParent.parentNode.record.properties[j] + for (let j = 0; j < node.outputDataset.referenceParent.parentNode.record.properties.length; j++) { + let property = node.outputDataset.referenceParent.parentNode.record.properties[j] element[property.config.codeName] = record[j] } @@ -234,10 +235,11 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process allPreviousElements[timeFrameArrayIndex] = [] // erasing these so as not to duplicate them. } - TS.projects.superalgos.functionLibraries.TS.projects.superalgos.functionLibraries.dataAggregationFunctions.nextDependencyDailyFile( + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.nextDependencyDailyFile( contextVariables, - dataDependencyNode, + node, aggregateAndWriteOutputFile, + fileStorage, processIndex, callBackFunction ) @@ -247,9 +249,9 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process Here we call the function that will aggregate all the information at the dependency file into standarized begin-end-elements. */ - TS.projects.superalgos.functionLibraries.TS.projects.superalgos.functionLibraries.dataAggregationFunctions.aggregateFileContent( - outputDatasetNode, - dataDependencyNode, + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.aggregateFileContent( + node, + contextVariables, dependencyDailyFile, outputElements[timeFrameArrayIndex], TIME_FRAME_VALUE @@ -262,16 +264,16 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process Here we will write the contents of the file to disk. */ try { - let fileContent = TS.projects.superalgos.functionLibraries.TS.projects.superalgos.functionLibraries.dataAggregationFunctions.generateOutputFileContent( - outputDatasetNode, + let fileContent = TS.projects.superalgos.functionLibraries.dataAggregationFunctions.generateOutputFileContent( + node, outputElements ) let fileName = 'Data.json'; let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + - outputDatasetNode.referenceParent.parentNode.config.codeName + "/" + - outputDatasetNode.referenceParent.config.codeName + "/" + + node.outputDataset.referenceParent.parentNode.config.codeName + "/" + + node.outputDataset.referenceParent.config.codeName + "/" + TIME_FRAME_LABEL filePath += '/' + fileName @@ -289,7 +291,10 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process } TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> writeOutputFile -> onFileCreated -> Finished with File @ " + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "_" + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName) + "[WARN] start -> writeOutputFile -> onFileCreated -> Finished with File @ " + + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "_" + + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName + ", " + + " filePath = " + filePath + "/" + fileName) callBack() } @@ -309,8 +314,9 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process if (timeFrameArrayIndex < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length) { loopBody() } else { - TS.projects.superalgos.functionLibraries.TS.projects.superalgos.functionLibraries.dataAggregationFunctions.writeStatusReport( + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.writeStatusReport( statusDependenciesModule, + contextVariables, contextVariables.datetimeLastProducedFile, moveToNextDay, processIndex, diff --git a/Projects/Superalgos/TS/Function-Libraries/DataAggregationFunctions.js b/Projects/Superalgos/TS/Function-Libraries/DataAggregationFunctions.js index c829ed9c6..f83cfeab1 100644 --- a/Projects/Superalgos/TS/Function-Libraries/DataAggregationFunctions.js +++ b/Projects/Superalgos/TS/Function-Libraries/DataAggregationFunctions.js @@ -12,17 +12,18 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { getContextVariables: getContextVariables, advanceTime: advanceTime, nextDependencyDailyFile: nextDependencyDailyFile, - generateOutputFileContent: generateOutputFileContent, + generateOutputFileContent: generateOutputFileContent, aggregateFileContent: aggregateFileContent, writeStatusReport: writeStatusReport } return thisObject function checkForKnownConstraints( - dataDependenciesModule, + dataDependenciesModule, + node, processIndex, callBackFunction - ) { + ) { /* This Framework have a few contraints that we are going to check right here. One of them is the fact that it can only accept one data dependency. The @@ -36,11 +37,11 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { */ if (dataDependenciesModule.curatedDependencyNodeArray.length !== 1) { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] initialize -> Validation Check Not Passed -> Expecting only one Data Dependency. Found = " + dataDependenciesModule.curatedDependencyNodeArray.length) + "[ERROR] checkForKnownConstraints -> Validation Check Not Passed -> Expecting only one Data Dependency. Found = " + dataDependenciesModule.curatedDependencyNodeArray.length) callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) return } else { - dataDependencyNode = dataDependenciesModule.curatedDependencyNodeArray[0] + node.dataDependency = dataDependenciesModule.curatedDependencyNodeArray[0] } let outputDatasets = TS.projects.superalgos.utilities.nodeFunctions.nodeBranchToArray( @@ -49,21 +50,22 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { if (outputDatasets.length !== 1) { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] initialize -> Validation Check Not Passed -> Expecting only one Output Dataset. Found = " + outputDatasets.length) + "[ERROR] checkForKnownConstraints -> Validation Check Not Passed -> Expecting only one Output Dataset. Found = " + outputDatasets.length) callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) return } else { - outputDatasetNode = outputDatasets[0] + node.outputDataset = outputDatasets[0] } callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE) } function getContextVariables( + statusDependenciesModule, contextVariables, loadExistingFiles, buildOutput, - advanceTime, + processIndex, callBackFunction ) { try { @@ -82,14 +84,14 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> detectWhereTheMarketBegins-> Status Report does not exist. Retrying Later. ") + "[WARN] getContextVariables -> detectWhereTheMarketBegins-> Status Report does not exist. Retrying Later. ") callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE) return } if (statusReport.status === "Status Report is corrupt.") { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> getContextVariables -> detectWhereTheMarketBegins-> Can not continue because dependecy Status Report is corrupt. ") + "[ERROR] getContextVariables -> detectWhereTheMarketBegins-> Can not continue because dependecy Status Report is corrupt. ") callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE) return } @@ -98,16 +100,16 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { if (thisReport.beginingOfMarket === undefined) { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> detectWhereTheMarketBegins-> Undefined Last File. ") + "[WARN] getContextVariables -> detectWhereTheMarketBegins-> Undefined Last File. ") TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[HINT] start -> getContextVariables -> detectWhereTheMarketBegins-> It is too early too run this process since the trade history of the market is not there yet.") + "[HINT] getContextVariables -> detectWhereTheMarketBegins-> It is too early too run this process since the trade history of the market is not there yet.") let customOK = { result: TS.projects.superalgos.globals.standardResponses.CUSTOM_OK_RESPONSE.result, message: "Dependency does not exist." } TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> detectWhereTheMarketBegins-> customOK = " + customOK.message) + "[WARN] getContextVariables -> detectWhereTheMarketBegins-> customOK = " + customOK.message) callBackFunction(customOK) return } @@ -123,14 +125,14 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> detectWhereTheMarketEnds-> Status Report does not exist. Retrying Later. ") + "[WARN] getContextVariables -> detectWhereTheMarketEnds-> Status Report does not exist. Retrying Later. ") callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE) return } if (statusReport.status === "Status Report is corrupt.") { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> getContextVariables -> detectWhereTheMarketEnds-> Can not continue because dependecy Status Report is corrupt. ") + "[ERROR] getContextVariables -> detectWhereTheMarketEnds-> Can not continue because dependecy Status Report is corrupt. ") callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE) return } @@ -139,14 +141,14 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { if (thisReport.lastFile === undefined) { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> detectWhereTheMarketEnds-> Undefined Last File.") + "[WARN] getContextVariables -> detectWhereTheMarketEnds-> Undefined Last File.") let customOK = { result: TS.projects.superalgos.globals.standardResponses.CUSTOM_OK_RESPONSE.result, message: "Dependency not ready." } TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> detectWhereTheMarketEnds-> customOK = " + customOK.message) + "[WARN] getContextVariables -> detectWhereTheMarketEnds-> customOK = " + customOK.message) callBackFunction(customOK) return } @@ -163,14 +165,14 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ") + "[WARN] getContextVariables -> Status Report does not exist. Retrying Later. ") callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE) return } if (statusReport.status === "Status Report is corrupt.") { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process.") + "[ERROR] getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process.") callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) return } @@ -188,7 +190,7 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { if (contextVariables.beginingOfMarket.valueOf() !== contextVariables.datetimeBeginingOfMarketFile.valueOf()) { // Reset Mechanism for Begining of the Market TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> getContextVariables -> getOwnStatusReport-> Reset Mechanism for Begining of the Market Activated.") + "[INFO] getContextVariables -> getOwnStatusReport-> Reset Mechanism for Begining of the Market Activated.") contextVariables.beginingOfMarket = new Date( contextVariables.datetimeBeginingOfMarketFile.getUTCFullYear() + "-" + @@ -245,20 +247,22 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { } catch (err) { TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> getContextVariables -> getOwnStatusReport-> err = " + err.stack) + "[ERROR] getContextVariables -> getOwnStatusReport-> err = " + err.stack) if (err.message === "Cannot read property 'file' of undefined") { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[HINT] start -> getContextVariables -> getOwnStatusReport-> Check the bot configuration to see if all of its statusDependencies declarations are correct. ") + "[HINT] getContextVariables -> getOwnStatusReport-> Check the bot configuration to see if all of its statusDependencies declarations are correct. ") TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[HINT] start -> getContextVariables -> getOwnStatusReport-> Dependencies loaded -> keys = " + JSON.stringify(statusDependenciesModule.keys)) + "[HINT] getContextVariables -> getOwnStatusReport-> Dependencies loaded -> keys = " + JSON.stringify(statusDependenciesModule.keys)) TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[HINT] start -> getContextVariables -> getOwnStatusReport-> Dependencies loaded -> Double check that you are not running a process that only can be run at noTime mode at a certain month when it is not prepared to do so.") + "[HINT] getContextVariables -> getOwnStatusReport-> Dependencies loaded -> Double check that you are not running a process that only can be run at noTime mode at a certain month when it is not prepared to do so.") } callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) } } function advanceTime( + fromDate, + lastDate, contextVariables, nextFunctionToCall, processIndex, @@ -274,7 +278,7 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> buildOutput -> advanceTime -> New processing time @ " + + "[INFO] advanceTime -> New processing time @ " + contextVariables.datetimeLastProducedFile.getUTCFullYear() + "/" + (contextVariables.datetimeLastProducedFile.getUTCMonth() + 1) + "/" + contextVariables.datetimeLastProducedFile.getUTCDate() + ".") @@ -283,7 +287,7 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { if (contextVariables.datetimeLastProducedFile.valueOf() > contextVariables.datetimeLastAvailableDependencyFile.valueOf()) { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> buildOutput -> advanceTime -> Head of the market found @ " + + "[INFO] advanceTime -> Head of the market found @ " + contextVariables.datetimeLastProducedFile.getUTCFullYear() + "/" + (contextVariables.datetimeLastProducedFile.getUTCMonth() + 1) + "/" + contextVariables.datetimeLastProducedFile.getUTCDate() + ".") @@ -312,8 +316,9 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { function nextDependencyDailyFile( contextVariables, - dataDependencyNode, + node, aggregateAndWriteOutputFile, + fileStorage, processIndex, callBackFunction ) { @@ -323,6 +328,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { many days needed and it should only stop when it reaches the head of the market. */ + const ONE_MIN_DATASET_TYPE = "One-Min" + let dateForPath = contextVariables.datetimeLastProducedFile.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCDate(), 2) @@ -334,18 +341,18 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { TS.projects.superalgos.globals.taskConstants.PROJECT_DEFINITION_NODE.config.codeName + "/" + TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.type.replace(' ', '-') + "/" + TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName + "/" + - dataDependencyNode.referenceParent.parentNode.parentNode.config.codeName + '/' + + node.dataDependency.referenceParent.parentNode.parentNode.config.codeName + '/' + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.config.codeName + "/" + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "-" + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName - let filePath = filePathRoot + "/Output/" + dataDependencyNode.referenceParent.parentNode.config.codeName + '/' + ONE_MIN_DATASET_TYPE + '/' + dateForPath; + let filePath = filePathRoot + "/Output/" + node.dataDependency.referenceParent.parentNode.config.codeName + '/' + ONE_MIN_DATASET_TYPE + '/' + dateForPath; filePath += '/' + fileName fileStorage.getTextFile(filePath, onFileReceived) TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> buildOutput -> timeframesLoop -> loopBody -> nextDependencyDailyFile -> getting file at dateForPath = " + dateForPath) + "[INFO] nextDependencyDailyFile -> getting file at dateForPath = " + dateForPath) function onFileReceived(err, text) { try { @@ -356,24 +363,24 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { dependencyDailyFile = JSON.parse(text) } catch (err) { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildOutput -> timeframesLoop -> loopBody -> nextDependencyDailyFile -> onFileReceived -> Error Parsing JSON -> err = " + err.stack) + "[ERROR] nextDependencyDailyFile -> onFileReceived -> Error Parsing JSON -> err = " + err.stack) TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildOutput -> timeframesLoop -> loopBody -> nextDependencyDailyFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.") + "[ERROR] nextDependencyDailyFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.") callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE) return } } else { if (err.message === 'File does not exist.' || err.code === 'The specified key does not exist.') { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> buildOutput -> timeframesLoop -> loopBody -> nextDependencyDailyFile -> onFileReceived -> Dependency Not Ready -> err = " + JSON.stringify(err)) + "[WARN] nextDependencyDailyFile -> onFileReceived -> Dependency Not Ready -> err = " + JSON.stringify(err)) TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> buildOutput -> timeframesLoop -> loopBody -> nextDependencyDailyFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.") + "[WARN] nextDependencyDailyFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.") callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE) return } else { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildOutput -> timeframesLoop -> loopBody -> nextDependencyDailyFile -> onFileReceived -> Error Received -> err = " + err.stack) + "[ERROR] nextDependencyDailyFile -> onFileReceived -> Error Received -> err = " + err.stack) callBackFunction(err) return } @@ -383,18 +390,17 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { } catch (err) { TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildOutput -> timeframesLoop -> loopBody -> nextDependencyDailyFile -> onFileReceived -> err = " + err.stack) + "[ERROR] nextDependencyDailyFile -> onFileReceived -> err = " + err.stack) callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) } } } function generateOutputFileContent( - outputDatasetNode, + node, outputElements ) { let separator = "" - let fileRecordCounter = 0 let fileContent = "" for (let i = 0; i < outputElements.length; i++) { @@ -403,8 +409,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { fileContent = fileContent + separator + '[' let propertySeparator = "" - for (let j = 0; j < outputDatasetNode.referenceParent.parentNode.record.properties.length; j++) { - let property = outputDatasetNode.referenceParent.parentNode.record.properties[j] + for (let j = 0; j < node.outputDataset.referenceParent.parentNode.record.properties.length; j++) { + let property = node.outputDataset.referenceParent.parentNode.record.properties[j] if (property.config.isString === true) { fileContent = fileContent + propertySeparator + '"' + element[property.config.codeName] + '"' } else { @@ -416,7 +422,6 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { fileContent = fileContent + "]" separator = "," - fileRecordCounter++ } fileContent = "[" + fileContent + "]" @@ -424,8 +429,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { } function aggregateFileContent( - outputDatasetNode, - dataDependencyNode, + node, + contextVariables, dependencyDailyFile, outputElementsArray, TIME_FRAME_VALUE @@ -449,8 +454,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { /* Set the output element the default values for each of it's properties. */ - for (let j = 0; j < outputDatasetNode.referenceParent.parentNode.record.properties.length; j++) { - let property = outputDatasetNode.referenceParent.parentNode.record.properties[j] + for (let j = 0; j < node.outputDataset.referenceParent.parentNode.record.properties.length; j++) { + let property = node.outputDataset.referenceParent.parentNode.record.properties[j] if (property.config.isString === true || property.config.isDate === true) { outputElement[property.config.codeName] = "" // Default Value @@ -486,8 +491,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { /* Set up the Data Dependency Record Map and Data Dependency Element Object */ - for (let k = 0; k < dataDependencyNode.referenceParent.parentNode.record.properties.length; k++) { - let property = dataDependencyNode.referenceParent.parentNode.record.properties[k] + for (let k = 0; k < node.dataDependency.referenceParent.parentNode.record.properties.length; k++) { + let property = node.dataDependency.referenceParent.parentNode.record.properties[k] record.map.set(property.config.codeName, record.values[k]) element[property.config.codeName] = record.values[k] } @@ -530,8 +535,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { */ if (saveElement === false) { - for (let j = 0; j < outputDatasetNode.referenceParent.parentNode.record.properties.length; j++) { - let property = outputDatasetNode.referenceParent.parentNode.record.properties[j] + for (let j = 0; j < node.outputDataset.referenceParent.parentNode.record.properties.length; j++) { + let property = node.outputDataset.referenceParent.parentNode.record.properties[j] if (property.config.aggregationMethod === 'First') { outputElement[property.config.codeName] = record.map.get(property.config.codeName) } @@ -546,8 +551,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { Everything that follows will be set for each element overiding the previous ones, so only the last values will survive. */ - for (let j = 0; j < outputDatasetNode.referenceParent.parentNode.record.properties.length; j++) { - let property = outputDatasetNode.referenceParent.parentNode.record.properties[j] + for (let j = 0; j < node.outputDataset.referenceParent.parentNode.record.properties.length; j++) { + let property = node.outputDataset.referenceParent.parentNode.record.properties[j] if (property.config.aggregationMethod === 'Last') { outputElement[property.config.codeName] = record.map.get(property.config.codeName) } @@ -561,8 +566,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { Note that to be able to calculate the minimum, we will be assigning to all properties the first element values, so as to have a baseline from where to compare later on. */ - for (let j = 0; j < outputDatasetNode.referenceParent.parentNode.record.properties.length; j++) { - let property = outputDatasetNode.referenceParent.parentNode.record.properties[j] + for (let j = 0; j < node.outputDataset.referenceParent.parentNode.record.properties.length; j++) { + let property = node.outputDataset.referenceParent.parentNode.record.properties[j] if (property.config.aggregationMethod === 'Min' || saveElement === false) { if (record.map.get(property.config.codeName) < outputElement[property.config.codeName]) { outputElement[property.config.codeName] = record.map.get(property.config.codeName) @@ -575,8 +580,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { /* This is the MAX type of aggregation. */ - for (let j = 0; j < outputDatasetNode.referenceParent.parentNode.record.properties.length; j++) { - let property = outputDatasetNode.referenceParent.parentNode.record.properties[j] + for (let j = 0; j < node.outputDataset.referenceParent.parentNode.record.properties.length; j++) { + let property = node.outputDataset.referenceParent.parentNode.record.properties[j] if (property.config.aggregationMethod === 'Max') { if (record.map.get(property.config.codeName) > outputElement[property.config.codeName]) { outputElement[property.config.codeName] = record.map.get(property.config.codeName) @@ -589,8 +594,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { /* This is the SUM type of aggregation. */ - for (let j = 0; j < outputDatasetNode.referenceParent.parentNode.record.properties.length; j++) { - let property = outputDatasetNode.referenceParent.parentNode.record.properties[j] + for (let j = 0; j < node.outputDataset.referenceParent.parentNode.record.properties.length; j++) { + let property = node.outputDataset.referenceParent.parentNode.record.properties[j] if (property.config.aggregationMethod === 'Sum') { outputElement[property.config.codeName] = outputElement[property.config.codeName] + record.map.get(property.config.codeName) } @@ -601,8 +606,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { /* This is the AVG type of aggregation. */ - for (let j = 0; j < outputDatasetNode.referenceParent.parentNode.record.properties.length; j++) { - let property = outputDatasetNode.referenceParent.parentNode.record.properties[j] + for (let j = 0; j < node.outputDataset.referenceParent.parentNode.record.properties.length; j++) { + let property = node.outputDataset.referenceParent.parentNode.record.properties[j] if (property.config.aggregationMethod === 'Average') { if (outputElementAverage[property.config.codeName] === undefined) { @@ -628,11 +633,12 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { function writeStatusReport( statusDependenciesModule, - lastFileDate, + contextVariables, + lastFileDate, callBack, processIndex, callBackFunction - ) { + ) { try { let thisReport = statusDependenciesModule.reportsByMainUtility.get('Self Reference') @@ -644,7 +650,7 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { catch (err) { TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> writeStatusReport -> err = " + err.stack) + "[ERROR] writeStatusReport -> err = " + err.stack) callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) } }