From ca5573459eb12efe767dd2a4dc46d95751411ffa Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 21 May 2021 19:02:17 +0200 Subject: [PATCH] Bug Fixes Daily partially running --- Projects/ProjectsSchema.json | 2 +- .../FromOneMinToMultiPeriodDaily.js | 853 ++++-------------- .../FromOneMinToMultiPeriodMarket.js | 20 +- .../DataAggregationFunctions.js | 10 +- 4 files changed, 180 insertions(+), 705 deletions(-) diff --git a/Projects/ProjectsSchema.json b/Projects/ProjectsSchema.json index 92ace9d47..449c49473 100644 --- a/Projects/ProjectsSchema.json +++ b/Projects/ProjectsSchema.json @@ -655,7 +655,7 @@ { "name": "From One Min To Multi Period Daily", "propertyName": "fromOneMinToMultiPeriodDaily", - "functionName": "newSuperalgosBotModulesFromOneMinToMultiPeriodMarket", + "functionName": "newSuperalgosBotModulesFromOneMinToMultiPeriodDaily", "fileName": "FromOneMinToMultiPeriodDaily.js", "folderName": "Indicator-Bot" }, diff --git a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodDaily.js b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodDaily.js index 392798e77..7f00bbdef 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodDaily.js +++ b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodDaily.js @@ -1,762 +1,237 @@ -exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (processIndex) { +exports.newSuperalgosBotModulesFromOneMinToMultiPeriodDaily = function (processIndex) { + /* + This module is about converting a One-Min Daily typeo of data set into a Multi Period Daily type. - const MODULE_NAME = "Candles Volumes Multi Period Daily"; - const CANDLES_FOLDER_NAME = "Candles"; - const CANDLES_ONE_MIN = "One-Min"; - const VOLUMES_FOLDER_NAME = "Volumes"; - const VOLUMES_ONE_MIN = "One-Min"; + The process to do so involves: + + Reading the elements (elements, volumens, bolllinger bands, news, asset metrics, etc.) from a + One-Min Dataset (a dataset that is organized with elements spanning one min, like one min begin-end-elements, + or elements with a timestamp captured approximatelly one minute from each other) + organized in Daily Files. + + It is going to output a Daily Files dataset for every Daily Time Frame, aggregating the + input information into elements with a begin and end property. + + Everytime this process runs, it will be able to resume its job and process everything pending until + reaching the head of the market. + */ + const MODULE_NAME = "From One Min To Multi Period Market" let thisObject = { initialize: initialize, + finalize: finalize, start: start } let fileStorage = TS.projects.superalgos.taskModules.fileStorage.newFileStorage(processIndex) - let statusDependencies; - let beginingOfMarket + + let statusDependenciesModule + let dataDependenciesModule + let node = {} // Usefull nodes for this module will be stored here. return thisObject; - function initialize(pStatusDependencies, callBackFunction) { - try { - statusDependencies = pStatusDependencies - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE) + function initialize(pStatusDependencies, pDataDependencies, callBackFunction) { - } catch (err) { - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] initialize -> err = " + err.stack) - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) - } + statusDependenciesModule = pStatusDependencies + dataDependenciesModule = pDataDependencies + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.checkForKnownConstraints( + dataDependenciesModule, + node, + processIndex, + callBackFunction + ) } - /* - This process is going to do the following: - Read the candles and volumes from Exchange Raw Data and produce a - file for each day and for each period with candles and volumes. - */ + + function finalize() { + fileStorage = undefined + statusDependenciesModule = undefined + dataDependenciesModule = undefined + thisObject = undefined + node = undefined + } + function start(callBackFunction) { + try { /* Context Variables */ let contextVariables = { datetimeLastProducedFile: undefined, // Datetime of the last file files successfully produced by this process. datetimeBeginingOfMarketFile: undefined, // Datetime of the first trade file in the whole market history. - datetimeLastAvailableDependencyFile: undefined // Datetime of the last file available to be used as an input of this process. - }; - - getContextVariables(); - - function getContextVariables() { - try { - let thisReport; - let reportKey; - let statusReport; - - /* We look first for Exchange Raw Data in order to get when the market starts. */ - reportKey = "Masters" + "-" + "Exchange-Raw-Data" + "-" + "Historic-OHLCVs" - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> getContextVariables -> reportKey = " + reportKey) - - statusReport = statusDependencies.statusReports.get(reportKey); - if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE); - return; - } - - if (statusReport.status === "Status Report is corrupt.") { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE); - return; - } - - thisReport = statusDependencies.statusReports.get(reportKey).file; - if (thisReport.beginingOfMarket === undefined) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet."); - - let customOK = { - result: TS.projects.superalgos.globals.standardResponses.CUSTOM_OK_RESPONSE.result, - message: "Dependency does not exist." - } - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> customOK = " + customOK.message) - callBackFunction(customOK) - return - } - - contextVariables.datetimeBeginingOfMarketFile = new Date( - thisReport.beginingOfMarket.year + "-" + - thisReport.beginingOfMarket.month + "-" + - thisReport.beginingOfMarket.days + " " + - thisReport.beginingOfMarket.hours + ":" + - thisReport.beginingOfMarket.minutes + - TS.projects.superalgos.globals.timeConstants.GMT_SECONDS) - - /* Second, we get the report from Exchange Raw Data, to know when the marted ends. */ - reportKey = "Masters" + "-" + "Exchange-Raw-Data" + "-" + "Historic-OHLCVs" - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> getContextVariables -> reportKey = " + reportKey) - - statusReport = statusDependencies.statusReports.get(reportKey); - if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE); - return - } - - if (statusReport.status === "Status Report is corrupt.") { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE); - return - } - - thisReport = statusDependencies.statusReports.get(reportKey).file - if (thisReport.lastFile === undefined) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); - - let customOK = { - result: TS.projects.superalgos.globals.standardResponses.CUSTOM_OK_RESPONSE.result, - message: "Dependency not ready." - } - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> customOK = " + customOK.message) - callBackFunction(customOK) - return - } - - contextVariables.datetimeLastAvailableDependencyFile = new Date( - thisReport.lastFile.year + "-" + - thisReport.lastFile.month + "-" + - thisReport.lastFile.days + " " + "00:00" + - TS.projects.superalgos.globals.timeConstants.GMT_SECONDS) - - /* Finally we get our own Status Report. */ - - reportKey = "Masters" + "-" + "Candles-Volumes" + "-" + "Multi-Period-Daily" - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> getContextVariables -> reportKey = " + reportKey) - - statusReport = statusDependencies.statusReports.get(reportKey) - - if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ") - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE) - return - } - - if (statusReport.status === "Status Report is corrupt.") { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process.") - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) - return - } - - thisReport = statusDependencies.statusReports.get(reportKey).file; - if (thisReport.lastFile !== undefined) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> getContextVariables -> Process Running not for the very first time. -> reportKey = " + reportKey) - - beginingOfMarket = new Date(thisReport.beginingOfMarket) - if (beginingOfMarket.valueOf() !== contextVariables.datetimeBeginingOfMarketFile.valueOf()) { // Reset Mechanism for Begining of the Market - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> getContextVariables -> Reset Mechanism for Begining of the Market Activated. -> reportKey = " + reportKey); - - beginingOfMarket = new Date(contextVariables.datetimeBeginingOfMarketFile.getUTCFullYear() + "-" + (contextVariables.datetimeBeginingOfMarketFile.getUTCMonth() + 1) + "-" + contextVariables.datetimeBeginingOfMarketFile.getUTCDate() + " " + "00:00" + TS.projects.superalgos.globals.timeConstants.GMT_SECONDS); - contextVariables.datetimeLastProducedFile = new Date( - contextVariables.datetimeBeginingOfMarketFile.getUTCFullYear() + "-" + - (contextVariables.datetimeBeginingOfMarketFile.getUTCMonth() + 1) + "-" + - contextVariables.datetimeBeginingOfMarketFile.getUTCDate() + " " + "00:00" + - TS.projects.superalgos.globals.timeConstants.GMT_SECONDS) - contextVariables.datetimeLastProducedFile = new Date( - contextVariables.datetimeLastProducedFile.valueOf() - - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) // Go back one day to start well. - - buildCandles() - return - } - - contextVariables.datetimeLastProducedFile = new Date(thisReport.lastFile); - /* - Here we assume that the last day written might contain incomplete information. - This actually happens every time the head of the market is reached. - For that reason we go back one day, the partial information is discarded - and added again with whatever new info is available. - */ - contextVariables.datetimeLastProducedFile = new Date(contextVariables.datetimeLastProducedFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) - buildCandles() - return - - } else { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> getContextVariables -> Process Running for the very first time. -> reportKey = " + reportKey); - - beginingOfMarket = new Date( - contextVariables.datetimeBeginingOfMarketFile.getUTCFullYear() + "-" + - (contextVariables.datetimeBeginingOfMarketFile.getUTCMonth() + 1) + "-" + - contextVariables.datetimeBeginingOfMarketFile.getUTCDate() + " " + "00:00" + - TS.projects.superalgos.globals.timeConstants.GMT_SECONDS) - contextVariables.datetimeLastProducedFile = new Date( - contextVariables.datetimeBeginingOfMarketFile.getUTCFullYear() + "-" + - (contextVariables.datetimeBeginingOfMarketFile.getUTCMonth() + 1) + "-" + - contextVariables.datetimeBeginingOfMarketFile.getUTCDate() + " " + "00:00" + - TS.projects.superalgos.globals.timeConstants.GMT_SECONDS) - contextVariables.datetimeLastProducedFile = new Date(contextVariables.datetimeLastProducedFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS); // Go back one day to start well. - - buildCandles() - return - } - - } catch (err) { - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> getContextVariables -> err = " + err.stack) - if (err.message === "Cannot read property 'file' of undefined") { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[HINT] start -> getContextVariables -> Check the bot configuration to see if all of its statusDependencies declarations are correct. "); - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys)) - } - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) - } + datetimeLastAvailableDependencyFile: undefined, // Datetime of the last file available to be used as an input of this process. + beginingOfMarket: undefined // Datetime of the begining of the market. } - function buildCandles() { + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.getContextVariables( + statusDependenciesModule, + contextVariables, + buildOutput, + buildOutput, + processIndex, + callBackFunction + ) + + function buildOutput(allPreviousElements) { + try { let fromDate = new Date(contextVariables.datetimeLastProducedFile.valueOf()) let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date()) - let outputCandles - let outputVolumes + /* + Firstly we prepere the arrays that will accumulate all the information for each output file. + */ + let outputElements = [] - advanceTime() + for (let timeFrameArrayIndex = 0; timeFrameArrayIndex < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; timeFrameArrayIndex++) { + const emptyArray = [] + outputElements.push(emptyArray) + } + moveToNextDay() - function advanceTime() { - contextVariables.datetimeLastProducedFile = new Date(contextVariables.datetimeLastProducedFile.valueOf() + TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS); - - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> buildCandles -> advanceTime -> New processing time @ " + contextVariables.datetimeLastProducedFile.getUTCFullYear() + "/" + (contextVariables.datetimeLastProducedFile.getUTCMonth() + 1) + "/" + contextVariables.datetimeLastProducedFile.getUTCDate() + ".") - - /* Validation that we are not going past the head of the market. */ - if (contextVariables.datetimeLastProducedFile.valueOf() > contextVariables.datetimeLastAvailableDependencyFile.valueOf()) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> buildCandles -> advanceTime -> Head of the market found @ " + contextVariables.datetimeLastProducedFile.getUTCFullYear() + "/" + (contextVariables.datetimeLastProducedFile.getUTCMonth() + 1) + "/" + contextVariables.datetimeLastProducedFile.getUTCDate() + ".") - - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE) // Here is where we finish processing and wait for the platform to run this module again. - return - } - /* Telling the world we are alive and doing well */ - let currentDateString = - contextVariables.datetimeLastProducedFile.getUTCFullYear() + '-' + - TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCMonth() + 1, 2) + '-' + - TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCDate(), 2) - let currentDate = new Date(contextVariables.datetimeLastProducedFile) - let percentage = TS.projects.superalgos.utilities.dateTimeFunctions.getPercentage(fromDate, currentDate, lastDate) - TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, currentDateString, percentage) - - if (TS.projects.superalgos.utilities.dateTimeFunctions.areTheseDatesEqual(currentDate, new Date()) === false) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.newInternalLoop(currentDate, percentage); - } - /* - We prepere the arrays that will accumulate all the information for each output file. - */ - outputCandles = [] - outputVolumes = [] - - for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; n++) { - const emptyArray1 = [] - const emptyArray2 = [] - outputCandles.push(emptyArray1) - outputVolumes.push(emptyArray2) - } - timeframesLoop(); + function moveToNextDay() { + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.advanceTime( + fromDate, + lastDate, + contextVariables, + timeframesLoop, + processIndex, + callBackFunction + ) } function timeframesLoop() { /* We will iterate through all posible time frames. */ - let n = 0 // loop Variable representing each possible period as defined at the periods array. - loopBody(); + let timeFrameArrayIndex = 0 // loop Variable representing each possible period as defined at the periods array. + + loopBody() function loopBody() { - const outputPeriod = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][0] - const timeFrame = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1] - nextCandleFile(); + const TIME_FRAME_VALUE = TS.projects.superalgos.globals.timeFrames.dailyFilesPeriods()[timeFrameArrayIndex][0] + const TIME_FRAME_LABEL = TS.projects.superalgos.globals.timeFrames.dailyFilesPeriods()[timeFrameArrayIndex][1] + /* + Here we are inside a Loop that is going to advance 1 day at the time, + at each pass, we will read one of Exchange Raw Data's daily files and + add all its elements to our in memory arrays. + */ + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.nextDependencyDailyFile( + contextVariables, + node, + aggregateAndWriteOutputFile, + fileStorage, + processIndex, + callBackFunction + ) - function nextCandleFile() { - let dateForPath = - contextVariables.datetimeLastProducedFile.getUTCFullYear() + '/' + - TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCMonth() + 1, 2) + '/' + - TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCDate(), 2) - let fileName = "Data.json" + function aggregateAndWriteOutputFile(dependencyDailyFile) { + /* + Here we call the function that will aggregate all the information + at the dependency file into standarized begin-end-elements. + */ + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.aggregateFileContent( + node, + contextVariables, + dependencyDailyFile, + outputElements[timeFrameArrayIndex], + TIME_FRAME_VALUE + ) - let filePathRoot = - 'Project/' + - TS.projects.superalgos.globals.taskConstants.PROJECT_DEFINITION_NODE.config.codeName + "/" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.type.replace(' ', '-') + "/" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName + "/" + - "Exchange-Raw-Data" + '/' + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.config.codeName + "/" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "-" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName - let filePath = filePathRoot + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath - filePath += '/' + fileName + writeOutputFile(outputElements[timeFrameArrayIndex], TIME_FRAME_LABEL, controlLoop) - fileStorage.getTextFile(filePath, onFileReceived) - - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> buildCandles -> timeframesLoop -> loopBody -> getting file at dateForPath = " + dateForPath) - - function onFileReceived(err, text) { + function writeOutputFile(outputElementsCurrentTimeFrame, TIME_FRAME_LABEL, callBack) { + /* + Here we will write the contents of the file to disk. + */ try { - let candlesFile; + let fileContent = TS.projects.superalgos.functionLibraries.dataAggregationFunctions.generateOutputFileContent( + node, + outputElementsCurrentTimeFrame + ) + /* + Once the file content is generated, we need to reset this array. + */ + outputElementsCurrentTimeFrame = [] + /* + Daily files are saved to disk inside a folder for each year / month / day + */ + let file = { date: new Date(contextVariables.datetimeLastProducedFile.valueOf()) } + file.year = file.date.getUTCFullYear() + file.month = file.date.getUTCMonth() + 1 + file.day = file.date.getUTCDate() + let dateForPath = file.year + '/' + + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(file.month, 2) + '/' + + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(file.day, 2) - if (err.result === TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - try { - candlesFile = JSON.parse(text); + let fileName = 'Data.json' + let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + + "/Output/" + + node.outputDataset.referenceParent.parentNode.config.codeName + "/" + + node.outputDataset.referenceParent.config.codeName + "/" + + TIME_FRAME_LABEL + "/" + + dateForPath - } catch (err) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> Error Parsing JSON -> err = " + err.stack) - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry."); - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE) - return; - } - } else { + filePath += '/' + fileName - if (err.message === 'File does not exist.' || err.code === 'The specified key does not exist.') { + fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated) - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> Dependency Not Ready -> err = " + err.stack) - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry."); - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE) - return + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[INFO] start -> writeOutputFile -> creating file at filePath = " + filePath) - } else { + function onFileCreated(err) { + if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> Error Received -> err = " + err.stack) - callBackFunction(err); + "[ERROR] start -> writeOutputFile -> onFileCreated -> err = " + err.stack) + callBackFunction(err) return } + + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[WARN] start -> writeOutputFile -> onFileCreated -> Finished with File @ " + + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "_" + + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName + ", " + + " filePath = " + filePath + "/" + fileName) + callBack() } - - const inputCandlesPerdiod = 60 * 1000; // 1 min - const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs - - let totalOutputCandles = inputFilePeriod / outputPeriod; - let beginingOutputTime = contextVariables.datetimeLastProducedFile.valueOf(); - - for (let i = 0; i < totalOutputCandles; i++) { - - let outputCandle = { - open: 0, - close: 0, - min: 0, - max: 0, - begin: 0, - end: 0 - } - - let saveCandle = false; - - outputCandle.begin = beginingOutputTime + i * outputPeriod; - outputCandle.end = beginingOutputTime + (i + 1) * outputPeriod - 1; - - for (let j = 0; j < candlesFile.length; j++) { - - let candle = { - open: candlesFile[j][2], - close: candlesFile[j][3], - min: candlesFile[j][0], - max: candlesFile[j][1], - begin: candlesFile[j][4], - end: candlesFile[j][5] - }; - - /* Here we discard all the candles out of range. */ - if (candle.begin >= outputCandle.begin && candle.end <= outputCandle.end) { - - if (saveCandle === false) { // this will set the value only once. - outputCandle.open = candle.open; - outputCandle.min = candle.min - outputCandle.max = candle.max - } - - saveCandle = true; - outputCandle.close = candle.close; // only the last one will be saved - if (candle.min < outputCandle.min) { - outputCandle.min = candle.min - } - if (candle.max > outputCandle.max) { - outputCandle.max = candle.max - } - } - } - if (saveCandle === true) { // then we have a valid candle, otherwise it means there were no candles to fill this one in its time range. - outputCandles[n].push(outputCandle) - } - } - nextVolumeFile(); - } catch (err) { + } + catch (err) { TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> err = " + err.stack); - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE); + "[ERROR] start -> writeOutputFile -> err = " + err.stack) + callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) } } } - - function nextVolumeFile() { - try { - let dateForPath = contextVariables.datetimeLastProducedFile.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCDate(), 2); - let fileName = "Data.json" - let filePathRoot = - 'Project/' + - TS.projects.superalgos.globals.taskConstants.PROJECT_DEFINITION_NODE.config.codeName + "/" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.type.replace(' ', '-') + "/" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName + "/" + - "Exchange-Raw-Data" + '/' + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.config.codeName + "/" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "-" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName - let filePath = - filePathRoot + - "/Output/" + - VOLUMES_FOLDER_NAME + '/' + - VOLUMES_ONE_MIN + '/' + - dateForPath - filePath += '/' + fileName - - fileStorage.getTextFile(filePath, onFileReceived); - - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> buildCandles -> timeframesLoop -> loopBody -> nextVolumeFile -> getting file at dateForPath = " + dateForPath); - - function onFileReceived(err, text) { - let volumesFile - - if (err.result === TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - try { - volumesFile = JSON.parse(text); - } catch (err) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Error Parsing JSON -> err = " + err.stack); - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.") - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE) - return - } - } else { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Error Received -> err = " + err.stack) - callBackFunction(err) - return - } - - const inputVolumesPerdiod = 60 * 1000; // 1 min - const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs - let totalOutputVolumes = inputFilePeriod / outputPeriod; // this should be 2 in this case. - let beginingOutputTime = contextVariables.datetimeLastProducedFile.valueOf(); - - for (let i = 0; i < totalOutputVolumes; i++) { - let outputVolume = { - buy: 0, - sell: 0, - begin: 0, - end: 0 - } - let saveVolume = false; - - outputVolume.begin = beginingOutputTime + i * outputPeriod; - outputVolume.end = beginingOutputTime + (i + 1) * outputPeriod - 1; - - for (let j = 0; j < volumesFile.length; j++) { - let volume = { - buy: volumesFile[j][0], - sell: volumesFile[j][1], - begin: volumesFile[j][2], - end: volumesFile[j][3] - } - - /* Here we discard all the Volumes out of range. */ - if (volume.begin >= outputVolume.begin && volume.end <= outputVolume.end) { - saveVolume = true - outputVolume.buy = outputVolume.buy + volume.buy - outputVolume.sell = outputVolume.sell + volume.sell - } - } - - if (saveVolume === true) { - outputVolumes[n].push(outputVolume); - } - } - writeFiles(outputCandles[n], outputVolumes[n], timeFrame, controlLoop) - } - } catch (err) { - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextVolumeFile -> onFileReceived -> err = " + err.stack); - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE); - } - } } function controlLoop() { - n++; - if (n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length) { + timeFrameArrayIndex++ + if (timeFrameArrayIndex < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length) { loopBody() } else { - writeDataRanges(onWritten); - - function onWritten(err) { - if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] writeDataRanges -> writeDataRanges -> onCandlesDataRangeWritten -> err = " + err.stack) - callBack(err) - return - } - writeStatusReport(contextVariables.datetimeLastProducedFile, advanceTime); - } + TS.projects.superalgos.functionLibraries.dataAggregationFunctions.writeStatusReport( + statusDependenciesModule, + contextVariables, + contextVariables.datetimeLastProducedFile, + moveToNextDay, + processIndex, + callBackFunction + ) } } } } catch (err) { - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> buildCandles -> err = " + err.stack); - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE); + "[ERROR] start -> buildOutput -> err = " + err.stack) + callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) } } - - function writeFiles(candles, volumes, timeFrame, callBack) { - /* - Here we will write the contents of the Candles and Volumens files. - */ - try { - writeCandles() - - function writeCandles() { - let separator = "" - let fileRecordCounter = 0 - let fileContent = "" - - for (let i = 0; i < candles.length; i++) { - let candle = candles[i] - fileContent = fileContent + separator + '[' + candles[i].min + "," + candles[i].max + "," + candles[i].open + "," + candles[i].close + "," + candles[i].begin + "," + candles[i].end + "]"; - if (separator === "") { separator = ","; } - fileRecordCounter++ - } - - fileContent = "[" + fileContent + "]"; - - let fileName = 'Data.json'; - let dateForPath = - contextVariables.datetimeLastProducedFile.getUTCFullYear() + '/' + - TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCMonth() + 1, 2) + '/' + - TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCDate(), 2) - let filePath = - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + - "/Output/" + - CANDLES_FOLDER_NAME + "/" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.codeName + "/" + - timeFrame + "/" + - dateForPath - filePath += '/' + fileName - - fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated) - - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> writeFiles -> writeCandles -> creating file at filePath = " + filePath); - - function onFileCreated(err) { - if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> writeFiles -> writeCandles -> onFileCreated -> err = " + err.stack); - callBackFunction(err); - return; - } - - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> writeFiles -> writeCandles -> onFileCreated -> Finished with File @ " + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "_" + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName); - - writeVolumes(); - } - } - - function writeVolumes() { - let separator = ""; - let fileRecordCounter = 0; - let fileContent = ""; - - for (let i = 0; i < volumes.length; i++) { - let candle = volumes[i]; - fileContent = fileContent + separator + '[' + volumes[i].buy + "," + volumes[i].sell + "," + volumes[i].begin + "," + volumes[i].end + "]"; - if (separator === "") { separator = ","; } - fileRecordCounter++ - } - - fileContent = "[" + fileContent + "]"; - - let fileName = 'Data.json'; - let dateForPath = - contextVariables.datetimeLastProducedFile.getUTCFullYear() + '/' + - TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCMonth() + 1, 2) + '/' + - TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCDate(), 2) - let filePath = - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + - "/Output/" + - VOLUMES_FOLDER_NAME + "/" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.codeName + "/" + - timeFrame + "/" + dateForPath - filePath += '/' + fileName - - fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated); - - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> writeFiles -> writeVolumes -> creating file at filePath = " + filePath); - - function onFileCreated(err) { - - if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> writeFiles -> writeVolumes -> onFileCreated -> err = " + err.stack) - callBackFunction(err) - return - } - - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> writeFiles -> writeVolumes -> onFileCreated -> Finished with File @ " + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "_" + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName); - - callBack() - } - } - } - catch (err) { - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> writeFiles -> err = " + err.stack); - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE); - } - } - - function writeDataRanges(callBack) { - try { - writeDataRange(contextVariables.datetimeBeginingOfMarketFile, contextVariables.datetimeLastProducedFile, CANDLES_FOLDER_NAME, onCandlesDataRangeWritten); - - function onCandlesDataRangeWritten(err) { - if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] writeDataRanges -> writeDataRanges -> onCandlesDataRangeWritten -> err = " + err.stack) - callBack(err) - return - } - - writeDataRange(contextVariables.datetimeBeginingOfMarketFile, contextVariables.datetimeLastProducedFile, VOLUMES_FOLDER_NAME, onVolumeDataRangeWritten); - - function onVolumeDataRangeWritten(err) { - if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] writeDataRanges -> writeDataRanges -> onVolumeDataRangeWritten -> err = " + err.stack) - callBack(err) - return - } - callBack(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE); - } - } - } - catch (err) { - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> writeDataRanges -> err = " + err.stack); - callBack(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE); - } - - } - - function writeDataRange(pBegin, pEnd, pProductFolder, callBack) { - try { - let dataRange = { - begin: pBegin.valueOf(), - end: pEnd.valueOf() + TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS - } - - let fileContent = JSON.stringify(dataRange); - - let fileName = 'Data.Range.json'; - let filePath = - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + pProductFolder + "/" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.codeName - filePath += '/' + fileName - - fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated); - - function onFileCreated(err) { - if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> writeDataRange -> onFileCreated -> err = " + err.stack) - callBack(err) - return - } - - let key = - TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName + "-" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.config.codeName + "-" + - pProductFolder + "-" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.config.codeName + "-" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + '/' + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName - let event = { - dateRange: dataRange - } - - TS.projects.superalgos.globals.taskConstants.EVENT_SERVER_CLIENT_MODULE_OBJECT.raiseEvent(key, 'Data Range Updated', event) - - callBack(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE); - } - } - catch (err) { - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> writeDataRange -> err = " + err.stack) - callBack(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) - } - } - - function writeStatusReport(lastFileDate, callBack) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> writeStatusReport -> lastFileDate = " + lastFileDate) - try { - let reportKey = "Masters" + "-" + "Candles-Volumes" + "-" + "Multi-Period-Daily" - let thisReport = statusDependencies.statusReports.get(reportKey) - - thisReport.file.lastExecution = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME; - thisReport.file.lastFile = lastFileDate; - thisReport.file.beginingOfMarket = beginingOfMarket.toUTCString() - thisReport.save(callBack); - } - catch (err) { - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> writeStatusReport -> err = " + err.stack); - callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE); - } - } - } catch (err) { TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] 'Start' - ERROR : " + err.stack); + "[ERROR] start -> err = " + err.stack) callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) } } diff --git a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodMarket.js b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodMarket.js index 8e89b3480..a5dbd14dc 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodMarket.js +++ b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/FromOneMinToMultiPeriodMarket.js @@ -1,9 +1,11 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (processIndex) { /* - This process is going to do the following: + This module is about converting a One-Min Daily typeo of data set into a Multi Period Market type. + + The process to do so involves: - Read the elements (elements, volumens, bolllinger bands, news, asset metrics, etc.) from a - One-Min dataset (a dataset that is organized with elements spanning one min, like one min begin-end-elements, + Reading the elements (elements, volumens, bolllinger bands, news, asset metrics, etc.) from a + One-Min Dataset (a dataset that is organized with elements spanning one min, like one min begin-end-elements, or elements with a timestamp captured approximatelly one minute from each other) organized in Daily Files. @@ -18,10 +20,8 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process 2. Then it will append to these arrays the new information it gets from the data dependency file. - 3. It knows from it's status report which was the last DAY it processed. Since that day might not have been - completed (maybe it was at the head of the market). The process will have to be carefull not to append elements - that are already belonging to the first day to process. To take care of that, it will discard all elements - of the last processed day, and then it will process that full day again adding all the elements found at the current run. + 3. The already saved elements belonging to the start processing day will be discarded and replaced by the + dependency elements of the same day, potentially completing missing elements. */ const MODULE_NAME = "From One Min To Multi Period Market" @@ -259,17 +259,17 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process writeOutputFile(outputElements[timeFrameArrayIndex], TIME_FRAME_LABEL, controlLoop) - function writeOutputFile(outputElements, TIME_FRAME_LABEL, callBack) { + function writeOutputFile(outputElementsCurrentTimeFrame, TIME_FRAME_LABEL, callBack) { /* Here we will write the contents of the file to disk. */ try { let fileContent = TS.projects.superalgos.functionLibraries.dataAggregationFunctions.generateOutputFileContent( node, - outputElements + outputElementsCurrentTimeFrame ) - let fileName = 'Data.json'; + let fileName = 'Data.json' let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + node.outputDataset.referenceParent.parentNode.config.codeName + "/" + diff --git a/Projects/Superalgos/TS/Function-Libraries/DataAggregationFunctions.js b/Projects/Superalgos/TS/Function-Libraries/DataAggregationFunctions.js index 764a499c1..6967e9930 100644 --- a/Projects/Superalgos/TS/Function-Libraries/DataAggregationFunctions.js +++ b/Projects/Superalgos/TS/Function-Libraries/DataAggregationFunctions.js @@ -63,8 +63,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { function getContextVariables( statusDependenciesModule, contextVariables, - loadExistingFiles, - buildOutput, + callbackNotFirstRun, + callbackFirstRun, processIndex, callBackFunction ) { @@ -205,7 +205,7 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { contextVariables.datetimeLastProducedFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) // Go back one day to start well. - buildOutput() + callbackFirstRun() return } @@ -218,7 +218,7 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { */ contextVariables.datetimeLastProducedFile = new Date(contextVariables.datetimeLastProducedFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) - loadExistingFiles() + callbackNotFirstRun() return } else { /* @@ -239,7 +239,7 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () { contextVariables.datetimeLastProducedFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) // Go back one day to start well. - buildOutput() + callbackFirstRun() return }