From 10698a49a773c6e17f7b866fc8d2477971e9cb7a Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 5 Feb 2021 12:02:22 +0100 Subject: [PATCH] Output Management Functions Implemented plus bug fixes --- Projects/ProjectsSchema.json | 6 + .../BollingerBandsMultiPeriodDaily.js | 2 +- .../BollingerBandsMultiPeriodMarket.js | 2 +- .../CandlesVolumesMultiPeriodDaily.js | 2 +- .../CandlesVolumesMultiPeriodMarket.js | 2 +- .../Indicator-Bot/MultiPeriodDaily.js | 2 +- .../Indicator-Bot/MultiPeriodMarket.js | 27 +-- .../Low-Frequency-Trading/TradingOutput.js | 188 ++-------------- .../Low-Frequency-Trading/TradingProcess.js | 6 +- .../TradingSimulation.js | 1 - .../OutputManagementFunctions.js | 203 ++++++++++++++++++ .../ProcessFilesFunctions.js | 4 +- .../TS/Process-Modules/DataDependencies.js | 4 +- .../TS/Utilities/DateTimeFunctions.js | 6 +- 14 files changed, 258 insertions(+), 197 deletions(-) create mode 100644 Projects/Superalgos/TS/Function-Libraries/OutputManagementFunctions.js diff --git a/Projects/ProjectsSchema.json b/Projects/ProjectsSchema.json index 542fa69d2..ba5a331ee 100644 --- a/Projects/ProjectsSchema.json +++ b/Projects/ProjectsSchema.json @@ -454,6 +454,12 @@ "propertyName": "processFilesFunctions", "functionName": "newSuperalgosFunctionLibrariesProcessFilesFunctions", "fileName": "ProcessFilesFunctions.js" + }, + { + "name": "Output Management Functions", + "propertyName": "outputManagementFunctions", + "functionName": "newSuperalgosFunctionLibrariesOutputManagementFunctions", + "fileName": "OutputManagementFunctions.js" } ], "utilities": [ diff --git a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Bollinger-Bands/BollingerBandsMultiPeriodDaily.js b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Bollinger-Bands/BollingerBandsMultiPeriodDaily.js index 726ed72cd..cde9181a9 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Bollinger-Bands/BollingerBandsMultiPeriodDaily.js +++ b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Bollinger-Bands/BollingerBandsMultiPeriodDaily.js @@ -258,7 +258,7 @@ let n; processDate = new Date(contextVariables.lastBandFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. let fromDate = new Date(processDate.valueOf()) - let lastDate = new Date() + let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date()) TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[INFO] start -> buildBands -> processDate = " + processDate) diff --git a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Bollinger-Bands/BollingerBandsMultiPeriodMarket.js b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Bollinger-Bands/BollingerBandsMultiPeriodMarket.js index 9e3d81aad..3f3125abb 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Bollinger-Bands/BollingerBandsMultiPeriodMarket.js +++ b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Bollinger-Bands/BollingerBandsMultiPeriodMarket.js @@ -409,7 +409,7 @@ /* Telling the world we are alive and doing well */ let currentDateString = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCDate(), 2); let currentDate = new Date(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME) - let lastDate = new Date() + let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date()) TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, currentDateString, TS.projects.superalgos.utilities.dateTimeFunctions.getPercentage(currentDate, currentDate, lastDate)) } catch (err) { diff --git a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Candles-Volumes/CandlesVolumesMultiPeriodDaily.js b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Candles-Volumes/CandlesVolumesMultiPeriodDaily.js index ed9686b0d..836424727 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Candles-Volumes/CandlesVolumesMultiPeriodDaily.js +++ b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Candles-Volumes/CandlesVolumesMultiPeriodDaily.js @@ -228,7 +228,7 @@ function buildCandles() { try { let fromDate = new Date(contextVariables.lastCandleFile.valueOf()) - let lastDate = new Date() + let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date()) let outputCandles let outputVolumes diff --git a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Candles-Volumes/CandlesVolumesMultiPeriodMarket.js b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Candles-Volumes/CandlesVolumesMultiPeriodMarket.js index 3bf3bb8e9..053a0f8d3 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Candles-Volumes/CandlesVolumesMultiPeriodMarket.js +++ b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Candles-Volumes/CandlesVolumesMultiPeriodMarket.js @@ -389,7 +389,7 @@ try { let fromDate = new Date(contextVariables.lastCandleFile.valueOf()) - let lastDate = new Date() + let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date()) /* Firstly we prepere the arrays that will accumulate all the information for each output file. */ diff --git a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/MultiPeriodDaily.js b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/MultiPeriodDaily.js index 99acc71f5..626408852 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/MultiPeriodDaily.js +++ b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/MultiPeriodDaily.js @@ -234,7 +234,7 @@ TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).DAILY_FILES_PROCESS_DATETIME = new Date(contextVariables.lastFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) // Go back one day to start well when we advance time at the begining of the loop. let fromDate = new Date(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).DAILY_FILES_PROCESS_DATETIME.valueOf()) - let lastDate = new Date() + let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date()) advanceTime() diff --git a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/MultiPeriodMarket.js b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/MultiPeriodMarket.js index 6dd051d6b..bb2ea38a2 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/MultiPeriodMarket.js +++ b/Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/MultiPeriodMarket.js @@ -16,7 +16,7 @@ return thisObject; function initialize(pStatusDependencies, pDataDependencies, callBackFunction) { - + statusDependencies = pStatusDependencies; dataDependenciesModule = pDataDependencies; @@ -80,7 +80,7 @@ function onFileReceived(err, text) { if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack) callBackFunction(err) return; @@ -136,8 +136,8 @@ } function writeTimeFramesFiles(callBack) { - let outputDatasets = - TS.projects.superalgos.utilities.nodeFunctions.nodeBranchToArray(TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.processOutput, 'Output Dataset') + let outputDatasets = + TS.projects.superalgos.utilities.nodeFunctions.nodeBranchToArray(TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.processOutput, 'Output Dataset') let outputDatasetIndex = -1; controlLoop() @@ -178,14 +178,14 @@ let fileContent = JSON.stringify(timeFramesArray) let fileName = '/Time.Frames.json'; - let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + productCodeName + "/" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.codeName + fileName; + let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + productCodeName + "/" + + TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.codeName + fileName; fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated) function onFileCreated(err) { if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] start -> writeTimeFramesFile -> onFileCreated -> err = " + err.stack) callBack(err) return @@ -195,8 +195,8 @@ } function writeStatusReport(callBack) { - let reportKey = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName + "-" + - TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.config.codeName + "-" + "Multi-Period-Market" + let reportKey = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName + "-" + + TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.config.codeName + "-" + "Multi-Period-Market" let thisReport = statusDependencies.statusReports.get(reportKey) thisReport.file.lastExecution = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME; @@ -214,13 +214,16 @@ /* Telling the world we are alive and doing well */ let currentDateString = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCDate(), 2) let currentDate = new Date(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME) - let lastDate = new Date() - TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, currentDateString, TS.projects.superalgos.utilities.dateTimeFunctions.getPercentage(currentDate, currentDate, lastDate)) + TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat( + processIndex, + currentDateString, + TS.projects.superalgos.utilities.dateTimeFunctions.getPercentage(currentDate, currentDate, currentDate) + ) } } catch (err) { TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] start -> err = " + err.stack) callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE) } diff --git a/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingOutput.js b/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingOutput.js index 348a4f064..10edf3d82 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingOutput.js +++ b/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingOutput.js @@ -34,187 +34,35 @@ exports.newSuperalgosBotModulesTradingOutput = function (processIndex) { let outputDatasets = TS.projects.superalgos.utilities.nodeFunctions.nodeBranchToArray(TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.processOutput, 'Output Dataset') let outputDatasetsMap = new Map() - if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES === true) { - /* - For Daily Files, we are going to initialize the outputs at the first execution - if we are not resuming, but also at every following execution except when - we are at the head of the market, because at the head of the market we need to - append more data to the same output files that already exists. - */ - if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_FIRST_LOOP === true && TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_RESUMING === false) { - await initializeOutputs() - } else { - if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.tradingEngine.current.episode.headOfTheMarket.value === true) { - await readFiles() - } else { - await initializeOutputs() - } - } - } else { - /* - Form Market Files, we are only going to initialize the outputs at the first run - and if we are not resuming execution. Subsequent runs means that we are at the - head of the market. Remember that backtests finishes the session once the reach - the final datetime. - */ - if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_FIRST_LOOP === true && TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_RESUMING === false) { - await initializeOutputs() - } else { - await readFiles() - } - } + await TS.projects.superalgos.functionLibraries.outputManagementFunctions.readFiles( + processIndex, + outputDatasets, + outputDatasetsMap, + timeFrameLabel, + tradingProcessDate, + fileStorage + ) await tradingSimulationModuleObject.runSimulation( chart, market, exchange, outputDatasetsMap, - writeFiles + writeOutputFiles ) tradingSimulationModuleObject.finalize() return - async function initializeOutputs() { - if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) { - await initializeDailyFiles() - } else { - await initializeMarketFiles() - } - } - - async function initializeDailyFiles() { - for (let i = 0; i < outputDatasets.length; i++) { - let outputDatasetNode = outputDatasets[i] - let dataset = outputDatasetNode.referenceParent - - if (dataset.config.type === 'Daily Files') { - outputDatasetsMap.set(dataset.parentNode.config.codeName, []) - } - } - } - - async function initializeMarketFiles() { - for (let i = 0; i < outputDatasets.length; i++) { - let outputDatasetNode = outputDatasets[i] - let dataset = outputDatasetNode.referenceParent - - if (dataset.config.type === 'Market Files') { - outputDatasetsMap.set(dataset.parentNode.config.codeName, []) - } - } - } - - async function readFiles() { - /* - This bot have an output of files that it generates. At every call to the bot, it needs to read the previously generated - files in order to later append more information after the execution is over. Here in this function we are going to - read those output files and get them ready for appending content during the simulation. - */ - if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) { - await readDailyFiles() - } else { - await readMarketFiles() - } - } - - async function readMarketFiles() { - for (let i = 0; i < outputDatasets.length; i++) { - let outputDatasetNode = outputDatasets[i] - let dataset = outputDatasetNode.referenceParent - - if (dataset.config.type === 'Market Files') { - - let fileName = 'Data.json' - let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel - - await readOutputFile(fileName, filePath, dataset.parentNode.config.codeName) - } - } - } - - async function readDailyFiles() { - for (let i = 0; i < outputDatasets.length; i++) { - let outputDatasetNode = outputDatasets[i] - let dataset = outputDatasetNode.referenceParent - - if (dataset.config.type === 'Daily Files') { - - let dateForPath = tradingProcessDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCDate(), 2); - let fileName = 'Data.json' - let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath - - await readOutputFile(fileName, filePath, dataset.parentNode.config.codeName) - } - } - } - - async function readOutputFile(fileName, filePath, productName) { - filePath += '/' + fileName - - let response = await fileStorage.asyncGetTextFile(filePath, true) - - if (response.err.message === 'File does not exist.') { - outputDatasetsMap.set(productName, []) - return - } - if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - throw (response.err) - } - outputDatasetsMap.set(productName, JSON.parse(response.text)) - } - - async function writeFiles() { - /* - The output of files which were appended with information during the simulation execution, now needs to be saved. - */ - if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) { - await writeDailyFiles() - } else { - await writeMarketFiles() - } - } - - async function writeMarketFiles() { - for (let i = 0; i < outputDatasets.length; i++) { - let outputDatasetNode = outputDatasets[i] - let dataset = outputDatasetNode.referenceParent - - if (dataset.config.type === 'Market Files') { - - let fileName = 'Data.json' - let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel - - await writeOutputFile(fileName, filePath, dataset.parentNode.config.codeName) - } - } - } - - async function writeDailyFiles() { - for (let i = 0; i < outputDatasets.length; i++) { - let outputDatasetNode = outputDatasets[i] - let dataset = outputDatasetNode.referenceParent - - if (dataset.config.type === 'Daily Files') { - - let dateForPath = tradingProcessDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCDate(), 2); - let fileName = 'Data.json' - let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath - - await writeOutputFile(fileName, filePath, dataset.parentNode.config.codeName) - } - } - } - - async function writeOutputFile(fileName, filePath, productName) { - filePath += '/' + fileName - let fileContent = JSON.stringify(outputDatasetsMap.get(productName)) - - let response = await fileStorage.asyncCreateTextFile(filePath, fileContent) - - if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - throw (response.err) - } + function writeOutputFiles() { + TS.projects.superalgos.functionLibraries.outputManagementFunctions.writeFiles( + processIndex, + outputDatasets, + outputDatasetsMap, + timeFrameLabel, + tradingProcessDate, + fileStorage + ) } } catch (err) { diff --git a/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingProcess.js b/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingProcess.js index 1f2713679..763fd0fb1 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingProcess.js +++ b/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingProcess.js @@ -131,7 +131,7 @@ if (checkThereAreCandles() === true) { TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Running') - await generateOutput() + await outputManagement() TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Saving') await TS.projects.superalgos.functionLibraries.processFilesFunctions.writeProcessFiles(processIndex, contextVariables, currentTimeFrame, tradingProcessDate, statusDependencies) TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Sleeping') @@ -190,7 +190,7 @@ */ if (checkThereAreCandles() === true) { TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Running') - await generateOutput() + await outputManagement() TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Saving') await TS.projects.superalgos.functionLibraries.processFilesFunctions.writeProcessFiles(processIndex, contextVariables, currentTimeFrame, tradingProcessDate, statusDependencies) TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Sleeping') @@ -259,7 +259,7 @@ } } - async function generateOutput() { + async function outputManagement() { await tradingOutputModuleObject.start( chart, market, diff --git a/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingSimulation.js b/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingSimulation.js index 0bf723599..cebb8d9ba 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingSimulation.js +++ b/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingSimulation.js @@ -310,7 +310,6 @@ exports.newSuperalgosBotModulesTradingSimulation = function (processIndex) { let currentDateString = heartBeatDate.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(heartBeatDate.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(heartBeatDate.getUTCDate(), 2) let currentDate = new Date(heartBeatDate) let percentage = TS.projects.superalgos.utilities.dateTimeFunctions.getPercentage(fromDate, currentDate, lastDate) - /* Theere are a few tasks that we need to do only when the date changes, otherwise it would be suboptimal. diff --git a/Projects/Superalgos/TS/Function-Libraries/OutputManagementFunctions.js b/Projects/Superalgos/TS/Function-Libraries/OutputManagementFunctions.js new file mode 100644 index 000000000..4181155ad --- /dev/null +++ b/Projects/Superalgos/TS/Function-Libraries/OutputManagementFunctions.js @@ -0,0 +1,203 @@ + +exports.newSuperalgosFunctionLibrariesOutputManagementFunctions = function () { + /* + This module helps a bot process to manage it's output files, since it load + the current state of it's output and allows the bot process to append info + to it. It also allows the bot process to save all updated output. + */ + const MODULE_NAME = "Output Management Functions" + + let thisObject = { + readFiles: readFiles, + writeFiles: writeFiles + } + return thisObject + + async function readFiles( + processIndex, + outputDatasets, + outputDatasetsMap, + timeFrameLabel, + tradingProcessDate, + fileStorage + ) { + if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES === true) { + /* + For Daily Files, we are going to initialize the outputs at the first execution + if we are not resuming, but also at every following execution except when + we are at the head of the market, because at the head of the market we need to + append more data to the same output files that already exists. + */ + if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_FIRST_LOOP === true && TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_RESUMING === false) { + await initializeOutputs() + } else { + if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.tradingEngine.current.episode.headOfTheMarket.value === true) { + await loadFiles() + } else { + await initializeOutputs() + } + } + } else { + /* + Form Market Files, we are only going to initialize the outputs at the first run + and if we are not resuming execution. Subsequent runs means that we are at the + head of the market. Remember that backtests finishes the session once the reach + the final datetime. + */ + if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_FIRST_LOOP === true && TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_RESUMING === false) { + await initializeOutputs() + } else { + await loadFiles() + } + } + + async function initializeOutputs() { + if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) { + await initializeDailyFiles() + } else { + await initializeMarketFiles() + } + + async function initializeDailyFiles() { + for (let i = 0; i < outputDatasets.length; i++) { + let outputDatasetNode = outputDatasets[i] + let dataset = outputDatasetNode.referenceParent + + if (dataset.config.type === 'Daily Files') { + outputDatasetsMap.set(dataset.parentNode.config.codeName, []) + } + } + } + + async function initializeMarketFiles() { + for (let i = 0; i < outputDatasets.length; i++) { + let outputDatasetNode = outputDatasets[i] + let dataset = outputDatasetNode.referenceParent + + if (dataset.config.type === 'Market Files') { + outputDatasetsMap.set(dataset.parentNode.config.codeName, []) + } + } + } + } + + async function loadFiles() { + /* + This bot have an output of files that it generates. At every call to the bot, it needs to read the previously generated + files in order to later append more information after the execution is over. Here in this function we are going to + read those output files and get them ready for appending content during the simulation. + */ + if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) { + await readDailyFiles() + } else { + await readMarketFiles() + } + } + + async function readMarketFiles() { + for (let i = 0; i < outputDatasets.length; i++) { + let outputDatasetNode = outputDatasets[i] + let dataset = outputDatasetNode.referenceParent + + if (dataset.config.type === 'Market Files') { + + let fileName = 'Data.json' + let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + + await readOutputFile(fileName, filePath, dataset.parentNode.config.codeName) + } + } + } + + async function readDailyFiles() { + for (let i = 0; i < outputDatasets.length; i++) { + let outputDatasetNode = outputDatasets[i] + let dataset = outputDatasetNode.referenceParent + + if (dataset.config.type === 'Daily Files') { + + let dateForPath = tradingProcessDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCDate(), 2); + let fileName = 'Data.json' + let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath + + await readOutputFile(fileName, filePath, dataset.parentNode.config.codeName) + } + } + } + + async function readOutputFile(fileName, filePath, productName) { + filePath += '/' + fileName + + let response = await fileStorage.asyncGetTextFile(filePath, true) + + if (response.err.message === 'File does not exist.') { + outputDatasetsMap.set(productName, []) + return + } + if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { + throw (response.err) + } + outputDatasetsMap.set(productName, JSON.parse(response.text)) + } + } + + async function writeFiles( + processIndex, + outputDatasets, + outputDatasetsMap, + timeFrameLabel, + tradingProcessDate, + fileStorage + ) { + /* + The output of files which were appended with information during the simulation execution, now needs to be saved. + */ + if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) { + await writeDailyFiles() + } else { + await writeMarketFiles() + } + + async function writeMarketFiles() { + for (let i = 0; i < outputDatasets.length; i++) { + let outputDatasetNode = outputDatasets[i] + let dataset = outputDatasetNode.referenceParent + + if (dataset.config.type === 'Market Files') { + + let fileName = 'Data.json' + let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + + await writeOutputFile(fileName, filePath, dataset.parentNode.config.codeName) + } + } + } + + async function writeDailyFiles() { + for (let i = 0; i < outputDatasets.length; i++) { + let outputDatasetNode = outputDatasets[i] + let dataset = outputDatasetNode.referenceParent + + if (dataset.config.type === 'Daily Files') { + + let dateForPath = tradingProcessDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCDate(), 2); + let fileName = 'Data.json' + let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath + + await writeOutputFile(fileName, filePath, dataset.parentNode.config.codeName) + } + } + } + + async function writeOutputFile(fileName, filePath, productName) { + filePath += '/' + fileName + let fileContent = JSON.stringify(outputDatasetsMap.get(productName)) + + let response = await fileStorage.asyncCreateTextFile(filePath, fileContent) + + if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { + throw (response.err) + } + } + } +} \ No newline at end of file diff --git a/Projects/Superalgos/TS/Function-Libraries/ProcessFilesFunctions.js b/Projects/Superalgos/TS/Function-Libraries/ProcessFilesFunctions.js index 0a0fdc9fd..4ad59c482 100644 --- a/Projects/Superalgos/TS/Function-Libraries/ProcessFilesFunctions.js +++ b/Projects/Superalgos/TS/Function-Libraries/ProcessFilesFunctions.js @@ -4,12 +4,13 @@ exports.newSuperalgosFunctionLibrariesProcessFilesFunctions = function () { process context variables before you do your work, and to save the process files once you are done. */ - const MODULE_NAME = "Process Files Function" + const MODULE_NAME = "Process Files Functions" let thisObject = { getContextVariables: getContextVariables, writeProcessFiles: writeProcessFiles } + return thisObject function getContextVariables(processIndex, contextVariables, statusDependencies, initialDatetime, callBackFunction) { try { @@ -222,5 +223,4 @@ exports.newSuperalgosFunctionLibrariesProcessFilesFunctions = function () { await thisReport.asyncSave() } } - return thisObject } \ No newline at end of file diff --git a/Projects/Superalgos/TS/Process-Modules/DataDependencies.js b/Projects/Superalgos/TS/Process-Modules/DataDependencies.js index 147fac5d3..786fb1507 100644 --- a/Projects/Superalgos/TS/Process-Modules/DataDependencies.js +++ b/Projects/Superalgos/TS/Process-Modules/DataDependencies.js @@ -113,8 +113,10 @@ we need to add the default exchange and market to these maps in order for the following sections to work. */ + thisObject.filters.market.list.set(thisObject.defaultMarket, true) + thisObject.filters.exchange.list.set(thisObject.defaultExchange, true) - thisObject.filters.exchange.markets.set(thisObject.defaultExchange + '-' + thisObject.defautMarket, true) + thisObject.filters.exchange.markets.set(thisObject.defaultExchange + '-' + thisObject.defaultMarket, true) } /* For each dependency declared at the curatedDependencyNodeArray, we will initialize a diff --git a/Projects/Superalgos/TS/Utilities/DateTimeFunctions.js b/Projects/Superalgos/TS/Utilities/DateTimeFunctions.js index 0dacf20b3..b922be9c4 100644 --- a/Projects/Superalgos/TS/Utilities/DateTimeFunctions.js +++ b/Projects/Superalgos/TS/Utilities/DateTimeFunctions.js @@ -14,9 +14,9 @@ exports.newSuperalgosUtilitiesDateTimeFunctions = function () { } function getPercentage(fromDate, currentDate, lastDate) { - let fromDays = Math.trunc(fromDate.valueOf() / TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) - let currentDays = Math.trunc(currentDate.valueOf() / TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) - let lastDays = Math.trunc((lastDate.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) / TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) + let fromDays = fromDate.valueOf() + let currentDays = currentDate.valueOf() + let lastDays = lastDate.valueOf() let percentage = (currentDays - fromDays) * 100 / (lastDays - fromDays) if ((lastDays - fromDays) === 0) { percentage = 100