From bc0ccd019c1096c5ff9eb0caa4be655c38c4b0d4 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 3 Feb 2021 11:40:05 +0100 Subject: [PATCH] Final Bug Fixes --- .../Low-Frequency-Trading/TradingProcess.js | 487 ++++++++++-------- .../UI/Spaces/Docs-Space/SearchEngine.js | 2 - 2 files changed, 279 insertions(+), 210 deletions(-) diff --git a/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingProcess.js b/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingProcess.js index 613f41d2f..f008588a4 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingProcess.js +++ b/Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingProcess.js @@ -331,224 +331,46 @@ } async function processSingleFiles() { - dataFiles = new Map() - for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.curatedDependencyNodeArray.length; dependencyIndex++) { - let dependency = dataDependenciesModule.curatedDependencyNodeArray[dependencyIndex] - let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex] - - if (datasetModule.node.config.codeName !== "Single-File") { - continue - } - - if (dataDependenciesModule.filters.exchange.timeFrames.get(datasetModule.exchange + '-' + datasetModule.market + '-' + datasetModule.product + '-' + 'atAnyTimeFrame') !== true) { - /* - If we can not find the current data set is used at the current time - frame we will skip this file. - */ - continue - } - - let fileName = "Data.json" - let filePath = datasetModule.node.parentNode.config.codeName + '/' + datasetModule.node.config.codeName - - /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */ - let response = await asyncGetDatasetFile(datasetModule, filePath, fileName) - - if (response.err.message === 'File does not exist.') { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] File not Found. This process will wait until you run your Data Mining and this file is created. File = " + filePath + '/' + fileName) - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] Your strategy depends on the dataset " + dependency.name + '.') - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] The reason that it depends on this dataset is becuase in one of your condition's Javascript Code node, you have written an expression that references this dataset. If you don't want to depend on this dataset, just locate that code and remove the mention.") - return false - } - - if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - throw (response.err) - } - - let dataFile = JSON.parse(response.text) - dataFiles.set(dependency.id, dataFile) - } - - let mapKey = "Single Files" - multiTimeFrameDataFiles.set(mapKey, dataFiles) - } - - async function processMarketFiles() { /* - We do market files first since if the simulation is run on daily files, there will - be a loop to get each of those files and we do not need that loop to reload market files. - - We will iterate through all posible timeFrames. + We will iterate through all the exchanges and markets involved. */ - for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; n++) { - const timeFrame = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][0] - const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1] + let exchangeList = Array.from(dataDependenciesModule.filters.exchange.list.keys()) + let marketList = Array.from(dataDependenciesModule.filters.market.list.keys()) - dataFiles = new Map() + for (let e = 0; e < exchangeList.length; e++) { + let currentExchange = exchangeList[e] + for (let m = 0; m < marketList.length; m++) { + let currentMarket = marketList[m] - /* Current Time Frame detection */ - if (TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeFrame.config.label === timeFrameLabel) { - currentTimeFrame = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][0] - currentTimeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1] - } + dataFiles = new Map() - /* Loop across the list of curated dependencies */ - for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.curatedDependencyNodeArray.length; dependencyIndex++) { - let dependency = dataDependenciesModule.curatedDependencyNodeArray[dependencyIndex] - let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex] + for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.curatedDependencyNodeArray.length; dependencyIndex++) { + let dependency = dataDependenciesModule.curatedDependencyNodeArray[dependencyIndex] + let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex] - if (dependency.referenceParent.config.codeName !== "Multi-Period-Market") { - continue - } - - if (dataDependenciesModule.filters.exchange.timeFrames.get(datasetModule.exchange + '-' + datasetModule.market + '-' + datasetModule.product + '-' + timeFrameLabel) !== true) { - /* - If we can not find the current data set is used at the current time - frame we will skip this file, unless we are in the only special - case that we are retrieving candles. - */ - if (!(TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { + if (datasetModule.node.config.codeName !== "Single-File") { continue } - } - let fileName = "Data.json" - let filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel - - /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */ - let response = await asyncGetDatasetFile(datasetModule, filePath, fileName) - - if (response.err.message === 'File does not exist.') { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] File not Found. This process will wait until you run your Data Mining and this file is created. File = " + filePath + '/' + fileName) - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] Your strategy depends on the dataset " + dependency.name + '.') - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] The reason that it depends on this dataset is becuase in one of your condition's Javascript Code node, you have written an expression that references this dataset. If you don't want to depend on this dataset, just locate that code and remove the mention.") - return false - } - - if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - throw (response.err) - } - - let dataFile = JSON.parse(response.text) - let trimmedDataFile = trimDataFile(dataFile, datasetModule.node.parentNode.record) - dataFiles.set(dependency.id, trimmedDataFile) - - function trimDataFile(dataFile, recordDefinition) { - /* - Here we will discard all the records in a file that are outside of the current time range. - We will include the las element previous to the begining of the time range. This is needed - because during the simulation, the current period is not the open one, but the previous to - the open, and if we do not include the previous to the initial datetime there will be no - current objects at the begining of the simulation for many time frames. - */ - let beginIndex - let endIndex - let result = [] - for (let i = 0; i < recordDefinition.properties.length; i++) { - let property = recordDefinition.properties[i] - if (property.config.codeName === 'begin') { - beginIndex = i - } - if (property.config.codeName === 'end') { - endIndex = i - } - } - for (let i = 0; i < dataFile.length; i++) { - let dataRecord = dataFile[i] - let begin = dataRecord[beginIndex] - let end = dataRecord[endIndex] - if (end + timeFrame < TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeRange.config.initialDatetime - 1) { continue } // /1 because we need the previous closed element - if (begin > TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeRange.config.finalDatetime) { continue } - result.push(dataRecord) - } - return result - } - } - - let mapKey = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1] - multiTimeFrameDataFiles.set(mapKey, dataFiles) - } - return true - } - - async function processDailyFiles() { - /* Telling the world we are alive and doing well and which date we are processing right now. */ - let processingDateString = tradingProcessDate.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCDate(), 2) - TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, processingDateString, undefined, "Running...") - - /* - We will iterate through all posible timeFrames. - */ - for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; n++) { - const timeFrame = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][0] - const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1] - - if (TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.framework.validtimeFrames !== undefined) { - let validPeriod = false - for (let i = 0; i < TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.framework.validtimeFrames.length; i++) { - let period = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.framework.validtimeFrames[i] - if (period === timeFrameLabel) { validPeriod = true } - } - if (validPeriod === false) { - continue - } - } - - if (TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeFrame.config.label === timeFrameLabel) { - currentTimeFrame = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][0] - currentTimeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1] - } - - dataFiles = new Map() - - /* - We will iterate through all dependencies, in order to load the - files that later will end up at the chart data structure. - */ - for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.curatedDependencyNodeArray.length; dependencyIndex++) { - let dependency = dataDependenciesModule.curatedDependencyNodeArray[dependencyIndex] - let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex] - - if (dependency.referenceParent.config.codeName !== "Multi-Period-Daily") { - continue - } - - if (dataDependenciesModule.filters.exchange.timeFrames.get(datasetModule.exchange + '-' + datasetModule.market + '-' + datasetModule.product + '-' + timeFrameLabel) !== true) { - /* - If we can not find the current data set is used at the current time - frame we will skip this file, unless we are in the only special - case that we are retrieving candles. - */ - if (!(TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { + if (datasetModule.exchange !== currentExchange) { continue } - } - /* - We will need to fetch the data of the current day and the previous day, in order for .previous properties in conditions and formulas to work well. - */ - let previousDate = new Date(tradingProcessDate.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) - let currentDate = new Date(tradingProcessDate.valueOf()) + if (datasetModule.market !== currentMarket) { + continue + } - let previousFile = await getDataFileFromDate(previousDate) - if (previousFile === false) { return false } - let currentFile = await getDataFileFromDate(currentDate) - if (currentFile === false) { return false } - let bothFiles = previousFile.concat(currentFile) - dataFiles.set(dependency.id, bothFiles) + if (dataDependenciesModule.filters.exchange.timeFrames.get(datasetModule.exchange + '-' + datasetModule.market + '-' + datasetModule.product + '-' + 'atAnyTimeFrame') !== true) { + /* + If we can not find the current data set is used at the current time + frame we will skip this file. + */ + continue + } - async function getDataFileFromDate(processDate) { - - let dateForPath = processDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processDate.getUTCDate(), 2) - let filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel + "/" + dateForPath let fileName = "Data.json" + let filePath = datasetModule.node.parentNode.config.codeName + '/' + datasetModule.node.config.codeName /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */ let response = await asyncGetDatasetFile(datasetModule, filePath, fileName) @@ -562,15 +384,264 @@ "[ERROR] The reason that it depends on this dataset is becuase in one of your condition's Javascript Code node, you have written an expression that references this dataset. If you don't want to depend on this dataset, just locate that code and remove the mention.") return false } + if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { throw (response.err) } - return JSON.parse(response.text) + let dataFile = JSON.parse(response.text) + dataFiles.set(dependency.id, dataFile) + } + + let mapKey = currentExchange + '-' + currentMarket + '-' + "Single Files" + multiTimeFrameDataFiles.set(mapKey, dataFiles) + } + } + } + + async function processMarketFiles() { + /* + We do market files first since if the simulation is run on daily files, there will + be a loop to get each of those files and we do not need that loop to reload market files. + */ + + /* + We will iterate through all the exchanges and markets involved. + */ + let exchangeList = Array.from(dataDependenciesModule.filters.exchange.list.keys()) + let marketList = Array.from(dataDependenciesModule.filters.market.list.keys()) + + for (let e = 0; e < exchangeList.length; e++) { + let currentExchange = exchangeList[e] + for (let m = 0; m < marketList.length; m++) { + let currentMarket = marketList[m] + /* + We will iterate through all posible timeFrames. + */ + for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; n++) { + const timeFrame = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][0] + const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1] + + dataFiles = new Map() + + /* Current Time Frame detection */ + if (TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeFrame.config.label === timeFrameLabel) { + currentTimeFrame = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][0] + currentTimeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1] + } + + /* Loop across the list of curated dependencies */ + for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.curatedDependencyNodeArray.length; dependencyIndex++) { + let dependency = dataDependenciesModule.curatedDependencyNodeArray[dependencyIndex] + let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex] + + if (dependency.referenceParent.config.codeName !== "Multi-Period-Market") { + continue + } + + if (datasetModule.exchange !== currentExchange) { + continue + } + + if (datasetModule.market !== currentMarket) { + continue + } + + if (dataDependenciesModule.filters.exchange.timeFrames.get(datasetModule.exchange + '-' + datasetModule.market + '-' + datasetModule.product + '-' + timeFrameLabel) !== true) { + /* + If we can not find the current data set is used at the current time + frame we will skip this file, unless we are in the only special + case that we are retrieving candles. + */ + if (!(TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { + continue + } + if (currentExchange !== dataDependenciesModule.defaultExchange || currentMarket !== dataDependenciesModule.defaultMarket) { + continue + } + } + + let fileName = "Data.json" + let filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel + + /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */ + let response = await asyncGetDatasetFile(datasetModule, filePath, fileName) + + if (response.err.message === 'File does not exist.') { + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[ERROR] File not Found. This process will wait until you run your Data Mining and this file is created. File = " + filePath + '/' + fileName) + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[ERROR] Your strategy depends on the dataset " + dependency.name + '.') + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[ERROR] The reason that it depends on this dataset is becuase in one of your condition's Javascript Code node, you have written an expression that references this dataset. If you don't want to depend on this dataset, just locate that code and remove the mention.") + return false + } + + if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { + throw (response.err) + } + + let dataFile = JSON.parse(response.text) + let trimmedDataFile = trimDataFile(dataFile, datasetModule.node.parentNode.record) + dataFiles.set(dependency.id, trimmedDataFile) + + function trimDataFile(dataFile, recordDefinition) { + /* + Here we will discard all the records in a file that are outside of the current time range. + We will include the las element previous to the begining of the time range. This is needed + because during the simulation, the current period is not the open one, but the previous to + the open, and if we do not include the previous to the initial datetime there will be no + current objects at the begining of the simulation for many time frames. + */ + let beginIndex + let endIndex + let result = [] + for (let i = 0; i < recordDefinition.properties.length; i++) { + let property = recordDefinition.properties[i] + if (property.config.codeName === 'begin') { + beginIndex = i + } + if (property.config.codeName === 'end') { + endIndex = i + } + } + for (let i = 0; i < dataFile.length; i++) { + let dataRecord = dataFile[i] + let begin = dataRecord[beginIndex] + let end = dataRecord[endIndex] + if (end + timeFrame < TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeRange.config.initialDatetime - 1) { continue } // /1 because we need the previous closed element + if (begin > TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeRange.config.finalDatetime) { continue } + result.push(dataRecord) + } + return result + } + } + + let mapKey = currentExchange + '-' + currentMarket + '-' + TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1] + multiTimeFrameDataFiles.set(mapKey, dataFiles) } } - let mapKey = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1] - multiTimeFrameDataFiles.set(mapKey, dataFiles) + } + return true + } + + async function processDailyFiles() { + /* Telling the world we are alive and doing well and which date we are processing right now. */ + let processingDateString = tradingProcessDate.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCDate(), 2) + TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, processingDateString, undefined, "Running...") + + /* + We will iterate through all the exchanges and markets involved. + */ + let exchangeList = Array.from(dataDependenciesModule.filters.exchange.list.keys()) + let marketList = Array.from(dataDependenciesModule.filters.market.list.keys()) + + for (let e = 0; e < exchangeList.length; e++) { + let currentExchange = exchangeList[e] + for (let m = 0; m < marketList.length; m++) { + let currentMarket = marketList[m] + /* + We will iterate through all posible timeFrames. + */ + for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; n++) { + const timeFrame = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][0] + const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1] + + if (TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.framework.validtimeFrames !== undefined) { + let validPeriod = false + for (let i = 0; i < TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.framework.validtimeFrames.length; i++) { + let period = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.framework.validtimeFrames[i] + if (period === timeFrameLabel) { validPeriod = true } + } + if (validPeriod === false) { + continue + } + } + + if (TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeFrame.config.label === timeFrameLabel) { + currentTimeFrame = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][0] + currentTimeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1] + } + + dataFiles = new Map() + + /* + We will iterate through all dependencies, in order to load the + files that later will end up at the chart data structure. + */ + for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.curatedDependencyNodeArray.length; dependencyIndex++) { + let dependency = dataDependenciesModule.curatedDependencyNodeArray[dependencyIndex] + let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex] + + if (dependency.referenceParent.config.codeName !== "Multi-Period-Daily") { + continue + } + + if (datasetModule.exchange !== currentExchange) { + continue + } + + if (datasetModule.market !== currentMarket) { + continue + } + + if (dataDependenciesModule.filters.exchange.timeFrames.get(datasetModule.exchange + '-' + datasetModule.market + '-' + datasetModule.product + '-' + timeFrameLabel) !== true) { + /* + If we can not find the current data set is used at the current time + frame we will skip this file, unless we are in the only special + case that we are retrieving candles. + */ + if (!(TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { + continue + } + if (currentExchange !== dataDependenciesModule.defaultExchange || currentMarket !== dataDependenciesModule.defaultMarket) { + continue + } + } + + /* + We will need to fetch the data of the current day and the previous day, in order for .previous properties in conditions and formulas to work well. + */ + let previousDate = new Date(tradingProcessDate.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) + let currentDate = new Date(tradingProcessDate.valueOf()) + + let previousFile = await getDataFileFromDate(previousDate) + if (previousFile === false) { return false } + let currentFile = await getDataFileFromDate(currentDate) + if (currentFile === false) { return false } + let bothFiles = previousFile.concat(currentFile) + dataFiles.set(dependency.id, bothFiles) + + async function getDataFileFromDate(processDate) { + + let dateForPath = processDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processDate.getUTCDate(), 2) + let filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel + "/" + dateForPath + let fileName = "Data.json" + + /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */ + let response = await asyncGetDatasetFile(datasetModule, filePath, fileName) + + if (response.err.message === 'File does not exist.') { + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[ERROR] File not Found. This process will wait until you run your Data Mining and this file is created. File = " + filePath + '/' + fileName) + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[ERROR] Your strategy depends on the dataset " + dependency.name + '.') + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[ERROR] The reason that it depends on this dataset is becuase in one of your condition's Javascript Code node, you have written an expression that references this dataset. If you don't want to depend on this dataset, just locate that code and remove the mention.") + return false + } + if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { + throw (response.err) + } + + return JSON.parse(response.text) + } + } + let mapKey = currentExchange + '-' + currentMarket + '-' + TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1] + multiTimeFrameDataFiles.set(mapKey, dataFiles) + } + + } } return true } @@ -655,7 +726,7 @@ /* Market Files */ for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; j++) { let timeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[j][1] - let dataFiles = multiTimeFrameDataFiles.get(timeFrameLabel) + let dataFiles = multiTimeFrameDataFiles.get(currentExchange + '-' + currentMarket + '-' + timeFrameLabel) let products = {} if (dataFiles !== undefined && dataFiles.size > 0) { @@ -669,7 +740,7 @@ /* Daily Files */ for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; j++) { let timeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[j][1] - let dataFiles = multiTimeFrameDataFiles.get(timeFrameLabel) + let dataFiles = multiTimeFrameDataFiles.get(currentExchange + '-' + currentMarket + '-' + timeFrameLabel) let products = {} if (dataFiles !== undefined && dataFiles.size > 0) { @@ -682,7 +753,7 @@ /* Single Files */ { - let dataFiles = multiTimeFrameDataFiles.get('Single Files') + let dataFiles = multiTimeFrameDataFiles.get(currentExchange + '-' + currentMarket + '-' + 'Single Files') let products = {} if (dataFiles !== undefined && dataFiles.size > 0) { diff --git a/Projects/Superalgos/UI/Spaces/Docs-Space/SearchEngine.js b/Projects/Superalgos/UI/Spaces/Docs-Space/SearchEngine.js index ea44ac22f..101ce7823 100644 --- a/Projects/Superalgos/UI/Spaces/Docs-Space/SearchEngine.js +++ b/Projects/Superalgos/UI/Spaces/Docs-Space/SearchEngine.js @@ -17,8 +17,6 @@ function newSuperalgosDocsSearchEngine() { } function setUpSearchEngine(callbackFunction) { - callbackFunction() - return let totalAsyncCallsMade = 0 let totalAsyncCallsFinished = 0