From d398f1cbb40fd85d46576f4d89d5e3fa7c54d365 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 18 Aug 2020 11:16:35 +0200 Subject: [PATCH] Trading Process RE-implemented --- FileStorage.js | 43 +- LowFrequencyTrading/TradingEpisode.js | 137 ++-- LowFrequencyTrading/TradingPosition.js | 90 ++- LowFrequencyTrading/TradingProcess.js | 847 +++++++++++-------------- StatusReport.js | 22 +- 5 files changed, 565 insertions(+), 574 deletions(-) diff --git a/FileStorage.js b/FileStorage.js index 7cb2bf26b..575df8a7d 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -17,6 +17,8 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { } let thisObject = { + asyncGetTextFile: asyncGetTextFile, + asyncCreateTextFile: asyncCreateTextFile, getTextFile: getTextFile, createTextFile: createTextFile, deleteTextFile: deleteTextFile @@ -24,6 +26,41 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { return thisObject + async function asyncGetTextFile(filePath, noRetry, canUsePrevious) { + /* This function allows its caller to work with async / await instead of callbacks */ + let promise = new Promise((resolve, reject) => { + + getTextFile(filePath, onFileRead, noRetry, canUsePrevious) + function onFileRead(err, text) { + + let response = { + err: err, + text: text + } + resolve(response) + } + }) + + return promise + } + + async function asyncCreateTextFile(filePath, fileContent, keepPrevious, noTemp) { + /* This function allows its caller to work with async / await instead of callbacks */ + let promise = new Promise((resolve, reject) => { + + createTextFile(filePath, fileContent, onFileWriten, keepPrevious, noTemp) + function onFileWriten(err) { + + let response = { + err: err + } + resolve(response) + } + }) + + return promise + } + function getTextFile(filePath, callBackFunction, noRetry, canUsePrevious) { let currentRetryGetTextFile = 0 @@ -216,12 +253,12 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { */ const fs = require('fs') if (noTemp === true) { - fs.writeFile(fileLocation, fileContent, onFileWritenn) + fs.writeFile(fileLocation, fileContent, onFileWriten) } else { - fs.writeFile(fileLocation + '.tmp', fileContent, onFileWritenn) + fs.writeFile(fileLocation + '.tmp', fileContent, onFileWriten) } - function onFileWritenn(err) { + function onFileWriten(err) { let retryTimeToUse = FAST_RETRY_TIME_IN_MILISECONDS if (currentRetryWriteTextFile > MAX_RETRY - 2) { retryTimeToUse = SLOW_RETRY_TIME_IN_MILISECONDS diff --git a/LowFrequencyTrading/TradingEpisode.js b/LowFrequencyTrading/TradingEpisode.js index 96b6e5adc..f6d544188 100644 --- a/LowFrequencyTrading/TradingEpisode.js +++ b/LowFrequencyTrading/TradingEpisode.js @@ -113,46 +113,10 @@ exports.newTradingEpisode = function newTradingEpisode(bot, logger, tradingEngin global.ONE_DAY_IN_MILISECONDS } - function calculateStatistics() { - calculateEpisodeStatistics() - calculateAssetsStatistics() + function calculateStatistics() { - function calculateEpisodeStatistics() { - /* Updating Profit Loss */ - tradingEngine.current.episode.episodeStatistics.profitLoss.value = - tradingEngine.current.episode.episodeBaseAsset.profitLoss.value * tradingEngine.current.episode.candle.close.value + - tradingEngine.current.episode.episodeQuotedAsset.profitLoss.value - - tradingEngine.current.episode.episodeStatistics.profitLoss.value = global.PRECISE(tradingEngine.current.episode.episodeStatistics.profitLoss.value, 10) - - /* Updating ROI */ - tradingEngine.current.episode.episodeStatistics.ROI.value = - ( - sessionParameters.sessionBaseAsset.config.initialBalance * tradingEngine.current.episode.candle.close.value + - tradingEngine.current.episode.episodeBaseAsset.profitLoss.value * tradingEngine.current.episode.candle.close.value + - sessionParameters.sessionQuotedAsset.config.initialBalance + - tradingEngine.current.episode.episodeQuotedAsset.profitLoss.value - ) / ( - sessionParameters.sessionBaseAsset.config.initialBalance * tradingEngine.current.episode.candle.close.value + - sessionParameters.sessionQuotedAsset.config.initialBalance - ) - 1 - - tradingEngine.current.episode.episodeStatistics.ROI.value = global.PRECISE(tradingEngine.current.episode.episodeStatistics.ROI.value, 10) - - /* Updating Annualized Rate Of Return */ - tradingEngine.current.episode.episodeStatistics.annualizedRateOfReturn.value = - tradingEngine.current.episode.episodeStatistics.ROI.value / - tradingEngine.current.episode.episodeStatistics.days.value * 365 - - tradingEngine.current.episode.episodeStatistics.annualizedRateOfReturn.value = global.PRECISE(tradingEngine.current.episode.episodeStatistics.annualizedRateOfReturn.value, 10) - - /* Updating Hit or Fail */ - if (tradingEngine.current.episode.episodeStatistics.profitLoss.value > 0) { - tradingEngine.current.episode.episodeStatistics.hitFail.value = 'Hit' - } else { - tradingEngine.current.episode.episodeStatistics.hitFail.value = 'Fail' - } - } + calculateAssetsStatistics() + calculateEpisodeStatistics() function calculateAssetsStatistics() { @@ -180,14 +144,18 @@ exports.newTradingEpisode = function newTradingEpisode(bot, logger, tradingEngin tradingEngine.current.episode.episodeBaseAsset.profitLoss.value = global.PRECISE(tradingEngine.current.episode.episodeBaseAsset.profitLoss.value, 10) tradingEngine.current.episode.episodeQuotedAsset.profitLoss.value = global.PRECISE(tradingEngine.current.episode.episodeQuotedAsset.profitLoss.value, 10) - /* Updating ROI */ + /* + Updating ROI + + https://www.investopedia.com/articles/basics/10/guide-to-calculating-roi.asp + */ tradingEngine.current.episode.episodeBaseAsset.ROI.value = - (sessionParameters.sessionBaseAsset.config.initialBalance + tradingEngine.current.episode.episodeBaseAsset.profitLoss.value) / - sessionParameters.sessionBaseAsset.config.initialBalance - 1 + tradingEngine.current.episode.episodeBaseAsset.profitLoss.value / + tradingEngine.current.episode.episodeBaseAsset.beginBalance.value * 100 tradingEngine.current.episode.episodeQuotedAsset.ROI.value = - (sessionParameters.sessionQuotedAsset.config.initialBalance + tradingEngine.current.episode.episodeQuotedAsset.profitLoss.value) / - sessionParameters.sessionQuotedAsset.config.initialBalance - 1 + tradingEngine.current.episode.episodeQuotedAsset.profitLoss.value / + tradingEngine.current.episode.episodeQuotedAsset.beginBalance.value * 100 tradingEngine.current.episode.episodeBaseAsset.ROI.value = global.PRECISE(tradingEngine.current.episode.episodeBaseAsset.ROI.value, 10) tradingEngine.current.episode.episodeQuotedAsset.ROI.value = global.PRECISE(tradingEngine.current.episode.episodeQuotedAsset.ROI.value, 10) @@ -204,14 +172,30 @@ exports.newTradingEpisode = function newTradingEpisode(bot, logger, tradingEngin tradingEngine.current.episode.episodeBaseAsset.hitRatio.value = global.PRECISE(tradingEngine.current.episode.episodeBaseAsset.hitRatio.value, 10) tradingEngine.current.episode.episodeQuotedAsset.hitRatio.value = global.PRECISE(tradingEngine.current.episode.episodeQuotedAsset.hitRatio.value, 10) - /* Updating Annualized Rate Of Return */ + /* + Updating Annualized Rate Of Return + + https://www.investopedia.com/terms/a/annualized-rate.asp + */ tradingEngine.current.episode.episodeBaseAsset.annualizedRateOfReturn.value = - tradingEngine.current.episode.episodeBaseAsset.ROI.value / - tradingEngine.current.episode.episodeStatistics.days.value * 365 + Math.pow( + ( + tradingEngine.current.episode.episodeBaseAsset.beginBalance.value + + tradingEngine.current.episode.episodeBaseAsset.profitLoss.value + ) / tradingEngine.current.episode.episodeBaseAsset.beginBalance.value + , + (365 / tradingEngine.current.episode.episodeStatistics.days.value) + ) - 1 tradingEngine.current.episode.episodeQuotedAsset.annualizedRateOfReturn.value = - tradingEngine.current.episode.episodeQuotedAsset.ROI.value / - tradingEngine.current.episode.episodeStatistics.days.value * 365 + Math.pow( + ( + tradingEngine.current.episode.episodeQuotedAsset.beginBalance.value + + tradingEngine.current.episode.episodeQuotedAsset.profitLoss.value + ) / tradingEngine.current.episode.episodeQuotedAsset.beginBalance.value + , + (365 / tradingEngine.current.episode.episodeStatistics.days.value) + ) - 1 tradingEngine.current.episode.episodeBaseAsset.annualizedRateOfReturn.value = global.PRECISE(tradingEngine.current.episode.episodeBaseAsset.annualizedRateOfReturn.value, 10) tradingEngine.current.episode.episodeQuotedAsset.annualizedRateOfReturn.value = global.PRECISE(tradingEngine.current.episode.episodeQuotedAsset.annualizedRateOfReturn.value, 10) @@ -228,6 +212,61 @@ exports.newTradingEpisode = function newTradingEpisode(bot, logger, tradingEngin tradingEngine.current.episode.episodeQuotedAsset.hitFail.value = 'Fail' } } + + function calculateEpisodeStatistics() { + /* Updating Profit Loss */ + tradingEngine.current.episode.episodeStatistics.profitLoss.value = + tradingEngine.current.episode.episodeBaseAsset.profitLoss.value * tradingEngine.current.episode.candle.close.value + + tradingEngine.current.episode.episodeQuotedAsset.profitLoss.value + + tradingEngine.current.episode.episodeStatistics.profitLoss.value = global.PRECISE(tradingEngine.current.episode.episodeStatistics.profitLoss.value, 10) + + /* + Updating ROI + + https://www.investopedia.com/articles/basics/10/guide-to-calculating-roi.asp + */ + tradingEngine.current.episode.episodeStatistics.ROI.value = + ( + tradingEngine.current.episode.episodeBaseAsset.profitLoss.value * tradingEngine.current.episode.endRate.value + + tradingEngine.current.episode.episodeQuotedAsset.profitLoss.value + ) / ( + tradingEngine.current.episode.episodeBaseAsset.beginBalance.value * tradingEngine.current.episode.beginRate.value + + tradingEngine.current.episode.episodeQuotedAsset.beginBalance.value + ) * 100 + + tradingEngine.current.episode.episodeStatistics.ROI.value = global.PRECISE(tradingEngine.current.episode.episodeStatistics.ROI.value, 10) + + /* + Updating Annualized Rate Of Return + + https://www.investopedia.com/terms/a/annualized-rate.asp + */ + tradingEngine.current.episode.episodeStatistics.annualizedRateOfReturn.value = + Math.pow( + ( + tradingEngine.current.episode.episodeBaseAsset.beginBalance.value * tradingEngine.current.episode.beginRate.value + + tradingEngine.current.episode.episodeQuotedAsset.beginBalance.value + + tradingEngine.current.episode.episodeBaseAsset.profitLoss.value + + tradingEngine.current.episode.episodeQuotedAsset.profitLoss.value + ) / + ( + tradingEngine.current.episode.episodeBaseAsset.beginBalance.value * tradingEngine.current.episode.beginRate.value + + tradingEngine.current.episode.episodeQuotedAsset.beginBalance.value + ) + , + (365 / tradingEngine.current.episode.episodeStatistics.days.value) + ) - 1 + + tradingEngine.current.episode.episodeStatistics.annualizedRateOfReturn.value = global.PRECISE(tradingEngine.current.episode.episodeStatistics.annualizedRateOfReturn.value, 10) + + /* Updating Hit or Fail */ + if (tradingEngine.current.episode.episodeStatistics.profitLoss.value > 0) { + tradingEngine.current.episode.episodeStatistics.hitFail.value = 'Hit' + } else { + tradingEngine.current.episode.episodeStatistics.hitFail.value = 'Fail' + } + } } diff --git a/LowFrequencyTrading/TradingPosition.js b/LowFrequencyTrading/TradingPosition.js index c5606741a..98c131fa2 100644 --- a/LowFrequencyTrading/TradingPosition.js +++ b/LowFrequencyTrading/TradingPosition.js @@ -257,20 +257,56 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng } function calculateStatistics() { - calculatePositionStatistics() + calculateAssetsStatistics() + calculatePositionStatistics() + + function calculateAssetsStatistics() { + /* Profit Loss Calculation */ + tradingEngine.current.position.positionBaseAsset.profitLoss.value = + tradingEngine.current.episode.episodeBaseAsset.balance.value - + tradingEngine.current.position.positionBaseAsset.beginBalance + + tradingEngine.current.position.positionQuotedAsset.profitLoss.value = + tradingEngine.current.episode.episodeQuotedAsset.balance.value - + tradingEngine.current.position.positionQuotedAsset.beginBalance + + tradingEngine.current.position.positionBaseAsset.profitLoss.value = global.PRECISE(tradingEngine.current.position.positionBaseAsset.profitLoss.value, 10) + tradingEngine.current.position.positionQuotedAsset.profitLoss.value = global.PRECISE(tradingEngine.current.position.positionQuotedAsset.profitLoss.value, 10) + + /* ROI Calculation */ + tradingEngine.current.position.positionBaseAsset.ROI.value = + tradingEngine.current.position.positionBaseAsset.profitLoss.value * 100 / + tradingEngine.current.position.positionBaseAsset.sizeFilled.value + + tradingEngine.current.position.positionQuotedAsset.ROI.value = + tradingEngine.current.position.positionQuotedAsset.profitLoss.value * 100 / + tradingEngine.current.position.positionQuotedAsset.sizeFilled.value + + tradingEngine.current.position.positionBaseAsset.ROI.value = global.PRECISE(tradingEngine.current.position.positionBaseAsset.ROI.value, 10) + tradingEngine.current.position.positionQuotedAsset.ROI.value = global.PRECISE(tradingEngine.current.position.positionQuotedAsset.ROI.value, 10) + + /* Hit Fail Calculation */ + if (tradingEngine.current.position.positionBaseAsset.ROI.value > 0) { + tradingEngine.current.position.positionBaseAsset.hitFail.value = 'Hit' + } else { + tradingEngine.current.position.positionBaseAsset.hitFail.value = 'Fail' + } + if (tradingEngine.current.position.positionQuotedAsset.ROI.value > 0) { + tradingEngine.current.position.positionQuotedAsset.hitFail.value = 'Hit' + } else { + tradingEngine.current.position.positionQuotedAsset.hitFail.value = 'Fail' + } + } function calculatePositionStatistics() { /* Profit Loss Calculation */ tradingEngine.current.position.positionStatistics.profitLoss.value = ( - tradingEngine.current.episode.episodeBaseAsset.balance.value * tradingEngine.current.position.endRate.value + - tradingEngine.current.episode.episodeQuotedAsset.balance.value - ) - - ( - tradingEngine.current.position.positionBaseAsset.beginBalance * tradingEngine.current.position.beginRate.value + - tradingEngine.current.position.positionQuotedAsset.beginBalance - ) + tradingEngine.current.episode.episodeBaseAsset.profitLoss.value * tradingEngine.current.position.endRate.value + + tradingEngine.current.episode.episodeQuotedAsset.profitLoss.value + ) + tradingEngine.current.position.positionStatistics.profitLoss.value = global.PRECISE(tradingEngine.current.position.positionStatistics.profitLoss.value, 10) /* ROI Calculation */ @@ -299,44 +335,6 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng tradingEngine.current.position.positionStatistics.days.value = global.PRECISE(tradingEngine.current.position.positionStatistics.days.value, 10) } - - function calculateAssetsStatistics() { - /* Profit Loss Calculation */ - tradingEngine.current.position.positionBaseAsset.profitLoss.value = - tradingEngine.current.episode.episodeBaseAsset.balance.value - - tradingEngine.current.position.positionBaseAsset.beginBalance - - tradingEngine.current.position.positionQuotedAsset.profitLoss.value = - tradingEngine.current.episode.episodeQuotedAsset.balance.value - - tradingEngine.current.position.positionQuotedAsset.beginBalance - - tradingEngine.current.position.positionBaseAsset.profitLoss.value = global.PRECISE(tradingEngine.current.position.positionBaseAsset.profitLoss.value, 10) - tradingEngine.current.position.positionQuotedAsset.profitLoss.value = global.PRECISE(tradingEngine.current.position.positionQuotedAsset.profitLoss.value, 10) - - /* ROI Calculation */ - tradingEngine.current.position.positionBaseAsset.ROI.value = - tradingEngine.current.position.positionBaseAsset.profitLoss.value * 100 / - tradingEngine.current.position.positionBaseAsset.entryTargetSize.value - - tradingEngine.current.position.positionQuotedAsset.ROI.value = - tradingEngine.current.position.positionQuotedAsset.profitLoss.value * 100 / - tradingEngine.current.position.positionQuotedAsset.entryTargetSize.value - - tradingEngine.current.position.positionBaseAsset.ROI.value = global.PRECISE(tradingEngine.current.position.positionBaseAsset.ROI.value, 10) - tradingEngine.current.position.positionQuotedAsset.ROI.value = global.PRECISE(tradingEngine.current.position.positionQuotedAsset.ROI.value, 10) - - /* Hit Fail Calculation */ - if (tradingEngine.current.position.positionBaseAsset.ROI.value > 0) { - tradingEngine.current.position.positionBaseAsset.hitFail.value = 'Hit' - } else { - tradingEngine.current.position.positionBaseAsset.hitFail.value = 'Fail' - } - if (tradingEngine.current.position.positionQuotedAsset.ROI.value > 0) { - tradingEngine.current.position.positionQuotedAsset.hitFail.value = 'Hit' - } else { - tradingEngine.current.position.positionQuotedAsset.hitFail.value = 'Fail' - } - } } function badDefinitionUnhandledException(err, message, node) { diff --git a/LowFrequencyTrading/TradingProcess.js b/LowFrequencyTrading/TradingProcess.js index 7e66fe1e9..48ca5e79b 100644 --- a/LowFrequencyTrading/TradingProcess.js +++ b/LowFrequencyTrading/TradingProcess.js @@ -4,7 +4,6 @@ After execution, will save the time range and status report of the process. */ const FULL_LOG = true; - const LOG_FILE_CONTENT = false; const MODULE_NAME = "Trading Process"; const GMT_SECONDS = ':00.000 GMT+0000'; @@ -61,7 +60,7 @@ thisObject = undefined } - function start(callBackFunction) { + async function start(callBackFunction) { try { let currentTimeFrame let currentTimeFrameLabel @@ -77,7 +76,53 @@ let previousDay; // Holds the date of the previous day relative to the processing date. - getContextVariables(); + getContextVariables() + await processSingleFiles() + await processMarketFiles() + + /* + Here we check if we need to get Daily Files or not. As an optimization, when + we are running on a Time Frame of 1hs or above, we are not going to load + dependencies on Daily Files. The way we recognize that is by checking if + we alreaady set a value to currentTimeFrame. We are also not going to loop + through days if we are processing market files. + */ + + if (currentTimeFrame) { + /* We are processing Market Files */ + + let chart = {} + buildCharts(chart) + + await generateOutput(chart) + await writeProcessFiles() + + checkIfSessionMustStop() + + } else { + /* We are processing Daily Files */ + + /* + Go back one day to start well when we advance time at the begining of the loop. + */ + bot.tradingProcessDate = new Date(contextVariables.lastFile.valueOf() - global.ONE_DAY_IN_MILISECONDS); + do { + if (advanceTime() === false) {break} + if (checkStopTaskGracefully() === false) {break} + if (checkStopProcessing() === false) {break} + + await processDailyFiles() + + let chart = {} + buildCharts(chart) + + await generateOutput(chart) + await writeProcessFiles() + } + while (true) + } + + callBackFunction(global.DEFAULT_OK_RESPONSE) function getContextVariables() { try { @@ -96,13 +141,13 @@ if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; + return } if (statusReport.status === "Status Report is corrupt.") { logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; + return } thisReport = statusReport.file; @@ -116,7 +161,7 @@ } logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); callBackFunction(customOK); - return; + return } contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf()); @@ -128,7 +173,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> You need to UPDATE your datasets in order to run a " + bot.SESSION.type) } callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; + return } @@ -138,13 +183,13 @@ if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; + return } if (statusReport.status === "Status Report is corrupt.") { logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process."); callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; + return } thisReport = statusReport.file; @@ -155,11 +200,10 @@ if (bot.RESUME !== true) { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Starting from the begining because bot has just started and resume execution was true."); } startFromBegining(); - return; + return } contextVariables.lastFile = new Date(thisReport.lastFile); - processSingleFiles(); - return; + return } else { /* @@ -169,14 +213,13 @@ */ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Starting from the begining of the market because own status report not found or lastFile was undefined."); } startFromBegining(); - return; + return } function startFromBegining() { contextVariables.lastFile = new Date(contextVariables.dateBeginOfMarket.getUTCFullYear() + "-" + (contextVariables.dateBeginOfMarket.getUTCMonth() + 1) + "-" + contextVariables.dateBeginOfMarket.getUTCDate() + " " + "00:00" + GMT_SECONDS); if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> startFromBegining -> contextVariables.lastFile = " + contextVariables.lastFile); } - processSingleFiles(); } } catch (err) { @@ -189,242 +232,158 @@ } } - function processSingleFiles() { - let dependencyIndex = 0; + async function processSingleFiles() { dataFiles = new Map(); - dependencyLoopBody(); - function dependencyLoopBody() { + for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.nodeArray.length; dependencyIndex++) { let dependency = dataDependenciesModule.nodeArray[dependencyIndex]; let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]; - getFile(); + + if (datasetModule.node.config.codeName !== "Single-File") { + continue + } - function getFile() { - if (datasetModule.node.config.codeName !== "Single-File") { - dependencyControlLoop(); - return - } + if (dataDependenciesModule.isItADepenency('atAnyTimeFrame', datasetModule.node.parentNode.config.singularVariableName) !== true) { + continue + } - if (dataDependenciesModule.isItADepenency('atAnyTimeFrame', datasetModule.node.parentNode.config.singularVariableName) !== true) { - dependencyControlLoop(); - return - } + let fileName = "Data.json"; + let filePath = datasetModule.node.parentNode.config.codeName + '/' + datasetModule.node.config.codeName; - let fileName = "Data.json"; - let filePath = datasetModule.node.parentNode.config.codeName + '/' + datasetModule.node.config.codeName; - datasetModule.getTextFile(filePath, fileName, onFileReceived); + /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */ + let response = await asyncGetDatasetFile(datasetModule, filePath, fileName) + onFileReceived(response.err, response.text) - function onFileReceived(err, text) { - try { - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - if (err.message === 'File does not exist.') { - logger.write(MODULE_NAME, "[ERROR] The file " + filePath + '/' + fileName + ' does not exist and it is required to continue. This process will retry to read it in a while. In the meantime make yourself sure that the process that generates it has ran properly.'); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - } else { - logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message); - callBackFunction(err); - } - return; + function onFileReceived(err, text) { + try { + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + if (err.message === 'File does not exist.') { + logger.write(MODULE_NAME, "[ERROR] The file " + filePath + '/' + fileName + ' does not exist and it is required to continue. This process will retry to read it in a while. In the meantime make yourself sure that the process that generates it has ran properly.'); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + } else { + logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> onFileReceived -> err = " + err.message); + callBackFunction(err); } - - let dataFile = JSON.parse(text); - dataFiles.set(dependency.id, dataFile); - - dependencyControlLoop(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return } + + let dataFile = JSON.parse(text); + dataFiles.set(dependency.id, dataFile); + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> onFileReceived -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } - function dependencyControlLoop() { - dependencyIndex++ - if (dependencyIndex < dataDependenciesModule.nodeArray.length) { - dependencyLoopBody() - } else { - let mapKey = "Single Files" - multiPeriodDataFiles.set(mapKey, dataFiles) - processMarketFiles() - } - } + let mapKey = "Single Files" + multiPeriodDataFiles.set(mapKey, dataFiles) } - function processMarketFiles() { + async function processMarketFiles() { /* - We do market files first since if the simulation is run on daily files, there will be a loop to getch each daus files and we do not need - that loop to reload market files. + We do market files first since if the simulation is run on daily files, there will + be a loop to get each of those files and we do not need that loop to reload market files. + + We will iterate through all posible timeFrames. */ - let n; - timeFramesLoop(); - - function timeFramesLoop() { - /* - We will iterate through all posible timeFrames. - */ - n = 0 // loop Variable representing each possible period as defined at the timeFrames array. - timeFramesLoopBody(); - } - - function timeFramesLoopBody() { + for (let n = 0; n < global.marketFilesPeriods.length; n++) { const timeFrame = global.marketFilesPeriods[n][0]; const timeFrameLabel = global.marketFilesPeriods[n][1]; - let dependencyIndex = 0; dataFiles = new Map(); if (bot.SESSION.parameters.timeFrame.config.label === timeFrameLabel) { currentTimeFrame = global.marketFilesPeriods[n][0]; currentTimeFrameLabel = global.marketFilesPeriods[n][1]; } - dependencyLoopBody(); - - function dependencyLoopBody() { + for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.nodeArray.length; dependencyIndex++) { let dependency = dataDependenciesModule.nodeArray[dependencyIndex]; let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]; - getFile(); - function getFile() { - if (dependency.referenceParent.config.codeName !== "Multi-Period-Market") { - dependencyControlLoop(); - return + if (dependency.referenceParent.config.codeName !== "Multi-Period-Market") { + continue + } + + if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.config.singularVariableName) !== true) { + if (!(bot.SESSION.parameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { + continue } + } - if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.config.singularVariableName) !== true) { - if (!(bot.SESSION.parameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { - dependencyControlLoop(); + let fileName = "Data.json"; + let filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel; + + /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */ + let response = await asyncGetDatasetFile(datasetModule, filePath, fileName) + onFileReceived(response.err, response.text) + + function onFileReceived(err, text) { + try { + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + if (err.message === 'File does not exist.') { + logger.write(MODULE_NAME, "[ERROR] The file " + filePath + '/' + fileName + ' does not exist and it is required to continue. This process will retry to read it in a while. In the meantime make yourself sure that the process that generates it has ran properly.'); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + } else { + logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> onFileReceived -> err = " + err.message); + callBackFunction(err); + } return } + + let dataFile = JSON.parse(text); + let trimmedDataFile = trimDataFile(dataFile, datasetModule.node.parentNode.record) + dataFiles.set(dependency.id, trimmedDataFile); } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> onFileReceived -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } - let fileName = "Data.json"; - let filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel; - datasetModule.getTextFile(filePath, fileName, onFileReceived); - - function onFileReceived(err, text) { - try { - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - if (err.message === 'File does not exist.') { - logger.write(MODULE_NAME, "[ERROR] The file " + filePath + '/' + fileName + ' does not exist and it is required to continue. This process will retry to read it in a while. In the meantime make yourself sure that the process that generates it has ran properly.'); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - } else { - logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message); - callBackFunction(err); - } - return; - } - - let dataFile = JSON.parse(text); - let trimmedDataFile = trimDataFile(dataFile, datasetModule.node.parentNode.record) - dataFiles.set(dependency.id, trimmedDataFile); - - dependencyControlLoop(); + function trimDataFile(dataFile, recordDefinition) { + /* + Here we will discard all the records in a file that are outside of the current time range. + */ + let beginIndex + let endIndex + let result = [] + for (let i = 0; i < recordDefinition.properties.length; i++) { + let property = recordDefinition.properties[i] + if (property.config.codeName === 'begin') { + beginIndex = i } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); + if (property.config.codeName === 'end') { + endIndex = i } } + for (let i = 0; i < dataFile.length; i++) { + let dataRecord = dataFile[i] + let begin = dataRecord[beginIndex] + let end = dataRecord[endIndex] + if (end < bot.SESSION.parameters.timeRange.config.initialDatetime - 1) { continue } // /1 because we need the previous closed element + if (begin > bot.SESSION.parameters.timeRange.config.finalDatetime) { continue } + result.push(dataRecord) + } + return result } } - function dependencyControlLoop() { - dependencyIndex++; - if (dependencyIndex < dataDependenciesModule.nodeArray.length) { - dependencyLoopBody(); - } else { - let mapKey = global.marketFilesPeriods[n][1]; - multiPeriodDataFiles.set(mapKey, dataFiles) - timeFramesControlLoop(); - } - } - } - - function timeFramesControlLoop() { - n++; - if (n < global.marketFilesPeriods.length) { - timeFramesLoopBody(); - } else { - processDailyFiles(); - } + let mapKey = global.marketFilesPeriods[n][1]; + multiPeriodDataFiles.set(mapKey, dataFiles) } } - function processDailyFiles() { - let n = 0; - if (currentTimeFrame) { - buildCharts() - return - } - bot.tradingProcessDate = new Date(contextVariables.lastFile.valueOf() - global.ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. - advanceTime(); - - function advanceTime() { - bot.tradingProcessDate = new Date(bot.tradingProcessDate.valueOf() + global.ONE_DAY_IN_MILISECONDS); - previousDay = new Date(bot.tradingProcessDate.valueOf() - global.ONE_DAY_IN_MILISECONDS); - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> bot.tradingProcessDate = " + bot.tradingProcessDate); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> previousDay = " + previousDay); } - - /* Validation that we are not going past the user defined end date. */ - if (bot.tradingProcessDate.valueOf() >= bot.SESSION.parameters.timeRange.config.finalDatetime) { - - const logText = "User Defined End Datetime reached @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> " + logText); } - - bot.SESSION.stop(logText) - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } - - /* Validation that we are not going past the head of the market. */ - if (bot.tradingProcessDate.valueOf() > contextVariables.dateEndOfMarket.valueOf() + global.ONE_DAY_IN_MILISECONDS - 1) { - - const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> " + logText); } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } - - checkStopTaskGracefully(); - } - - function checkStopTaskGracefully() { - /* Validation that we dont need to stop. */ - if (global.STOP_TASK_GRACEFULLY === true) { - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } - checkStopProcessing(); - } - - function checkStopProcessing() { - /* Validation that we dont need to stop. */ - if (bot.STOP_SESSION === true) { - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } - timeFramesLoop(); - } - - function timeFramesLoop() { - /* Telling the world we are alive and doing well */ - let processingDate = bot.tradingProcessDate.getUTCFullYear() + '-' + utilities.pad(bot.tradingProcessDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(bot.tradingProcessDate.getUTCDate(), 2); - bot.processHeartBeat(processingDate) - /* - We will iterate through all posible timeFrames. - */ - n = 0 // loop Variable representing each possible period as defined at the timeFrames array. - timeFramesLoopBody(); - } - - function timeFramesLoopBody() { + async function processDailyFiles() { + /* Telling the world we are alive and doing well and which date we are processing right now. */ + let processingDate = bot.tradingProcessDate.getUTCFullYear() + '-' + utilities.pad(bot.tradingProcessDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(bot.tradingProcessDate.getUTCDate(), 2); + bot.processHeartBeat(processingDate) + + /* + We will iterate through all posible timeFrames. + */ + for (let n = 0; n < global.dailyFilePeriods.length; n++) { const timeFrame = global.dailyFilePeriods[n][0]; const timeFrameLabel = global.dailyFilePeriods[n][1]; @@ -435,8 +394,7 @@ if (period === timeFrameLabel) { validPeriod = true } } if (validPeriod === false) { - timeFramesControlLoop(); - return; + continue } } @@ -445,101 +403,114 @@ currentTimeFrameLabel = global.dailyFilePeriods[n][1]; } - let dependencyIndex = 0; dataFiles = new Map(); - dependencyLoopBody(); - function dependencyLoopBody() { + for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.nodeArray.length; dependencyIndex++) { let dependency = dataDependenciesModule.nodeArray[dependencyIndex]; let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]; let file - getFile() - function getFile() { - if (dependency.referenceParent.config.codeName !== "Multi-Period-Daily") { - dependencyControlLoop(); - return + if (dependency.referenceParent.config.codeName !== "Multi-Period-Daily") { + continue + } + + if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.config.singularVariableName) !== true) { + if (!(bot.SESSION.parameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { + continue } + } - if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.config.singularVariableName) !== true) { - if (!(bot.SESSION.parameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { - dependencyControlLoop(); + let dateForPath = bot.tradingProcessDate.getUTCFullYear() + '/' + utilities.pad(bot.tradingProcessDate.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.tradingProcessDate.getUTCDate(), 2); + let filePath + if (dependency.referenceParent.config.codeName === "Multi-Period-Daily") { + filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel + "/" + dateForPath; + } else { + filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + dateForPath; + } + let fileName = "Data.json"; + + /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */ + let response = await asyncGetDatasetFile(datasetModule, filePath, fileName) + onFileReceived(response.err, response.text) + + function onFileReceived(err, text) { + try { + if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { + logger.write(MODULE_NAME, "[ERROR] The file " + filePath + '/' + fileName + ' does not exist and it is required to continue. This process will retry to read it in a while. In the meantime make yourself sure that the process that generates it has ran properly.'); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); return } - } - - let dateForPath = bot.tradingProcessDate.getUTCFullYear() + '/' + utilities.pad(bot.tradingProcessDate.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.tradingProcessDate.getUTCDate(), 2); - let filePath - if (dependency.referenceParent.config.codeName === "Multi-Period-Daily") { - filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel + "/" + dateForPath; - } else { - filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + dateForPath; - } - let fileName = "Data.json"; - - datasetModule.getTextFile(filePath, fileName, onFileReceived); - - function onFileReceived(err, text) { - try { - if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { - logger.write(MODULE_NAME, "[ERROR] The file " + filePath + '/' + fileName + ' does not exist and it is required to continue. This process will retry to read it in a while. In the meantime make yourself sure that the process that generates it has ran properly.'); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> Not Ok -> err = " + err.code); - callBackFunction(err); - return; - } - - file = JSON.parse(text); - dataFiles.set(dependency.id, file); - dependencyControlLoop(); + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> onFileReceived -> Not Ok -> err = " + err.code); + callBackFunction(err); + return } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } + + file = JSON.parse(text); + dataFiles.set(dependency.id, file); + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> onFileReceived -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } - function dependencyControlLoop() { - dependencyIndex++; - - if (dependencyIndex < dataDependenciesModule.nodeArray.length) { - dependencyLoopBody(); - } else { - let mapKey = global.dailyFilePeriods[n][1]; - multiPeriodDataFiles.set(mapKey, dataFiles) - timeFramesControlLoop(); - } - } + let mapKey = global.dailyFilePeriods[n][1]; + multiPeriodDataFiles.set(mapKey, dataFiles) } - function timeFramesControlLoop() { - n++; - if (n < global.dailyFilePeriods.length) { - timeFramesLoopBody(); - } else { - n = 0; - if (currentTimeFrame !== undefined) { - buildCharts(advanceTime); - } else { - logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesControlLoop -> Time Frame not Recognized. Can not continue."); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } + if (currentTimeFrame !== undefined) { + buildCharts(advanceTime); + } else { + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> Time Frame not Recognized. Can not continue."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } - function buildCharts(advanceTime) { + function advanceTime() { + bot.tradingProcessDate = new Date(bot.tradingProcessDate.valueOf() + global.ONE_DAY_IN_MILISECONDS); + previousDay = new Date(bot.tradingProcessDate.valueOf() - global.ONE_DAY_IN_MILISECONDS); + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> advanceTime -> bot.tradingProcessDate = " + bot.tradingProcessDate); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> advanceTime -> previousDay = " + previousDay); } + + /* Validation that we are not going past the user defined end date. */ + if (bot.tradingProcessDate.valueOf() >= bot.SESSION.parameters.timeRange.config.finalDatetime) { + const logText = "User Defined End Datetime reached @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> advanceTime -> " + logText); } + + bot.SESSION.stop(logText) + return false + } + + /* Validation that we are not going past the head of the market. */ + if (bot.tradingProcessDate.valueOf() > contextVariables.dateEndOfMarket.valueOf() + global.ONE_DAY_IN_MILISECONDS - 1) { + const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> advanceTime -> " + logText); } + return false + } + } + + function checkStopTaskGracefully() { + /* Validation that we dont need to stop. */ + if (global.STOP_TASK_GRACEFULLY === true) { + return false + } + } + + function checkStopProcessing() { + /* Validation that we dont need to stop. */ + if (bot.STOP_SESSION === true) { + return false + } + } + + function buildCharts(chart) { const COMMONS = require('../Commons.js') let commons = COMMONS.newCommons(bot, logger, UTILITIES, FILE_STORAGE) - let chart = {} let mainDependency = {} /* The first phase here is about checking that we have everything we need at the definition level. */ @@ -586,222 +557,130 @@ let propertyName = 'atAnyTimeFrame' chart[propertyName] = products } + } + async function generateOutput(chart) { tradingOutputModule.start( chart, currentTimeFrame, currentTimeFrameLabel, - bot.tradingProcessDate, - onOutputGenerated); + bot.tradingProcessDate + ) - function onOutputGenerated(err) { - try { - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - callBackFunction(err); - return; + bot.FIRST_EXECUTION = false // From here on, all other loops executions wont be the first execution. + } + + async function writeProcessFiles() { + + await writeTimeFramesFiles(currentTimeFrame, currentTimeFrameLabel) + await writeDataRanges(currentTimeFrameLabel) + + if (currentTimeFrame > global.dailyFilePeriods[0][0]) { + await writeMarketStatusReport(currentTimeFrameLabel) + } else { + await writeDailyStatusReport(bot.tradingProcessDate, currentTimeFrameLabel) + } + + async function writeDataRanges(currentTimeFrameLabel) { + for (let outputDatasetIndex = 0; outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length; outputDatasetIndex++) { + let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; + await writeDataRange(contextVariables.dateBeginOfMarket, bot.tradingProcessDate, productCodeName, currentTimeFrameLabel, controlLoop); + } + + async function writeDataRange(pBegin, pEnd, productCodeName, currentTimeFrameLabel) { + let dataRange = { + begin: pBegin.valueOf(), + end: pEnd.valueOf() + global.ONE_DAY_IN_MILISECONDS } - - bot.FIRST_EXECUTION = false // From here on, all other loops executions wont be the first execution. - - writeTimeFramesFiles(currentTimeFrame, currentTimeFrameLabel, onTimeFrameFilesWritten) - function onTimeFrameFilesWritten() { - writeDataRanges(currentTimeFrameLabel, onDataRangesWritten); + + let fileContent = JSON.stringify(dataRange); + let fileName = '/Data.Range.json'; + let filePath = bot.filePathRoot + "/Output/" + bot.SESSION.folderName + "/" + productCodeName + "/" + 'Multi-Period-Daily' + fileName; + + let response = await fileStorage.asyncCreateTextFile(filePath, fileContent + '\n') + + if (response.err.result !== global.DEFAULT_OK_RESPONSE.result) { + throw(response.err) } - - function onDataRangesWritten(err) { - try { - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> buildCharts -> onOutputGenerated -> onWritten -> err = " + err.stack); - callBackFunction(err); - return; - } - - if (currentTimeFrame > global.dailyFilePeriods[0][0]) { - writeMarketStatusReport(currentTimeFrameLabel, onMarketStatusReport) - } else { - writeDailyStatusReport(bot.tradingProcessDate, currentTimeFrameLabel, onDailyStatusReport); - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> buildCharts -> onOutputGenerated -> onWritten -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); + + /* + Raise the event that the Data Range was Updated. + */ + let key = bot.dataMine + "-" + bot.codeName + "-" + productCodeName + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset + let event = { + dateRange: dataRange + } + + global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Data Range Updated', event) + } + } + + async function writeTimeFramesFiles(currentTimeFrame, currentTimeFrameLabel) { + + for (let outputDatasetIndex = 0; outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length; outputDatasetIndex++) { + let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; + await writeTimeFramesFile(currentTimeFrame, currentTimeFrameLabel, productCodeName, 'Multi-Period-Daily') + await writeTimeFramesFile(currentTimeFrame, currentTimeFrameLabel, productCodeName, 'Multi-Period-Market') + + async function writeTimeFramesFile(currentTimeFrame, currentTimeFrameLabel, productCodeName, processType) { + + let timeFramesArray = [] + timeFramesArray.push(currentTimeFrameLabel) + + let fileContent = JSON.stringify(timeFramesArray) + let fileName = '/Time.Frames.json'; + + let filePath = bot.filePathRoot + "/Output/" + bot.SESSION.folderName + "/" + productCodeName + "/" + processType + fileName; + + let response = await fileStorage.asyncCreateTextFile(filePath, fileContent + '\n') + if (response.err.result !== global.DEFAULT_OK_RESPONSE.result) { + throw(response.err) } } - - function onDailyStatusReport() { - advanceTime(); - } - - function onMarketStatusReport() { - /* This is where we check if we need reached the user defined end datetime. */ - const now = new Date() - if (now.valueOf() >= bot.SESSION.parameters.timeRange.config.finalDatetime) { - const logText = "User Defined End Datetime reached @ " + now.getUTCFullYear() + "/" + (now.getUTCMonth() + 1) + "/" + now.getUTCDate() + "."; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> buildCharts -> onOutputGenerated -> onMarketStatusReport -> " + logText); } - - bot.SESSION.stop('because it just finished.') - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } - if (bot.SESSION.type === 'Backtesting Session') { - bot.SESSION.stop('Backtesting Session over Market Files Finished.') - } - callBackFunction(global.DEFAULT_OK_RESPONSE); - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> buildCharts -> onOutputGenerated -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - - function writeDataRanges(currentTimeFrameLabel, callBack) { - let outputDatasetIndex = -1; - controlLoop(); - - function productLoopBody() { - let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; - writeDataRange(contextVariables.dateBeginOfMarket, bot.tradingProcessDate, productCodeName, currentTimeFrameLabel, controlLoop); - } - - function controlLoop() { - outputDatasetIndex++; - if (outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length) { - productLoopBody(); - } else { - callBack(global.DEFAULT_OK_RESPONSE); - } - } - } - - function writeDataRange(pBegin, pEnd, productCodeName, currentTimeFrameLabel, callBack) { - let dataRange = { - begin: pBegin.valueOf(), - end: pEnd.valueOf() + global.ONE_DAY_IN_MILISECONDS - }; - - let fileContent = JSON.stringify(dataRange); - let fileName = '/Data.Range.json'; - let filePath = bot.filePathRoot + "/Output/" + bot.SESSION.folderName + "/" + productCodeName + "/" + 'Multi-Period-Daily' + fileName; - - fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated); - - function onFileCreated(err) { - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> writeDataRange -> onFileCreated -> err = " + err.stack); - callBack(err); - return; - } - - if (LOG_FILE_CONTENT === true) { - logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> onFileCreated -> Content written = " + fileContent); - } - - let key = bot.dataMine + "-" + bot.codeName + "-" + productCodeName + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset - let event = { - dateRange: dataRange - } - - global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Data Range Updated', event) - - callBack(global.DEFAULT_OK_RESPONSE); - } - } - - function writeDailyStatusReport(lastFileDate, currentTimeFrameLabel, callBack) { - let reportKey = bot.dataMine + "-" + bot.codeName + "-" + bot.processNode.referenceParent.config.codeName - let thisReport = statusDependencies.statusReports.get(reportKey); - - thisReport.file.lastExecution = bot.currentDaytime; - thisReport.file.lastFile = lastFileDate; - thisReport.file.simulationState = bot.simulationState; - thisReport.file.timeFrames = currentTimeFrameLabel - thisReport.save(callBack); - } - - function writeMarketStatusReport(currentTimeFrameLabel, callBack) { - let reportKey = bot.dataMine + "-" + bot.codeName + "-" + bot.processNode.referenceParent.config.codeName - let thisReport = statusDependencies.statusReports.get(reportKey); - - thisReport.file.lastExecution = bot.processDatetime; - thisReport.file.simulationState = bot.simulationState; - thisReport.file.timeFrames = currentTimeFrameLabel - - logger.newInternalLoop(bot.codeName, bot.process, bot.tradingProcessDate); - thisReport.save(callBack); - } - - function writeTimeFramesFiles(currentTimeFrame, currentTimeFrameLabel, callBack) { - let outputDatasetIndex = -1; - controlLoop() - - function productLoopBody() { - let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; - writeTimeFramesFile(currentTimeFrame, currentTimeFrameLabel, productCodeName, 'Multi-Period-Daily', dailyFileWritten) - - function dailyFileWritten() { - writeTimeFramesFile(currentTimeFrame, currentTimeFrameLabel, productCodeName, 'Multi-Period-Market', controlLoop) } } - function controlLoop() { - outputDatasetIndex++ - if (outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length) { - productLoopBody() - } else { - callBack() - } + async function writeDailyStatusReport(lastFileDate, currentTimeFrameLabel) { + let reportKey = bot.dataMine + "-" + bot.codeName + "-" + bot.processNode.referenceParent.config.codeName + let thisReport = statusDependencies.statusReports.get(reportKey) + + thisReport.file.lastExecution = bot.currentDaytime + thisReport.file.lastFile = lastFileDate + thisReport.file.simulationState = bot.simulationState + thisReport.file.timeFrames = currentTimeFrameLabel + await thisReport.asyncSave() + } + + async function writeMarketStatusReport(currentTimeFrameLabel) { + let reportKey = bot.dataMine + "-" + bot.codeName + "-" + bot.processNode.referenceParent.config.codeName + let thisReport = statusDependencies.statusReports.get(reportKey); + + thisReport.file.lastExecution = bot.processDatetime + thisReport.file.simulationState = bot.simulationState + thisReport.file.timeFrames = currentTimeFrameLabel + + logger.newInternalLoop(bot.codeName, bot.process, bot.tradingProcessDate) + await thisReport.asyncSave() } } - function writeTimeFramesFile(currentTimeFrame, currentTimeFrameLabel, productCodeName, processType, callBack) { - - let timeFramesArray = [] - timeFramesArray.push(currentTimeFrameLabel) - - let fileContent = JSON.stringify(timeFramesArray) - let fileName = '/Time.Frames.json'; - - let filePath = bot.filePathRoot + "/Output/" + bot.SESSION.folderName + "/" + productCodeName + "/" + processType + fileName; - - fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated) - - function onFileCreated(err) { - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> writeTimeFramesFile -> onFileCreated -> err = " + err.stack) - callBack(err) - return - } - callBack() - } - } - - function trimDataFile(dataFile, recordDefinition) { + function checkIfSessionMustStop() { /* - Here we will discard all the records in a file that are outside of the current time range. + The natural way for the session to stop is when we reached the + user defined end datetime. */ - let beginIndex - let endIndex - let result = [] - for (let i = 0; i < recordDefinition.properties.length; i++) { - let property = recordDefinition.properties[i] - if (property.config.codeName === 'begin') { - beginIndex = i - } - if (property.config.codeName === 'end') { - endIndex = i - } + const now = new Date() + if (now.valueOf() >= bot.SESSION.parameters.timeRange.config.finalDatetime) { + const logText = "User Defined End Datetime reached @ " + now.getUTCFullYear() + "/" + (now.getUTCMonth() + 1) + "/" + now.getUTCDate() + "."; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeProcessFiles -> onMarketStatusReport -> " + logText); } + + bot.SESSION.stop('because it just finished.') + return } - for (let i = 0; i < dataFile.length; i++) { - let dataRecord = dataFile[i] - let begin = dataRecord[beginIndex] - let end = dataRecord[endIndex] - if (end < bot.SESSION.parameters.timeRange.config.initialDatetime - 1) { continue } // /1 because we need the previous closed element - if (begin > bot.SESSION.parameters.timeRange.config.finalDatetime) { continue } - result.push(dataRecord) + if (bot.SESSION.type === 'Backtesting Session') { + bot.SESSION.stop('Backtesting Session over Market Files Finished.') + return } - return result } } catch (err) { @@ -814,4 +693,22 @@ str = str.toString(); return str.length < max ? pad(" " + str, max) : str; } -}; + + async function asyncGetDatasetFile(datasetModule, filePath, fileName) { + + let promise = new Promise((resolve, reject) => { + + datasetModule.getTextFile(filePath, fileName, onFileReceived); + function onFileReceived(err, text) { + + let response = { + err: err, + text: text + } + resolve(response) + } + }); + + return promise + } +} diff --git a/StatusReport.js b/StatusReport.js index d39a254fb..2600fca26 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -20,6 +20,7 @@ finalize: finalize, load: load, save: save, + asyncSave: asyncSave, status: undefined }; @@ -343,4 +344,23 @@ } } -}; + async function asyncSave() { + + let fileName = "Status.Report.json"; + let filePath = bot.filePathRoot + "/Reports/" + sessionPath + statusDependencyNode.process; + + filePath += '/' + fileName + let fileContent = JSON.stringify(thisObject.file); + + fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated, true); + let response = await fileStorage.asyncCreateTextFile(filePath, fileContent + '\n', true) + + if (response.err.result !== global.DEFAULT_OK_RESPONSE.result) { + throw(response.err) + } + + /* All good, lets emit the event that means data has been updated. */ + let processOutput = PROCESS_OUTPUT.newProcessOutput(bot, logger) + processOutput.raiseEvents(thisObject.file.lastFile, thisObject.file.timeFrames, callBackFunction); + } +}