From f1a14df0d05835ee751901c15c2bedc89326c0d2 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 30 Aug 2020 19:23:55 +0200 Subject: [PATCH] Output Dataset Folders Implemented --- IndicatorOutput.js | 2 +- LowFrequencyTrading/TradingOutput.js | 2 +- LowFrequencyTrading/TradingProcess.js | 11 +- LowFrequencyTrading/TradingRecords.js | 3 +- MultiPeriodDaily.js | 5 +- MultiPeriodMarket.js | 5 +- ProcessOutput.js | 181 +++++++++++++------------- StatusReport.js | 2 +- server.js | 31 +++++ 9 files changed, 135 insertions(+), 107 deletions(-) diff --git a/IndicatorOutput.js b/IndicatorOutput.js index 0ae431043..988acda77 100644 --- a/IndicatorOutput.js +++ b/IndicatorOutput.js @@ -56,7 +56,7 @@ let dataDependencies = bot.processNode.referenceParent.processDependencies.dataDependencies if (commons.validateDataDependencies(dataDependencies, callBackFunction) !== true) { return } - let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets + let outputDatasets = global.NODE_BRANCH_TO_ARRAY (bot.processNode.referenceParent.processOutput, 'Output Dataset') if (commons.validateOutputDatasets(outputDatasets, callBackFunction) !== true) { return } /* The second phase is about transforming the inputs into a format that can be used to apply the user defined code. */ diff --git a/LowFrequencyTrading/TradingOutput.js b/LowFrequencyTrading/TradingOutput.js index 37b7bad6e..1f582d090 100644 --- a/LowFrequencyTrading/TradingOutput.js +++ b/LowFrequencyTrading/TradingOutput.js @@ -40,7 +40,7 @@ exports.newTradingOutput = function newTradingOutput(bot, logger, tradingEngineM const TRADING_SIMULATION = require('./TradingSimulation.js') let tradingSimulation = TRADING_SIMULATION.newTradingSimulation(bot, logger, tradingEngineModule, UTILITIES) - let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets + let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset') let outputDatasetsMap = new Map() if (bot.processingDailyFiles === true) { diff --git a/LowFrequencyTrading/TradingProcess.js b/LowFrequencyTrading/TradingProcess.js index 7e0a2e418..40f6dce13 100644 --- a/LowFrequencyTrading/TradingProcess.js +++ b/LowFrequencyTrading/TradingProcess.js @@ -507,7 +507,7 @@ let dataDependencies = bot.processNode.referenceParent.processDependencies.dataDependencies if (commons.validateDataDependencies(dataDependencies, callBackFunction) !== true) { return } - let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets + let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset') if (commons.validateOutputDatasets(outputDatasets, callBackFunction) !== true) { return } /* The second phase is about transforming the inputs into a format that can be used to apply the user defined code. */ @@ -566,6 +566,7 @@ } async function writeProcessFiles() { + let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset') await writeTimeFramesFiles(currentTimeFrame, currentTimeFrameLabel) await writeDataRanges() @@ -579,10 +580,10 @@ async function writeDataRanges() { for ( let outputDatasetIndex = 0; - outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length; + outputDatasetIndex < outputDatasets.length; outputDatasetIndex++ ) { - let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; + let productCodeName = outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; await writeDataRange(productCodeName); } @@ -618,10 +619,10 @@ for ( let outputDatasetIndex = 0; - outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length; + outputDatasetIndex < outputDatasets.length; outputDatasetIndex++ ) { - let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; + let productCodeName = outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; await writeTimeFramesFile( currentTimeFrameLabel, productCodeName, 'Multi-Period-Daily') await writeTimeFramesFile( currentTimeFrameLabel, productCodeName, 'Multi-Period-Market') diff --git a/LowFrequencyTrading/TradingRecords.js b/LowFrequencyTrading/TradingRecords.js index fd019fbcf..f3de8553c 100644 --- a/LowFrequencyTrading/TradingRecords.js +++ b/LowFrequencyTrading/TradingRecords.js @@ -37,7 +37,8 @@ exports.newTradingRecords = function newTradingRecords(bot, logger) { we have a pointer to the node that have the information we need to extract. Later, based on the product record definition we will extract each individual value. */ - let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets + let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset') + for (let i = 0; i < outputDatasets.length; i++) { let outputDatasetNode = outputDatasets[i] let dataset = outputDatasetNode.referenceParent diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 7ae3cc3b5..cdc4a892b 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -469,17 +469,18 @@ } function writeDataRanges(callBack) { + let outputDatasets = global.NODE_BRANCH_TO_ARRAY (bot.processNode.referenceParent.processOutput, 'Output Dataset') let outputDatasetIndex = -1; controlLoop() function productLoopBody() { - let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; + let productCodeName = outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodDailyProcessDatetime, productCodeName, controlLoop) } function controlLoop() { outputDatasetIndex++ - if (outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length) { + if (outputDatasetIndex < outputDatasets.length) { productLoopBody() } else { callBack(global.DEFAULT_OK_RESPONSE) diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index 02a35a702..76ba55b28 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -141,17 +141,18 @@ } function writeTimeFramesFiles(callBack) { + let outputDatasets = global.NODE_BRANCH_TO_ARRAY (bot.processNode.referenceParent.processOutput, 'Output Dataset') let outputDatasetIndex = -1; controlLoop() function productLoopBody() { - let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; + let productCodeName = outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; writeTimeFramesFile(productCodeName, controlLoop) } function controlLoop() { outputDatasetIndex++ - if (outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length) { + if (outputDatasetIndex < outputDatasets.length) { productLoopBody() } else { callBack() diff --git a/ProcessOutput.js b/ProcessOutput.js index 71d7cdc81..3496ffe15 100644 --- a/ProcessOutput.js +++ b/ProcessOutput.js @@ -16,96 +16,11 @@ return thisObject; function raiseEvents(lastFile, timeFrames, callBackFunction) { - try { - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] raiseEvents -> Entering function."); } - /* We will reaise the events for the datasets impacted by the process that just finished. */ if (bot.processNode.referenceParent.processOutput !== undefined) { - if (bot.processNode.referenceParent.processOutput.outputDatasets !== undefined) { - let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets - - for (let j = 0; j < outputDatasets.length; j++) { - let outputDataset = outputDatasets[j] - - /* Some validations to verify that everything is in place. */ - if (outputDataset.referenceParent !== undefined) { - if (outputDataset.referenceParent.config.codeName === undefined) { - logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Dataset Definition not defined. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent)); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - } - if (outputDataset.referenceParent.parentNode !== undefined) { - if (outputDataset.referenceParent.parentNode.config.codeName === undefined) { - logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Product not defined. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode)); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - } - if (outputDataset.referenceParent.parentNode.parentNode !== undefined) { - if (outputDataset.referenceParent.parentNode.parentNode.config.codeName === undefined) { - logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Bot not defined. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode)); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - } - if (outputDataset.referenceParent.parentNode.parentNode.parentNode !== undefined) { - if (outputDataset.referenceParent.parentNode.parentNode.parentNode.config.codeName === undefined) { - logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Data Mine not defined. Data Mine = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode.parentNode)); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - } - } else { - logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Bot not attached to a Data Mine. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode)); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - } - } else { - logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Product not attached to a Bot. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode)); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - } - } else { - logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Dataset Definition not attached to a Product. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent)); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - } - } else { - logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Output Dataset with no reference to a Dataset Definition. Output Dataset = " + JSON.stringify(outputDataset)); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - } - - /* All good, lets emit the event. */ - - let dataMine = outputDataset.referenceParent.parentNode.parentNode.parentNode.config.codeName - let botName = outputDataset.referenceParent.parentNode.parentNode.config.codeName - let product = outputDataset.referenceParent.parentNode.config.codeName - let dataset = outputDataset.referenceParent.config.codeName - - key = dataMine + "-" + botName + "-" + product + "-" + dataset + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset - let event = { - lastFile: lastFile, - timeFrames: timeFrames - } - global.EVENT_SERVER_CLIENT.createEventHandler(key, 'Dataset Updated') - global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Dataset Updated', event) - } - } - } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] raiseEvents -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function asyncRaiseEvents(lastFile, timeFrames) { - - if (bot.processNode.referenceParent.processOutput !== undefined) { - if (bot.processNode.referenceParent.processOutput.outputDatasets !== undefined) { - let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets + let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset') for (let j = 0; j < outputDatasets.length; j++) { let outputDataset = outputDatasets[j] @@ -114,38 +29,46 @@ if (outputDataset.referenceParent !== undefined) { if (outputDataset.referenceParent.config.codeName === undefined) { logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Dataset Definition not defined. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent)); - throw(global.DEFAULT_FAIL_RESPONSE); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return } if (outputDataset.referenceParent.parentNode !== undefined) { if (outputDataset.referenceParent.parentNode.config.codeName === undefined) { logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Product not defined. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode)); - throw(global.DEFAULT_FAIL_RESPONSE); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return } if (outputDataset.referenceParent.parentNode.parentNode !== undefined) { if (outputDataset.referenceParent.parentNode.parentNode.config.codeName === undefined) { logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Bot not defined. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode)); - throw(global.DEFAULT_FAIL_RESPONSE); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return } if (outputDataset.referenceParent.parentNode.parentNode.parentNode !== undefined) { if (outputDataset.referenceParent.parentNode.parentNode.parentNode.config.codeName === undefined) { logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Data Mine not defined. Data Mine = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode.parentNode)); - throw(global.DEFAULT_FAIL_RESPONSE); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return } } else { logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Bot not attached to a Data Mine. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode)); - throw(global.DEFAULT_FAIL_RESPONSE); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return } } else { logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Product not attached to a Bot. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode)); - throw(global.DEFAULT_FAIL_RESPONSE); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return } } else { logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Dataset Definition not attached to a Product. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent)); - throw(global.DEFAULT_FAIL_RESPONSE); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return } } else { logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Output Dataset with no reference to a Dataset Definition. Output Dataset = " + JSON.stringify(outputDataset)); - throw(global.DEFAULT_FAIL_RESPONSE); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return } /* All good, lets emit the event. */ @@ -164,6 +87,76 @@ global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Dataset Updated', event) } } + + callBackFunction(global.DEFAULT_OK_RESPONSE); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] raiseEvents -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function asyncRaiseEvents(lastFile, timeFrames) { + + if (bot.processNode.referenceParent.processOutput !== undefined) { + let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset') + + for (let j = 0; j < outputDatasets.length; j++) { + let outputDataset = outputDatasets[j] + + /* Some validations to verify that everything is in place. */ + if (outputDataset.referenceParent !== undefined) { + if (outputDataset.referenceParent.config.codeName === undefined) { + logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Dataset Definition not defined. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent)); + throw (global.DEFAULT_FAIL_RESPONSE); + } + if (outputDataset.referenceParent.parentNode !== undefined) { + if (outputDataset.referenceParent.parentNode.config.codeName === undefined) { + logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Product not defined. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode)); + throw (global.DEFAULT_FAIL_RESPONSE); + } + if (outputDataset.referenceParent.parentNode.parentNode !== undefined) { + if (outputDataset.referenceParent.parentNode.parentNode.config.codeName === undefined) { + logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Bot not defined. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode)); + throw (global.DEFAULT_FAIL_RESPONSE); + } + if (outputDataset.referenceParent.parentNode.parentNode.parentNode !== undefined) { + if (outputDataset.referenceParent.parentNode.parentNode.parentNode.config.codeName === undefined) { + logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Data Mine not defined. Data Mine = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode.parentNode)); + throw (global.DEFAULT_FAIL_RESPONSE); + } + } else { + logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Bot not attached to a Data Mine. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode)); + throw (global.DEFAULT_FAIL_RESPONSE); + } + } else { + logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Product not attached to a Bot. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode)); + throw (global.DEFAULT_FAIL_RESPONSE); + } + } else { + logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Dataset Definition not attached to a Product. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent)); + throw (global.DEFAULT_FAIL_RESPONSE); + } + } else { + logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Output Dataset with no reference to a Dataset Definition. Output Dataset = " + JSON.stringify(outputDataset)); + throw (global.DEFAULT_FAIL_RESPONSE); + } + + /* All good, lets emit the event. */ + + let dataMine = outputDataset.referenceParent.parentNode.parentNode.parentNode.config.codeName + let botName = outputDataset.referenceParent.parentNode.parentNode.config.codeName + let product = outputDataset.referenceParent.parentNode.config.codeName + let dataset = outputDataset.referenceParent.config.codeName + + key = dataMine + "-" + botName + "-" + product + "-" + dataset + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset + let event = { + lastFile: lastFile, + timeFrames: timeFrames + } + global.EVENT_SERVER_CLIENT.createEventHandler(key, 'Dataset Updated') + global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Dataset Updated', event) + } } } }; diff --git a/StatusReport.js b/StatusReport.js index 50cf8c2a1..39d116fd5 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -83,7 +83,7 @@ function validationFailed(errorInNode, errorMessage) { let nodeString = JSON.stringify(errorInNode) logger.write(MODULE_NAME, "[ERROR] initialize -> " + errorMessage + ' -> nodeString = ' + nodeString) - bot.global.PROCESS_ERROR(bot.processKey, errorInNode, errorMessage) + global.PROCESS_ERROR(bot.processKey, errorInNode, errorMessage) callBackFunction(global.DEFAULT_FAIL_RESPONSE); } diff --git a/server.js b/server.js index 02cd778c6..3e6bb2824 100644 --- a/server.js +++ b/server.js @@ -103,7 +103,38 @@ global.PROCESS_INFO = function (processKey, node, infoMessage) { } global.NODE_BRANCH_TO_ARRAY = function (node, nodeType) { + let resultArray = [] + scanNodeBranch(node, nodeType) + return resultArray + function scanNodeBranch(startingNode) { + let nodeDefinition = global.APP_SCHEMA_MAP.get(startingNode.type) + if (nodeDefinition === undefined) { return } + + if (startingNode.type === nodeType) { + resultArray.push(startingNode) + return + } + + if (nodeDefinition.properties === undefined) { return } + for (let i = 0; i < nodeDefinition.properties.length; i++) { + let property = nodeDefinition.properties[i] + + switch (property.type) { + case 'node': { + scanNodeBranch(startingNode[property.name]) + } + break + case 'array': { + let startingNodePropertyArray = startingNode[property.name] + for (let m = 0; m < startingNodePropertyArray.length; m++) { + scanNodeBranch(startingNodePropertyArray[m]) + } + break + } + } + } + } } process.on('uncaughtException', function (err) {