Output Dataset Folders Implemented

This commit is contained in:
Luis Fernando Molina committed 2020-08-30 19:23:55 +02:00
1 parent cdb7bb14f8
commit f1a14df0d0
9 files changed
+135 -107

No files matched your search

+1 -1
View File
@@ -56,7 +56,7 @@
let dataDependencies = bot.processNode.referenceParent.processDependencies.dataDependencies
if (commons.validateDataDependencies(dataDependencies, callBackFunction) !== true) { return }
let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets
let outputDatasets = global.NODE_BRANCH_TO_ARRAY (bot.processNode.referenceParent.processOutput, 'Output Dataset')
if (commons.validateOutputDatasets(outputDatasets, callBackFunction) !== true) { return }
/* The second phase is about transforming the inputs into a format that can be used to apply the user defined code. */
+1 -1
View File
@@ -40,7 +40,7 @@ exports.newTradingOutput = function newTradingOutput(bot, logger, tradingEngineM
const TRADING_SIMULATION = require('./TradingSimulation.js')
let tradingSimulation = TRADING_SIMULATION.newTradingSimulation(bot, logger, tradingEngineModule, UTILITIES)
let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets
let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset')
let outputDatasetsMap = new Map()
if (bot.processingDailyFiles === true) {
+6 -5
View File
@@ -507,7 +507,7 @@
let dataDependencies = bot.processNode.referenceParent.processDependencies.dataDependencies
if (commons.validateDataDependencies(dataDependencies, callBackFunction) !== true) { return }
let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets
let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset')
if (commons.validateOutputDatasets(outputDatasets, callBackFunction) !== true) { return }
/* The second phase is about transforming the inputs into a format that can be used to apply the user defined code. */
@@ -566,6 +566,7 @@
}
async function writeProcessFiles() {
let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset')
await writeTimeFramesFiles(currentTimeFrame, currentTimeFrameLabel)
await writeDataRanges()
@@ -579,10 +580,10 @@
async function writeDataRanges() {
for (
let outputDatasetIndex = 0;
outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length;
outputDatasetIndex < outputDatasets.length;
outputDatasetIndex++
) {
let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName;
let productCodeName = outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName;
await writeDataRange(productCodeName);
}
@@ -618,10 +619,10 @@
for (
let outputDatasetIndex = 0;
outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length;
outputDatasetIndex < outputDatasets.length;
outputDatasetIndex++
) {
let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName;
let productCodeName = outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName;
await writeTimeFramesFile( currentTimeFrameLabel, productCodeName, 'Multi-Period-Daily')
await writeTimeFramesFile( currentTimeFrameLabel, productCodeName, 'Multi-Period-Market')
+2 -1
View File
@@ -37,7 +37,8 @@ exports.newTradingRecords = function newTradingRecords(bot, logger) {
we have a pointer to the node that have the information we need to extract.
Later, based on the product record definition we will extract each individual value.
*/
let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets
let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset')
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
+3 -2
View File
@@ -469,17 +469,18 @@
}
function writeDataRanges(callBack) {
let outputDatasets = global.NODE_BRANCH_TO_ARRAY (bot.processNode.referenceParent.processOutput, 'Output Dataset')
let outputDatasetIndex = -1;
controlLoop()
function productLoopBody() {
let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName;
let productCodeName = outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName;
writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodDailyProcessDatetime, productCodeName, controlLoop)
}
function controlLoop() {
outputDatasetIndex++
if (outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length) {
if (outputDatasetIndex < outputDatasets.length) {
productLoopBody()
} else {
callBack(global.DEFAULT_OK_RESPONSE)
+3 -2
View File
@@ -141,17 +141,18 @@
}
function writeTimeFramesFiles(callBack) {
let outputDatasets = global.NODE_BRANCH_TO_ARRAY (bot.processNode.referenceParent.processOutput, 'Output Dataset')
let outputDatasetIndex = -1;
controlLoop()
function productLoopBody() {
let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName;
let productCodeName = outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName;
writeTimeFramesFile(productCodeName, controlLoop)
}
function controlLoop() {
outputDatasetIndex++
if (outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length) {
if (outputDatasetIndex < outputDatasets.length) {
productLoopBody()
} else {
callBack()
+87 -94
View File
@@ -16,96 +16,11 @@
return thisObject;
function raiseEvents(lastFile, timeFrames, callBackFunction) {
try {
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] raiseEvents -> Entering function."); }
/* We will reaise the events for the datasets impacted by the process that just finished. */
if (bot.processNode.referenceParent.processOutput !== undefined) {
if (bot.processNode.referenceParent.processOutput.outputDatasets !== undefined) {
let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets
for (let j = 0; j < outputDatasets.length; j++) {
let outputDataset = outputDatasets[j]
/* Some validations to verify that everything is in place. */
if (outputDataset.referenceParent !== undefined) {
if (outputDataset.referenceParent.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Dataset Definition not defined. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (outputDataset.referenceParent.parentNode !== undefined) {
if (outputDataset.referenceParent.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Product not defined. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (outputDataset.referenceParent.parentNode.parentNode !== undefined) {
if (outputDataset.referenceParent.parentNode.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Bot not defined. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (outputDataset.referenceParent.parentNode.parentNode.parentNode !== undefined) {
if (outputDataset.referenceParent.parentNode.parentNode.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Data Mine not defined. Data Mine = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Bot not attached to a Data Mine. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Product not attached to a Bot. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Dataset Definition not attached to a Product. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Output Dataset with no reference to a Dataset Definition. Output Dataset = " + JSON.stringify(outputDataset));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
/* All good, lets emit the event. */
let dataMine = outputDataset.referenceParent.parentNode.parentNode.parentNode.config.codeName
let botName = outputDataset.referenceParent.parentNode.parentNode.config.codeName
let product = outputDataset.referenceParent.parentNode.config.codeName
let dataset = outputDataset.referenceParent.config.codeName
key = dataMine + "-" + botName + "-" + product + "-" + dataset + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset
let event = {
lastFile: lastFile,
timeFrames: timeFrames
}
global.EVENT_SERVER_CLIENT.createEventHandler(key, 'Dataset Updated')
global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Dataset Updated', event)
}
}
}
callBackFunction(global.DEFAULT_OK_RESPONSE);
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function asyncRaiseEvents(lastFile, timeFrames) {
if (bot.processNode.referenceParent.processOutput !== undefined) {
if (bot.processNode.referenceParent.processOutput.outputDatasets !== undefined) {
let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets
let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset')
for (let j = 0; j < outputDatasets.length; j++) {
let outputDataset = outputDatasets[j]
@@ -114,38 +29,46 @@
if (outputDataset.referenceParent !== undefined) {
if (outputDataset.referenceParent.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Dataset Definition not defined. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent));
throw(global.DEFAULT_FAIL_RESPONSE);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (outputDataset.referenceParent.parentNode !== undefined) {
if (outputDataset.referenceParent.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Product not defined. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode));
throw(global.DEFAULT_FAIL_RESPONSE);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (outputDataset.referenceParent.parentNode.parentNode !== undefined) {
if (outputDataset.referenceParent.parentNode.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Bot not defined. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode));
throw(global.DEFAULT_FAIL_RESPONSE);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (outputDataset.referenceParent.parentNode.parentNode.parentNode !== undefined) {
if (outputDataset.referenceParent.parentNode.parentNode.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Data Mine not defined. Data Mine = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode.parentNode));
throw(global.DEFAULT_FAIL_RESPONSE);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Bot not attached to a Data Mine. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode));
throw(global.DEFAULT_FAIL_RESPONSE);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Product not attached to a Bot. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode));
throw(global.DEFAULT_FAIL_RESPONSE);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Dataset Definition not attached to a Product. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent));
throw(global.DEFAULT_FAIL_RESPONSE);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Output Dataset with no reference to a Dataset Definition. Output Dataset = " + JSON.stringify(outputDataset));
throw(global.DEFAULT_FAIL_RESPONSE);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
/* All good, lets emit the event. */
@@ -164,6 +87,76 @@
global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Dataset Updated', event)
}
}
callBackFunction(global.DEFAULT_OK_RESPONSE);
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function asyncRaiseEvents(lastFile, timeFrames) {
if (bot.processNode.referenceParent.processOutput !== undefined) {
let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset')
for (let j = 0; j < outputDatasets.length; j++) {
let outputDataset = outputDatasets[j]
/* Some validations to verify that everything is in place. */
if (outputDataset.referenceParent !== undefined) {
if (outputDataset.referenceParent.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Dataset Definition not defined. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent));
throw (global.DEFAULT_FAIL_RESPONSE);
}
if (outputDataset.referenceParent.parentNode !== undefined) {
if (outputDataset.referenceParent.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Product not defined. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode));
throw (global.DEFAULT_FAIL_RESPONSE);
}
if (outputDataset.referenceParent.parentNode.parentNode !== undefined) {
if (outputDataset.referenceParent.parentNode.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Bot not defined. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode));
throw (global.DEFAULT_FAIL_RESPONSE);
}
if (outputDataset.referenceParent.parentNode.parentNode.parentNode !== undefined) {
if (outputDataset.referenceParent.parentNode.parentNode.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Data Mine not defined. Data Mine = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode.parentNode));
throw (global.DEFAULT_FAIL_RESPONSE);
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Bot not attached to a Data Mine. Bot = " + JSON.stringify(outputDataset.referenceParent.parentNode.parentNode));
throw (global.DEFAULT_FAIL_RESPONSE);
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Product not attached to a Bot. Product = " + JSON.stringify(outputDataset.referenceParent.parentNode));
throw (global.DEFAULT_FAIL_RESPONSE);
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Dataset Definition not attached to a Product. Dataset Definition = " + JSON.stringify(outputDataset.referenceParent));
throw (global.DEFAULT_FAIL_RESPONSE);
}
} else {
logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Output Dataset with no reference to a Dataset Definition. Output Dataset = " + JSON.stringify(outputDataset));
throw (global.DEFAULT_FAIL_RESPONSE);
}
/* All good, lets emit the event. */
let dataMine = outputDataset.referenceParent.parentNode.parentNode.parentNode.config.codeName
let botName = outputDataset.referenceParent.parentNode.parentNode.config.codeName
let product = outputDataset.referenceParent.parentNode.config.codeName
let dataset = outputDataset.referenceParent.config.codeName
key = dataMine + "-" + botName + "-" + product + "-" + dataset + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset
let event = {
lastFile: lastFile,
timeFrames: timeFrames
}
global.EVENT_SERVER_CLIENT.createEventHandler(key, 'Dataset Updated')
global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Dataset Updated', event)
}
}
}
};
+1 -1
View File
@@ -83,7 +83,7 @@
function validationFailed(errorInNode, errorMessage) {
let nodeString = JSON.stringify(errorInNode)
logger.write(MODULE_NAME, "[ERROR] initialize -> " + errorMessage + ' -> nodeString = ' + nodeString)
bot.global.PROCESS_ERROR(bot.processKey, errorInNode, errorMessage)
global.PROCESS_ERROR(bot.processKey, errorInNode, errorMessage)
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
+31
View File
@@ -103,7 +103,38 @@ global.PROCESS_INFO = function (processKey, node, infoMessage) {
}
global.NODE_BRANCH_TO_ARRAY = function (node, nodeType) {
let resultArray = []
scanNodeBranch(node, nodeType)
return resultArray
function scanNodeBranch(startingNode) {
let nodeDefinition = global.APP_SCHEMA_MAP.get(startingNode.type)
if (nodeDefinition === undefined) { return }
if (startingNode.type === nodeType) {
resultArray.push(startingNode)
return
}
if (nodeDefinition.properties === undefined) { return }
for (let i = 0; i < nodeDefinition.properties.length; i++) {
let property = nodeDefinition.properties[i]
switch (property.type) {
case 'node': {
scanNodeBranch(startingNode[property.name])
}
break
case 'array': {
let startingNodePropertyArray = startingNode[property.name]
for (let m = 0; m < startingNodePropertyArray.length; m++) {
scanNodeBranch(startingNodePropertyArray[m])
}
break
}
}
}
}
}
process.on('uncaughtException', function (err) {