From 6ed401c50fbca18057771465bd19b3438bd5233e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 18 Apr 2020 16:22:14 +0200 Subject: [PATCH] Dependencies Filters Implemented --- BackendServers/WebServer/AppLoader.js | 3 +- .../FunctionLibraries/DependenciesFilter.js | 74 +++++++++++++++++++ .../FunctionLibraries/SessionFunctions.js | 7 +- CanvasApp/DesignSpace/Workspace/Workspace.js | 3 +- TaskServer/DataDependencies.js | 20 ++++- TaskServer/MultiPeriod.js | 68 +++++++++-------- TaskServer/Session.js | 1 + TaskServer/TradingSimulation.js | 10 ++- 8 files changed, 148 insertions(+), 38 deletions(-) create mode 100644 CanvasApp/DesignSpace/Workspace/FunctionLibraries/DependenciesFilter.js diff --git a/BackendServers/WebServer/AppLoader.js b/BackendServers/WebServer/AppLoader.js index dda11c6ab..c4cf638d9 100644 --- a/BackendServers/WebServer/AppLoader.js +++ b/BackendServers/WebServer/AppLoader.js @@ -51,7 +51,8 @@ function newAppLoader() { 'DesignSpace/Workspace/FunctionLibraries/OnFocus.js', 'DesignSpace/Workspace/FunctionLibraries/SuperScripts.js', 'DesignSpace/Workspace/FunctionLibraries/CCXTFunctions.js', - 'DesignSpace/Workspace/FunctionLibraries/WebhookFunctions.js', + 'DesignSpace/Workspace/FunctionLibraries/WebhookFunctions.js', + 'DesignSpace/Workspace/FunctionLibraries/DependenciesFilter.js', 'Utilities/RoundedCornersBackground.js', diff --git a/CanvasApp/DesignSpace/Workspace/FunctionLibraries/DependenciesFilter.js b/CanvasApp/DesignSpace/Workspace/FunctionLibraries/DependenciesFilter.js new file mode 100644 index 000000000..2f88bf30d --- /dev/null +++ b/CanvasApp/DesignSpace/Workspace/FunctionLibraries/DependenciesFilter.js @@ -0,0 +1,74 @@ +function newDependenciesFilter () { + thisObject = { + createFilter: createFilter + } + + return thisObject + + function createFilter (node) { + let filters = new Map() + recursiveFilter(node) + + return Array.from(filters.keys()) + + function recursiveFilter (node) { + filter(node.code) + + let nodeDefinition = APP_SCHEMA_MAP.get(node.type) + if (nodeDefinition !== undefined) { + if (nodeDefinition.properties !== undefined) { + let previousPropertyName // Since there are cases where there are many properties with the same name,because they can hold nodes of different types but only one at the time, we have to avoind counting each property of those as individual children. + for (let i = 0; i < nodeDefinition.properties.length; i++) { + let property = nodeDefinition.properties[i] + + switch (property.type) { + case 'node': { + if (property.name !== previousPropertyName) { + if (node[property.name] !== undefined) { + recursiveFilter(node[property.name]) + } + previousPropertyName = property.name + } + } + break + case 'array': { + let nodePropertyArray = node[property.name] + if (nodePropertyArray !== undefined) { + for (let j = 0; j < nodePropertyArray.length; j++) { + let nodeProperty = nodePropertyArray[j] + recursiveFilter(nodeProperty) + } + } + } + break + } + } + } + } + } + + function filter (code) { + if (code === undefined) { return } + + let instructionsArray = code.split(' ') + for (let i = 0; i < instructionsArray.length; i++) { + let instruction = instructionsArray[i] + if (instruction.indexOf('chart') >= 0) { + let parts = instruction.split('.') + let timeFrame = parts[1] + let product = parts[2] + let productParts = product.split(',') + product = productParts[0] + productParts = product.split(')') + product = productParts[0] + + // Example: chart.at01hs.popularSMA.sma200 - chart.at01hs.popularSMA.sma100 < 10 + if (timeFrame !== 'atAnyTimeFrame') { + timeFrame = timeFrame.substring(2, 4) + '-' + timeFrame.substring(4, 7) + } + filters.set(timeFrame + '-' + product, true) + } + } + } + } +} diff --git a/CanvasApp/DesignSpace/Workspace/FunctionLibraries/SessionFunctions.js b/CanvasApp/DesignSpace/Workspace/FunctionLibraries/SessionFunctions.js index 13e33b381..c29f97884 100644 --- a/CanvasApp/DesignSpace/Workspace/FunctionLibraries/SessionFunctions.js +++ b/CanvasApp/DesignSpace/Workspace/FunctionLibraries/SessionFunctions.js @@ -6,7 +6,7 @@ function newSessionFunctions () { return thisObject - function runSession (node, functionLibraryProtocolNode, callBackFunction) { + function runSession (node, functionLibraryProtocolNode, functionLibraryDependenciesFilter, callBackFunction) { /* We can not run a sessionif its parent process is not running. Less if it does not have a parent. */ if (node.payload.parentNode === undefined) { node.payload.uiObject.setErrorMessage('Session needs a Process Instance parent to be able to run.') @@ -59,10 +59,13 @@ function newSessionFunctions () { let session = functionLibraryProtocolNode.getProtocolNode(node, false, true, true, false, false, lightingPath) + let dependencyFilter = functionLibraryDependenciesFilter.createFilter(node.payload.referenceParent) + /* Raise event to run the session */ let event = { session: JSON.stringify(session), - tradingSystem: JSON.stringify(tradingSystem) + tradingSystem: JSON.stringify(tradingSystem), + dependencyFilter: JSON.stringify(dependencyFilter) } eventsServerClient.raiseEvent(key, 'Run Session', event) diff --git a/CanvasApp/DesignSpace/Workspace/Workspace.js b/CanvasApp/DesignSpace/Workspace/Workspace.js index 5bb9a0a88..dedea1c5b 100644 --- a/CanvasApp/DesignSpace/Workspace/Workspace.js +++ b/CanvasApp/DesignSpace/Workspace/Workspace.js @@ -60,6 +60,7 @@ function newWorkspace () { let functionLibrarySuperScripts = newSuperScriptsFunctions() let functionLibraryCCXTFunctions = newCCXTFunctions() let functionLibraryWebhookFunctions = newWebhookFunctions() + let functionLibraryDependenciesFilter = newDependenciesFilter() thisObject.nodeChildren = newNodeChildren() @@ -420,7 +421,7 @@ function newWorkspace () { break case 'Run Session': { - functionLibrarySessionFunctions.runSession(payload.node, functionLibraryProtocolNode, callBackFunction) + functionLibrarySessionFunctions.runSession(payload.node, functionLibraryProtocolNode, functionLibraryDependenciesFilter, callBackFunction) } break case 'Stop Session': diff --git a/TaskServer/DataDependencies.js b/TaskServer/DataDependencies.js index f5ce0e3d2..a833ad6a3 100644 --- a/TaskServer/DataDependencies.js +++ b/TaskServer/DataDependencies.js @@ -9,9 +9,12 @@ let thisObject = { nodeArray: undefined, dataSetsModulesArray: [], + isItADepenency: isItADepenency, initialize: initialize }; + let filter = new Map() + return thisObject; function initialize(callBackFunction) { @@ -42,10 +45,17 @@ callBackFunction(global.DEFAULT_OK_RESPONSE); return; } + + /* Session based dependency filters */ + if (bot.DEPENDENCY_FILTER !== undefined) { + for (let i = 0; i < bot.DEPENDENCY_FILTER.length; i++) { + let key = bot.DEPENDENCY_FILTER[i] + filter.set(key, true) + } + } + /* - For each dependency declared at the nodeArray, we will initialize a DataSet as part of this initialization process. - */ let alreadyCalledBack = false; let addCount = 0; @@ -92,4 +102,10 @@ callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } + + function isItADepenency(timeFrame, product) { + let key = timeFrame + '-' + product + + return filter.get(key) + } }; diff --git a/TaskServer/MultiPeriod.js b/TaskServer/MultiPeriod.js index ff2c5d19c..9a54e8978 100644 --- a/TaskServer/MultiPeriod.js +++ b/TaskServer/MultiPeriod.js @@ -71,7 +71,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); } let currentTimeFrame - let currentOutputPeriodName + let currentTimeFrameLabel let market = bot.market; let botNeverRan = true; @@ -246,21 +246,21 @@ getFile(); function getFile() { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyLoopBody -> getFile -> Entering function."); } - let fileName = "Data.json"; - if (datasetModule.node.code.codeName !== "Single-File") { - dependencyControlLoop(); return } - let filePath = datasetModule.node.parentNode.code.codeName + '/' + datasetModule.node.code.codeName; + if (dataDependenciesModule.isItADepenency('atAnyTimeFrame', datasetModule.node.parentNode.code.singularVariableName) !== true) { + dependencyControlLoop(); + return + } + let fileName = "Data.json"; + let filePath = datasetModule.node.parentNode.code.codeName + '/' + datasetModule.node.code.codeName; datasetModule.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { @@ -356,9 +356,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoop -> Entering function."); } /* - We will iterate through all posible timeFrames. - */ n = 0 // loop Variable representing each possible period as defined at the timeFrames array. @@ -386,7 +384,7 @@ if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) { currentTimeFrame = global.marketFilesPeriods[n][0]; - currentOutputPeriodName = global.marketFilesPeriods[n][1]; + currentTimeFrameLabel = global.marketFilesPeriods[n][1]; } dependencyLoopBody(); @@ -402,21 +400,23 @@ getFile(); function getFile() { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> Entering function."); } - let fileName = "Data.json"; - if (dependency.referenceParent.code.codeName !== "Multi-Period-Market") { - dependencyControlLoop(); return } - let filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel; + if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) { + if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) { + dependencyControlLoop(); + return + } + } + let fileName = "Data.json"; + let filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel; datasetModule.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { @@ -685,7 +685,7 @@ if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) { currentTimeFrame = global.dailyFilePeriods[n][0]; - currentOutputPeriodName = global.dailyFilePeriods[n][1]; + currentTimeFrameLabel = global.dailyFilePeriods[n][1]; } let dependencyIndex = 0; @@ -712,17 +712,21 @@ } function getPreviousFile() { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } - if (dependency.referenceParent.code.codeName === "Multi-Period-Market") { - + if (dependency.referenceParent.code.codeName !== "Multi-Period-Daily") { dependencyControlLoop(); return } + if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) { + if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) { + dependencyControlLoop(); + return + } + } + let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2); let filePath @@ -789,17 +793,21 @@ } function getCurrentFile() { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } - if (dependency.referenceParent.code.codeName === "Multi-Period-Market") { - + if (dependency.referenceParent.code.codeName !== "Multi-Period-Daily") { dependencyControlLoop(); return } + if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) { + if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) { + dependencyControlLoop(); + return + } + } + let dateForPath = bot.multiPeriodProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCDate(), 2); let filePath if (dependency.referenceParent.code.codeName === "Multi-Period-Daily") { @@ -932,7 +940,7 @@ botInstance.start( multiPeriodDataFiles, currentTimeFrame, - currentOutputPeriodName, + currentTimeFrameLabel, bot.multiPeriodProcessDatetime, interExecutionMemoryArray[n], onBotFinished); @@ -954,7 +962,7 @@ writeMarketStatusReport(onMarketStatusReport) } else { - writeDataRanges(currentOutputPeriodName, onWritten); + writeDataRanges(currentTimeFrameLabel, onWritten); } function onWritten(err) { @@ -1035,7 +1043,7 @@ } } - function writeDataRanges(currentOutputPeriodName, callBack) { + function writeDataRanges(currentTimeFrameLabel, callBack) { try { @@ -1046,7 +1054,7 @@ function productLoopBody() { let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.code.codeName; - writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodProcessDatetime, productCodeName, currentOutputPeriodName, controlLoop); + writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodProcessDatetime, productCodeName, currentTimeFrameLabel, controlLoop); } function controlLoop() { @@ -1067,7 +1075,7 @@ } - function writeDataRange(pBegin, pEnd, productCodeName, currentOutputPeriodName, callBack) { + function writeDataRange(pBegin, pEnd, productCodeName, currentTimeFrameLabel, callBack) { try { diff --git a/TaskServer/Session.js b/TaskServer/Session.js index 8de5c2c7a..68a12b7f2 100644 --- a/TaskServer/Session.js +++ b/TaskServer/Session.js @@ -54,6 +54,7 @@ /* We are going to run the Definition comming at the event. */ bot.TRADING_SYSTEM = JSON.parse(message.event.tradingSystem) bot.SESSION = JSON.parse(message.event.session) + bot.DEPENDENCY_FILTER = JSON.parse(message.event.dependencyFilter) /* Set the folderName for logging, reports, context and data output */ let code diff --git a/TaskServer/TradingSimulation.js b/TaskServer/TradingSimulation.js index a97fd1bc4..4f6ce4397 100644 --- a/TaskServer/TradingSimulation.js +++ b/TaskServer/TradingSimulation.js @@ -571,7 +571,10 @@ let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName let pluralVariableName = dataDependencyNode.referenceParent.parentNode.code.pluralVariableName let elementArray = thisChart[pluralVariableName] - let currentElement = elementArray[elementArray.length - 1] + let currentElement + if (elementArray !== undefined) { + currentElement = elementArray[elementArray.length - 1] + } thisChart[singularVariableName] = currentElement } @@ -615,6 +618,7 @@ previousLoopingDay = loopingDay.valueOf() /* If any of the needed data dependencies is missing for this particular candle, then we jump the candle*/ + /* for (let k = 0; k < dataDependencies.length; k++) { let dataDependencyNode = dataDependencies[k] let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName @@ -629,6 +633,7 @@ return } } + */ periods++; days = periods * timeFrame / ONE_DAY_IN_MILISECONDS; @@ -2905,7 +2910,7 @@ } function getElement(pArray, currentCandle, datasetName) { - + if (pArray === undefined) {return} try { let element; for (let i = 0; i < pArray.length; i++) { @@ -2938,6 +2943,7 @@ } catch (err) { logger.write(MODULE_NAME, "[ERROR] runSimulation -> getElement -> datasetName = " + datasetName); + logger.write(MODULE_NAME, "[ERROR] runSimulation -> getElement -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } }