Dependencies Filters Implemented

BackendServers/WebServer/AppLoader.js

@@ -51,7 +51,8 @@ function newAppLoader() {
                 'DesignSpace/Workspace/FunctionLibraries/OnFocus.js',
                 'DesignSpace/Workspace/FunctionLibraries/SuperScripts.js',
                 'DesignSpace/Workspace/FunctionLibraries/CCXTFunctions.js',
-                'DesignSpace/Workspace/FunctionLibraries/WebhookFunctions.js',
+                'DesignSpace/Workspace/FunctionLibraries/WebhookFunctions.js', 
+                'DesignSpace/Workspace/FunctionLibraries/DependenciesFilter.js',
                                
                 'Utilities/RoundedCornersBackground.js',
 

CanvasApp/DesignSpace/Workspace/FunctionLibraries/DependenciesFilter.js

@@ -0,0 +1,74 @@
+function newDependenciesFilter () {
+  thisObject = {
+    createFilter: createFilter
+  }
+
+  return thisObject
+
+  function createFilter (node) {
+    let filters = new Map()
+    recursiveFilter(node)
+
+    return Array.from(filters.keys())
+
+    function recursiveFilter (node) {
+      filter(node.code)
+
+      let nodeDefinition = APP_SCHEMA_MAP.get(node.type)
+      if (nodeDefinition !== undefined) {
+        if (nodeDefinition.properties !== undefined) {
+          let previousPropertyName // Since there are cases where there are many properties with the same name,because they can hold nodes of different types but only one at the time, we have to avoind counting each property of those as individual children.
+          for (let i = 0; i < nodeDefinition.properties.length; i++) {
+            let property = nodeDefinition.properties[i]
+
+            switch (property.type) {
+              case 'node': {
+                if (property.name !== previousPropertyName) {
+                  if (node[property.name] !== undefined) {
+                    recursiveFilter(node[property.name])
+                  }
+                  previousPropertyName = property.name
+                }
+              }
+                break
+              case 'array': {
+                let nodePropertyArray = node[property.name]
+                if (nodePropertyArray !== undefined) {
+                  for (let j = 0; j < nodePropertyArray.length; j++) {
+                    let nodeProperty = nodePropertyArray[j]
+                    recursiveFilter(nodeProperty)
+                  }
+                }
+              }
+                break
+            }
+          }
+        }
+      }
+    }
+
+    function filter (code) {
+      if (code === undefined) { return }
+
+      let instructionsArray = code.split(' ')
+      for (let i = 0; i < instructionsArray.length; i++) {
+        let instruction = instructionsArray[i]
+        if (instruction.indexOf('chart') >= 0) {
+          let parts = instruction.split('.')
+          let timeFrame = parts[1]
+          let product = parts[2]
+          let productParts = product.split(',')
+          product = productParts[0]
+          productParts = product.split(')')
+          product = productParts[0]
+
+          // Example: chart.at01hs.popularSMA.sma200 - chart.at01hs.popularSMA.sma100  < 10
+          if (timeFrame !== 'atAnyTimeFrame') {
+            timeFrame = timeFrame.substring(2, 4) + '-' + timeFrame.substring(4, 7)
+          }
+          filters.set(timeFrame + '-' + product, true)
+        }
+      }
+    }
+  }
+}

CanvasApp/DesignSpace/Workspace/FunctionLibraries/SessionFunctions.js

@@ -6,7 +6,7 @@ function newSessionFunctions () {
 
   return thisObject
 
-  function runSession (node, functionLibraryProtocolNode, callBackFunction) {
+  function runSession (node, functionLibraryProtocolNode, functionLibraryDependenciesFilter, callBackFunction) {
     /* We can not run a sessionif its parent process is not running. Less if it does not have a parent. */
     if (node.payload.parentNode === undefined) {
       node.payload.uiObject.setErrorMessage('Session needs a Process Instance parent to be able to run.')
@@ -59,10 +59,13 @@ function newSessionFunctions () {
 
     let session = functionLibraryProtocolNode.getProtocolNode(node, false, true, true, false, false, lightingPath)
 
+    let dependencyFilter = functionLibraryDependenciesFilter.createFilter(node.payload.referenceParent)
+
     /* Raise event to run the session */
     let event = {
       session: JSON.stringify(session),
-      tradingSystem: JSON.stringify(tradingSystem)
+      tradingSystem: JSON.stringify(tradingSystem),
+      dependencyFilter: JSON.stringify(dependencyFilter)
     }
 
     eventsServerClient.raiseEvent(key, 'Run Session', event)

CanvasApp/DesignSpace/Workspace/Workspace.js

@@ -60,6 +60,7 @@ function newWorkspace () {
   let functionLibrarySuperScripts = newSuperScriptsFunctions()
   let functionLibraryCCXTFunctions = newCCXTFunctions()
   let functionLibraryWebhookFunctions = newWebhookFunctions()
+  let functionLibraryDependenciesFilter = newDependenciesFilter()
 
   thisObject.nodeChildren = newNodeChildren()
 
@@ -420,7 +421,7 @@ function newWorkspace () {
         break
       case 'Run Session':
         {
-          functionLibrarySessionFunctions.runSession(payload.node, functionLibraryProtocolNode, callBackFunction)
+          functionLibrarySessionFunctions.runSession(payload.node, functionLibraryProtocolNode, functionLibraryDependenciesFilter, callBackFunction)
         }
         break
       case 'Stop Session':

TaskServer/DataDependencies.js

@@ -9,9 +9,12 @@
     let thisObject = {
         nodeArray: undefined,
         dataSetsModulesArray: [],
+        isItADepenency: isItADepenency, 
         initialize: initialize
     };
 
+    let filter = new Map()
+
     return thisObject;
 
     function initialize(callBackFunction) {
@@ -42,10 +45,17 @@
                 callBackFunction(global.DEFAULT_OK_RESPONSE);
                 return;
             }
+
+            /* Session based dependency filters */
+            if (bot.DEPENDENCY_FILTER !== undefined) {
+                for (let i = 0; i < bot.DEPENDENCY_FILTER.length; i++) {
+                    let key = bot.DEPENDENCY_FILTER[i]
+                        filter.set(key, true)
+                }
+            }
+
             /*
-
             For each dependency declared at the nodeArray, we will initialize a DataSet as part of this initialization process.
-
             */
             let alreadyCalledBack = false;
             let addCount = 0;
@@ -92,4 +102,10 @@
             callBackFunction(global.DEFAULT_FAIL_RESPONSE);
         }
     }
+
+    function isItADepenency(timeFrame, product) {
+        let key = timeFrame + '-' + product 
+
+        return filter.get(key)
+    }
 };

TaskServer/MultiPeriod.js

@@ -71,7 +71,7 @@
             if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); }
 
             let currentTimeFrame
-            let currentOutputPeriodName  
+            let currentTimeFrameLabel  
 
             let market = bot.market;
             let botNeverRan = true;
@@ -246,21 +246,21 @@
                             getFile();
 
                             function getFile() {
-
                                 try {
-
                                     if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles ->  dependencyLoopBody -> getFile -> Entering function."); }
 
-                                    let fileName = "Data.json";
-
                                     if (datasetModule.node.code.codeName !== "Single-File") {
-
                                         dependencyControlLoop();
                                         return
                                     }
 
-                                    let filePath = datasetModule.node.parentNode.code.codeName + '/' + datasetModule.node.code.codeName;
+                                    if (dataDependenciesModule.isItADepenency('atAnyTimeFrame', datasetModule.node.parentNode.code.singularVariableName) !== true) {
+                                        dependencyControlLoop();
+                                        return
+                                    }
 
+                                    let fileName = "Data.json";
+                                    let filePath = datasetModule.node.parentNode.code.codeName + '/' + datasetModule.node.code.codeName;
                                     datasetModule.getTextFile(filePath, fileName, onFileReceived);
 
                                     function onFileReceived(err, text) {
@@ -356,9 +356,7 @@
                             if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoop -> Entering function."); }
 
                             /*
-
                             We will iterate through all posible timeFrames.
-
                             */
 
                             n = 0   // loop Variable representing each possible period as defined at the timeFrames array.
@@ -386,7 +384,7 @@
 
                             if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) {
                                 currentTimeFrame = global.marketFilesPeriods[n][0];
-                                currentOutputPeriodName = global.marketFilesPeriods[n][1];
+                                currentTimeFrameLabel = global.marketFilesPeriods[n][1];
                             }
                             dependencyLoopBody();
 
@@ -402,21 +400,23 @@
                                     getFile();
 
                                     function getFile() {
-
                                         try {
-
                                             if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> Entering function."); }
 
-                                            let fileName =  "Data.json";
-
                                             if (dependency.referenceParent.code.codeName !== "Multi-Period-Market") {
-
                                                 dependencyControlLoop();
                                                 return
                                             }
 
-                                            let filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel;
+                                            if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) {
+                                                if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) {
+                                                    dependencyControlLoop();
+                                                    return
+                                                }
+                                            }
 
+                                            let fileName = "Data.json";
+                                            let filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel;
                                             datasetModule.getTextFile(filePath, fileName, onFileReceived);
 
                                             function onFileReceived(err, text) {
@@ -685,7 +685,7 @@
 
                             if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) {
                                 currentTimeFrame = global.dailyFilePeriods[n][0];
-                                currentOutputPeriodName = global.dailyFilePeriods[n][1];
+                                currentTimeFrameLabel = global.dailyFilePeriods[n][1];
                             }
 
                             let dependencyIndex = 0;
@@ -712,17 +712,21 @@
                                     }
 
                                     function getPreviousFile() {
-
                                         try {
-
                                             if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); }
 
-                                            if (dependency.referenceParent.code.codeName === "Multi-Period-Market") {
-
+                                            if (dependency.referenceParent.code.codeName !== "Multi-Period-Daily") {
                                                 dependencyControlLoop();
                                                 return
                                             }
 
+                                            if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) {
+                                                if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) {
+                                                    dependencyControlLoop();
+                                                    return
+                                                }
+                                            }
+
                                             let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2);
                                             let filePath
 
@@ -789,17 +793,21 @@
                                     }
 
                                     function getCurrentFile() {
-
                                         try {
-
                                             if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); }
 
-                                            if (dependency.referenceParent.code.codeName === "Multi-Period-Market") {
-
+                                            if (dependency.referenceParent.code.codeName !== "Multi-Period-Daily") {
                                                 dependencyControlLoop();
                                                 return
                                             }
 
+                                            if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) {
+                                                if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) {
+                                                    dependencyControlLoop();
+                                                    return
+                                                }
+                                            }
+
                                             let dateForPath = bot.multiPeriodProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCDate(), 2);
                                             let filePath
                                             if (dependency.referenceParent.code.codeName === "Multi-Period-Daily") {
@@ -932,7 +940,7 @@
                             botInstance.start(
                                 multiPeriodDataFiles,
                                 currentTimeFrame,
-                                currentOutputPeriodName,
+                                currentTimeFrameLabel,
                                 bot.multiPeriodProcessDatetime,
                                 interExecutionMemoryArray[n],
                                 onBotFinished);
@@ -954,7 +962,7 @@
                                         writeMarketStatusReport(onMarketStatusReport)
 
                                     } else {
-                                        writeDataRanges(currentOutputPeriodName, onWritten);
+                                        writeDataRanges(currentTimeFrameLabel, onWritten);
                                     }
 
                                     function onWritten(err) {
@@ -1035,7 +1043,7 @@
                 }
             }
 
-            function writeDataRanges(currentOutputPeriodName, callBack) {
+            function writeDataRanges(currentTimeFrameLabel, callBack) {
 
                 try {
 
@@ -1046,7 +1054,7 @@
 
                     function productLoopBody() {
                         let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.code.codeName;
-                        writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodProcessDatetime, productCodeName, currentOutputPeriodName, controlLoop);
+                        writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodProcessDatetime, productCodeName, currentTimeFrameLabel, controlLoop);
                     }
 
                     function controlLoop() {
@@ -1067,7 +1075,7 @@
 
             }
 
-            function writeDataRange(pBegin, pEnd, productCodeName, currentOutputPeriodName, callBack) {
+            function writeDataRange(pBegin, pEnd, productCodeName, currentTimeFrameLabel, callBack) {
 
                 try {
 

TaskServer/Session.js

@@ -54,6 +54,7 @@
                     /* We are going to run the Definition comming at the event. */
                     bot.TRADING_SYSTEM = JSON.parse(message.event.tradingSystem)
                     bot.SESSION = JSON.parse(message.event.session)
+                    bot.DEPENDENCY_FILTER = JSON.parse(message.event.dependencyFilter)
 
                     /* Set the folderName for logging, reports, context and data output */
                     let code

TaskServer/TradingSimulation.js

@@ -571,7 +571,10 @@
                     let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName
                     let pluralVariableName = dataDependencyNode.referenceParent.parentNode.code.pluralVariableName
                     let elementArray = thisChart[pluralVariableName]
-                    let currentElement = elementArray[elementArray.length - 1]
+                    let currentElement
+                    if (elementArray !== undefined) {
+                        currentElement = elementArray[elementArray.length - 1]
+                    }
                     thisChart[singularVariableName] = currentElement
                 }
 
@@ -615,6 +618,7 @@
                 previousLoopingDay = loopingDay.valueOf()
 
                 /* If any of the needed data dependencies is missing for this particular candle, then we jump the candle*/
+                /*
                 for (let k = 0; k < dataDependencies.length; k++) {
                     let dataDependencyNode = dataDependencies[k]
                     let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName
@@ -629,6 +633,7 @@
                         return
                     }
                 }
+                */
 
                 periods++;
                 days = periods * timeFrame / ONE_DAY_IN_MILISECONDS;
@@ -2905,7 +2910,7 @@
             }
 
             function getElement(pArray, currentCandle, datasetName) {
-
+                if (pArray === undefined) {return}
                 try {
                     let element;
                     for (let i = 0; i < pArray.length; i++) {
@@ -2938,6 +2943,7 @@
                 }
                 catch (err) {
                     logger.write(MODULE_NAME, "[ERROR] runSimulation -> getElement -> datasetName = " + datasetName);
+                    logger.write(MODULE_NAME, "[ERROR] runSimulation -> getElement -> err = " + err.stack);
                     callBackFunction(global.DEFAULT_FAIL_RESPONSE);
                 }
             }