Home Network fully implemented

BackendServers/taskManagerServer.js

@@ -51,17 +51,17 @@
             //console.log('[INFO] BackendServers -> Task Manager Server -> runTask -> Entering function.') 
 
             if (message.event === undefined) {
-                console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without Event -> message = ' + JSON.stringify(message))
+                console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without Event -> message = ' + JSON.stringify(message).substring(0, 1000))
                 return
             }
 
             if (message.event.taskId === undefined) {
-                console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without taskId -> message = ' + JSON.stringify(message))
+                console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without taskId -> message = ' + JSON.stringify(message).substring(0, 1000))
                 return
             }
 
-            if (message.event.definition === undefined) {
-                console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without Definition -> message = ' + JSON.stringify(message))
+            if (message.event.taskDefinition === undefined) {
+                console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without taskDefinition -> message = ' + JSON.stringify(message).substring(0, 1000))
                 return
             }
 
@@ -116,12 +116,12 @@
             //console.log('[INFO] BackendServers -> Task Manager Server -> stopTask -> Entering function.')
 
             if (message.event === undefined) {
-                console.log('[WARN] BackendServers -> Task Manager Server -> stopTask -> Message Received Without Event -> message = ' + JSON.stringify(message))
+                console.log('[WARN] BackendServers -> Task Manager Server -> stopTask -> Message Received Without Event -> message = ' + JSON.stringify(message).substring(0, 1000))
                 return
             }
 
             if (message.event.taskId === undefined) {
-                console.log('[WARN] BackendServers -> Task Manager Server -> stopTask -> Message Received Without taskId -> message = ' + JSON.stringify(message))
+                console.log('[WARN] BackendServers -> Task Manager Server -> stopTask -> Message Received Without taskId -> message = ' + JSON.stringify(message).substring(0, 1000))
                 return
             }
 

BackendServers/webSocketsServer.js

@@ -47,7 +47,7 @@
 
                     try {
                         if (LOG_INFO === true) {
-                            console.log('Message Received: ' + message)
+                            console.log('Message Received: ' + message.substring(0, 1000))
                         }
 
                         let messageArray = message.split('|*|')
@@ -58,7 +58,7 @@
 
                         if (origin === 'Web Browser') {
                             if (isNaN(nonce) || nonce === "") {
-                                console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce is not a Number. message = ' + message)
+                                console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce is not a Number. message = ' + message.substring(0, 1000))
                                 console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce is not a Number. nonce = ' + nonce)
                                 return
                             }
@@ -68,7 +68,7 @@
                             }
 
                             if (Number(nonce) < Number(lastNonce)) {
-                                console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce received is less than Last Nonce. message = ' + message)
+                                console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce received is less than Last Nonce. message = ' + message.substring(0, 1000))
                                 console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce received is less than Last Nonce. nonce = ' + nonce)
                                 console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce received is less than Last Nonce. lastNonce = ' + lastNonce)
                                 return
@@ -79,7 +79,7 @@
                             try {
                                 let jsonCheck = JSON.parse(messageToEventServer)
                             } catch (err) {
-                                console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Message received from the browser is not a valid JSON. message = ' + message)
+                                console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Message received from the browser is not a valid JSON. message = ' + message.substring(0, 1000))
                                 console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Message received from the browser is not a valid JSON. messageToEventServer = ' + messageToEventServer)
                                 return
                             }
@@ -106,7 +106,7 @@
 
             console.log('Web Sockets Server Started.')
         } catch (err) {
-            console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> err.message = ' + err.message)
+            console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> err.message = ' + err.message.substring(0, 1000))
         }
     }
 }

CanvasApp/DesignSpace/Workspace/FunctionLibraries/ProtocolNode.js

@@ -24,23 +24,19 @@ function newProtocolNode () {
 
       /* Children Nodes */
       if (nodeDefinition.properties !== undefined) {
-        let previousPropertyName // Since there are cases where there are many properties with the same name,because they can hold nodes of different types but only one at the time, we have to avoind counting each property of those as individual children.
         for (let i = 0; i < nodeDefinition.properties.length; i++) {
           let property = nodeDefinition.properties[i]
 
           switch (property.type) {
             case 'node': {
               if (node[property.name] !== undefined) {
-                if (property.name !== previousPropertyName) {
-                  if (lightingPath === undefined) {
-                    object[property.name] = getProtocolNode(node[property.name], removePersonalData, parseCode, includeIds, includeReferencesInSavedPayload, includePayload)
-                  } else {
-                    let newLightingPath = getNewLightingPath(lightingPath, node.type, property.childType)
-                    if (newLightingPath !== undefined) {
-                      object[property.name] = getProtocolNode(node[property.name], removePersonalData, parseCode, includeIds, includePayload, includeReferencesInSavedPayload, newLightingPath)
-                    }
+                if (lightingPath === undefined) {
+                  object[property.name] = getProtocolNode(node[property.name], removePersonalData, parseCode, includeIds, includeReferencesInSavedPayload, includePayload)
+                } else {
+                  let newLightingPath = getNewLightingPath(lightingPath, node.type, property.childType)
+                  if (newLightingPath !== undefined) {
+                    object[property.name] = getProtocolNode(node[property.name], removePersonalData, parseCode, includeIds, includePayload, includeReferencesInSavedPayload, newLightingPath)
                   }
-                  previousPropertyName = property.name
                 }
               }
               break

CanvasApp/DesignSpace/Workspace/FunctionLibraries/TaskFunctions.js

@@ -26,7 +26,7 @@ function newTaskFunctions () {
       process.payload.uiObject.run(eventsServerClient)
     }
 
-    let lightingPath = '->Task->' +
+    let taskLightingPath = '->Task->' +
     'Sensor Bot Instance->' +
     'Indicator Bot Instance->' +
     'Trading Bot Instance->' +
@@ -56,12 +56,26 @@ function newTaskFunctions () {
     'Trading Bot->' +
     'Data Mine->'
 
-    let definition = functionLibraryProtocolNode.getProtocolNode(node, false, true, true, false, false, lightingPath)
+    let taskDefinition = functionLibraryProtocolNode.getProtocolNode(node, false, true, true, false, false, taskLightingPath)
+
+    let networkLightingPath = '->Network->Network Node->' +
+   'Data Storage->Session Independent Data->Exchange Data Products->' +
+   'Single Market Data->Data Product->Product Definition->' +
+   'Data Mining->Testing Environment->Production Environment->' +
+   'Exchange Tasks->Crypto Exchange->' +
+   'Task Manager->Task->' +
+   'Indicator Bot Instance->Sensor Bot Instance->' +
+   'Indicator Process Instance->Sensor Process Instance->' +
+   'Market Reference->Market->' +
+   'Process Definition->'
+
+    let networkDefinition = functionLibraryProtocolNode.getProtocolNode(networkNode.payload.parentNode, false, true, true, false, false, networkLightingPath)
 
     let event = {
       taskId: node.id,
       taskName: node.name,
-      definition: JSON.stringify(definition) // <-  We need to do this workaround in order no to send unescaped charactars to the taskManager.
+      taskDefinition: JSON.stringify(taskDefinition),
+      networkDefinition: JSON.stringify(networkDefinition)
     }
 
     if (isDebugging === true) {
@@ -129,6 +143,28 @@ function newTaskFunctions () {
       node.payload.uiObject.setErrorMessage('Task needs to be inside a Network Node.')
       return
     }
+
+    if (node.payload.parentNode.payload.parentNode.payload.parentNode.payload.parentNode.payload.parentNode === undefined) {
+      node.payload.uiObject.setErrorMessage('Task needs to be inside a Network.')
+      return
+    }
+
+    let networkNode = node.payload.parentNode.payload.parentNode.payload.parentNode.payload.parentNode
+    if (loadPropertyFromNodeConfig(networkNode.payload, 'host') === undefined) {
+      node.payload.uiObject.setErrorMessage('Network Node needs to have a valid Host property at its config.')
+      return
+    }
+
+    if (loadPropertyFromNodeConfig(networkNode.payload, 'webPort') === undefined) {
+      node.payload.uiObject.setErrorMessage('Network Node needs to have a valid webPort property at its config.')
+      return
+    }
+
+    if (loadPropertyFromNodeConfig(networkNode.payload, 'webSocketsPort') === undefined) {
+      node.payload.uiObject.setErrorMessage('Network Node needs to have a valid webSocketsPort property at its config.')
+      return
+    }
+
     return true
   }
 

CanvasApp/DesignSpace/Workspace/Workspace.js

@@ -126,7 +126,7 @@ function newWorkspace () {
           if (host === undefined) { host = 'localhost' }
           if (webSocketsPort === undefined) { webSocketsPort = '8080' }
 
-          let eventsServerClient = newEventsServerClient(host, webSocketsPort)
+          let eventsServerClient = newEventsServerClient(host, webSocketsPort, networkNode.name)
           eventsServerClient.initialize()
 
           thisObject.eventsServerClients.set(networkNode.id, eventsServerClient)

CanvasApp/EventsServerClient.js

@@ -1,5 +1,5 @@
 
-function newEventsServerClient (host, port) {
+function newEventsServerClient (host, port, hostName) {
   /* Web Sockets Connection */
 
   const MODULE_NAME = 'System Event Handler'
@@ -153,7 +153,7 @@ function newEventsServerClient (host, port) {
     }
 
     if (thisObject.isConnected() !== true) {
-      canvas.cockpitSpace.setStatus('Connecting with ' + host + '. Please hold on until the connection can be stablished.', 100, canvas.cockpitSpace.statusTypes.WARNING)
+      canvas.cockpitSpace.setStatus('Connecting to ' + hostName + ' Network Node. Please hold on until the connection can be stablished.', 100, canvas.cockpitSpace.statusTypes.WARNING)
     } else {
       retryCommandsPhysics()
     }

CanvasApp/FloatingSpace/UiObject.js

@@ -1424,7 +1424,7 @@ function newUiObject () {
     let label
 
     if (radius > 6) {
-      const MAX_LABEL_LENGTH = 65
+      const MAX_LABEL_LENGTH = 100
 
       label = currentStatus
 

CanvasApp/FloatingSpace/UiObjectConstructor.js

@@ -157,6 +157,17 @@ function newUiObjectConstructor () {
       }
     }
 
+    /* Default Values in case there was no way to set a value previous to this. */
+    if (floatingObject.angleToParent === undefined) {
+      floatingObject.angleToParent = ANGLE_TO_PARENT.RANGE_360
+    }
+    if (floatingObject.distanceToParent === undefined) {
+      floatingObject.distanceToParent = DISTANCE_TO_PARENT.PARENT_100X
+    }
+    if (floatingObject.arrangementStyle === undefined) {
+      floatingObject.arrangementStyle = ARRANGEMENT_STYLE.CONCAVE
+    }
+
     let uiObject = newUiObject()
     payload.uiObject = uiObject
     uiObject.fitFunction = canvas.floatingSpace.fitIntoVisibleArea
@@ -243,7 +254,7 @@ function newUiObjectConstructor () {
       )
     menuItemsInitialValues.push(
       {
-        action: 'change Distance to Paarent',
+        action: 'Change Distance to Paarent',
         actionFunction: floatingObject.distanceToParentToggle,
         actionStatus: floatingObject.getDistanceToParent,
         currentStatus: true,
@@ -258,7 +269,7 @@ function newUiObjectConstructor () {
         )
     menuItemsInitialValues.push(
       {
-        action: 'change Arrangement Style',
+        action: 'Change Arrangement Style',
         actionFunction: floatingObject.arrangementStyleToggle,
         actionStatus: floatingObject.getArrangementStyle,
         currentStatus: true,

TaskServer/DataDependencies.js

@@ -10,7 +10,8 @@
         nodeArray: undefined,
         dataSetsModulesArray: [],
         isItADepenency: isItADepenency, 
-        initialize: initialize
+        initialize: initialize,
+        finalize: finalize
     };
 
     let filter = new Map()
@@ -103,6 +104,17 @@
         }
     }
 
+    function finalize() {
+        for (let i = 0; i < thisObject.dataSetsModulesArray.length; i++) {
+            let dataSetModule = thisObject.dataSetsModulesArray[i] 
+            dataSetModule.finalize()
+        }
+        thisObject.dataSetsModulesArray = undefined
+        filter = undefined
+        bot = undefined
+        thisObject = undefined
+    }
+
     function isItADepenency(timeFrame, product) {
         let key = timeFrame + '-' + product 
 

TaskServer/DataSet.js

@@ -6,7 +6,9 @@ exports.newDataSet = function newDataSet(BOT, logger) {
 
     let thisObject = {
         node: undefined,
+        networkNode: undefined,
         initialize: initialize,
+        finalize: finalize,
         getTextFile: getTextFile,
         createTextFile: createTextFile
     };
@@ -14,7 +16,7 @@ exports.newDataSet = function newDataSet(BOT, logger) {
     /* Storage account to be used here. */
 
     const FILE_STORAGE = require('./FileStorage.js');
-    let fileStorage = FILE_STORAGE.newFileStorage(logger);
+    let fileStorage 
 
     return thisObject;
 
@@ -76,7 +78,48 @@ exports.newDataSet = function newDataSet(BOT, logger) {
                 return
             }
 
-            callBackFunction(global.DEFAULT_OK_RESPONSE);
+            /* Now we will see where do we need to fetch this data from. */
+            let network = global.TASK_NETWORK 
+            let datasetProductDefinition = thisObject.node.parentNode
+
+            for (let i = 0; i < network.networkNodes.length; i++) {
+                let networkNode = network.networkNodes[i]
+                if (networkNode.dataStorage !== undefined) {
+                    if (networkNode.dataStorage.sessionIndependentData !== undefined) {
+                        for (let j = 0; j < networkNode.dataStorage.sessionIndependentData.exchangeDataProducts.length; j++) {
+                            let exchangeDataProduct = networkNode.dataStorage.sessionIndependentData.exchangeDataProducts[j]
+                            for (let k = 0; k < exchangeDataProduct.singleMarketData.length; k++) {
+                                let singleMarketData = exchangeDataProduct.singleMarketData[k]
+                                if (singleMarketData.referenceParent !== undefined) {
+                                    let market = singleMarketData.referenceParent
+                                    let currentProcessMarket = bot.processNode.marketReference.referenceParent
+
+                                    if (currentProcessMarket.id === market.id) {
+                                        for (let m = 0; m < singleMarketData.dataProducts.length; m++) {
+                                            let dataProduct = singleMarketData.dataProducts[m]
+                                            if (dataProduct.referenceParent !== undefined) {
+                                                let productDefinition = dataProduct.referenceParent
+                                                if (datasetProductDefinition.id === productDefinition.id) {
+
+                                                    /* We found where the data is located on the network. */
+                                                    if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving data from " + networkNode.name + "  -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); }
+
+                                                    fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort);
+                                                    callBackFunction(global.DEFAULT_OK_RESPONSE);
+                                                    return
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            callBackFunction(global.DEFAULT_FAIL_RESPONSE);
+            logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this dataset is located within the network. Check the logs for more info.");
 
         } catch (err) {
             logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack);
@@ -84,6 +127,13 @@ exports.newDataSet = function newDataSet(BOT, logger) {
         }
     }
 
+    function finalize() {
+        fileStorage = undefined
+        thisObject.networkNode = undefined
+        bot = undefined
+        thisObject = undefined
+    }
+
     function getTextFile(pFolderPath, pFileName, callBackFunction) {
 
         try {

TaskServer/EventServerClient.js

@@ -1,4 +1,4 @@
-exports.newEventsServerClient = function newEventsServerClient() {
+exports.newEventsServerClient = function newEventsServerClient(host, port) {
 
     const MODULE = "Task Server "
     const ERROR_LOG = true
@@ -19,7 +19,14 @@
 
     let WEB_SOCKETS_CLIENT
     const WEB_SOCKET = require('ws')
-    let port = process.env.WEB_SOCKETS_SERVER_PORT  
+
+    if (host === undefined) {
+        host = 'localhost'
+    }
+    if (port === undefined) {
+        port = process.env.WEB_SOCKETS_SERVER_PORT  
+    }
+     
 
     return thisObject
 
@@ -32,7 +39,7 @@
     function setuptWebSockets(callBackFunction) {
         try {
 
-            WEB_SOCKETS_CLIENT = new WEB_SOCKET('ws://localhost:' + port ) 
+            WEB_SOCKETS_CLIENT = new WEB_SOCKET('ws://' + host + ':' + port ) 
 
             WEB_SOCKETS_CLIENT.onerror = error => {
                 console.log('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> On connection error -> error = ' + JSON.stringify(error))

TaskServer/FileStorage.js

@@ -1,7 +1,7 @@
 
 const path = require('path')
 
-exports.newFileStorage = function newFileStorage(logger) {
+exports.newFileStorage = function newFileStorage(logger, host, port) {
 
     const MODULE_NAME = 'FileStorage'
     const MAX_RETRY = 30
@@ -51,9 +51,15 @@ exports.newFileStorage = function newFileStorage(logger) {
 
                 logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> fileLocation: ' + fileLocation)
 
-                /* Here we actually write the file. */
-                const fs = require('fs')
-                fs.readFile(fileLocation, onFileRead)
+                /* Here we actually reaad the file. */
+                if (host === undefined || host === 'localhost') {
+                    /* We read the file from the local file system. */
+                    const fs = require('fs')
+                    fs.readFile(fileLocation, onFileRead)
+                } else {
+                    /* We read the file via a web server over http */
+                    getFileViaHTTP(fileLocation, onFileRead)
+                }
 
                 function onFileRead(err, text) {
                     let retryTimeToUse = FAST_RETRY_TIME_IN_MILISECONDS
@@ -118,7 +124,14 @@ exports.newFileStorage = function newFileStorage(logger) {
                         if (canUsePrevious === true) {
                             logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Could read the file, but could not parse it as it is not a valid JSON. Will try to read the PREVIOUS version instead. -> file = ' + fileLocation)
 
-                            fs.readFile(fileLocation + '.Previous.json', onPreviousFileRead)
+                            if (host === undefined || host === 'localhost') {
+                                /* We read the file from the local file system. */
+                                const fs = require('fs')
+                                fs.readFile(fileLocation + '.Previous.json', onPreviousFileRead)
+                            } else {
+                                /* We read the file via a web server over http */
+                                getFileViaHTTP(fileLocation + '.Previous.json', onPreviousFileRead)
+                            }
 
                             function onPreviousFileRead(err, text) {
 
@@ -136,6 +149,7 @@ exports.newFileStorage = function newFileStorage(logger) {
                         } else {
                             logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Could read the file, but could not parse it as it is not a valid JSON.')
                             logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> err = ' + err.stack)
+                            logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> text.toString() = ' + text.toString())
                             setTimeout(retry, retryTimeToUse)
                             return
                         }
@@ -390,9 +404,8 @@ exports.newFileStorage = function newFileStorage(logger) {
         }
     }
 
- /* Function to create folders of missing folders at any path. */
-
-  function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
+    /* Function to create folders of missing folders at any path. */
+    function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
     const sep = '/';
     const initDir = path.isAbsolute(targetDir) ? sep : '';
     const baseDir = isRelativeToScript ? __dirname : '.';
@@ -420,5 +433,50 @@ exports.newFileStorage = function newFileStorage(logger) {
 
       return curDir;
     }, initDir);
-  }
+    }
+
+    function getFileViaHTTP(filePath, callback) {
+        try {
+
+            /* The filePath received is the one that is needed to fetch data from with fs. To do it via http we need to remove the prefix that includes this: ./Data-Storage/  */
+            filePath = filePath.substring(15, filePath.length)
+
+            let http = require('http');
+            let url = 'http://' + host +
+                ':' + port +
+                '/Storage/' +
+                filePath
+
+            http.get(url, onResponse);
+
+            function onResponse(response) {
+
+                const chunks = []
+
+                response.on('data', onMessegesArrived)
+                response.on('end', onEnd)
+
+                function onMessegesArrived(chunk) {
+                    chunks.push(chunk)
+                }
+
+                function onEnd() {
+                    let fileContent = Buffer.concat(chunks).toString('utf8')
+                    let err = null
+                    if (fileContent === 'The specified key does not exist.') {
+                        err = {
+                            code: "ENOENT" // This is how fs would have returned upon this situation.
+                        }
+                        fileContent = undefined
+                    }
+                    callback(err, fileContent)
+                }
+            }
+
+        } catch (err) {
+            logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> err = " + err.stack);
+            callBackFunction(global.DEFAULT_FAIL_RESPONSE);
+            abort = true
+        }
+    }
 }

TaskServer/IndicatorBotProcessMainLoop.js

@@ -252,6 +252,12 @@
                                     switch (err.result) {
                                         case global.DEFAULT_OK_RESPONSE.result: {
                                             logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well.");
+
+                                            if (global.STOP_TASK_GRACEFULLY === true) {
+                                                loopControl()
+                                                return
+                                            }
+
                                             initializeStatusDependencies();
                                             return;
                                         }
@@ -776,10 +782,13 @@
                             function onFinished(err) {
 
                                 try {
-
                                     if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> onFinished -> Entering function."); }
                                     processFramework.finalize()
                                     processFramework = undefined
+                                    dataDependencies.finalize()
+                                    dataDependencies = undefined
+                                    statusDependencies.finalize()
+                                    statusDependencies = undefined
 
                                     switch (err.result) {
                                         case global.DEFAULT_OK_RESPONSE.result: {
@@ -876,11 +885,12 @@
                             processExecutionEvents.finish(onFinished);
 
                             function onFinished(err) {
-
                                 try {
-
                                     if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents ->  onFinished -> Entering function."); }
 
+                                    processExecutionEvents.finalize()
+                                    processExecutionEvents = undefined
+
                                     switch (err.result) {
                                         case global.DEFAULT_OK_RESPONSE.result: {
                                             logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Execution finished well.");

TaskServer/ProcessExecutionEvents.js

@@ -8,27 +8,33 @@
     */
 
     const MODULE_NAME = "Process Execution Events";
+    logger.fileName = MODULE_NAME 
 
     let bot = BOT;
 
     let thisObject = {
+        networkNode: undefined,
         initialize, initialize,
+        finalize: finalize,
         start: start,
         finish: finish
     };
 
     let processThisDependsOn;                       // This is the process this process needs to wait for it to finishes in order to start.
-
     let currentProcessKey
 
+    let EVENT_SERVER_CLIENT = require('./EventServerClient.js');
+
     return thisObject;
 
     function initialize(callBackFunction) {
 
         try {
+
             if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
 
             let name = 'Not Depends on any Process'
+            logger.fileName = MODULE_NAME + "." + name;
 
             if (bot.processNode.referenceParent !== undefined) {
 
@@ -48,9 +54,63 @@
                         }
                     }
                 }
+
+                /* We need to find at which network node is running the process that we need to hear from when it finished. */
+                let network = global.TASK_NETWORK 
+
+                for (let i = 0; i < network.networkNodes.length; i++) {
+                    let networkNode = network.networkNodes[i]
+                    if (networkNode.dataMining !== undefined) {
+                        for (let j = 0; j < networkNode.dataMining.exchangeTasks.length; j++) {
+                            let exchangeTasks = networkNode.dataMining.exchangeTasks[j]
+                            for (let k = 0; k < exchangeTasks.taskManagers.length; k++) {
+                                let taskManager = exchangeTasks.taskManagers[k]
+                                for (let m = 0; m < taskManager.tasks.length; m++) {
+                                    let task = taskManager.tasks[m]
+                                    if (task.bot !== undefined) {
+                                        for (let n = 0; n < task.bot.processes.length; n++) {
+                                            let process = task.bot.processes[n]
+                                            if (process.marketReference !== undefined) {
+                                                if (process.marketReference.referenceParent !== undefined) {
+                                                    let market = process.marketReference.referenceParent
+                                                    let currentProcessMarket = bot.processNode.marketReference.referenceParent
+
+                                                    if (currentProcessMarket.id === market.id) { 
+                                                        if (process.referenceParent !== undefined) {
+                                                            let processDefinition = process.referenceParent
+                                                            if (processThisDependsOn.id === processDefinition.id) {
+
+                                                                /* We found where the task that runs the process definition we are waiting for is located on the network. */
+
+                                                                thisObject.networkNode = networkNode
+                                                                if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Connecting to Websockets Server " + networkNode.name + "  -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webSocketsPort + '.'); }
+
+                                                                EVENT_SERVER_CLIENT = EVENT_SERVER_CLIENT.newEventsServerClient(networkNode.code.host, networkNode.code.webSocketsPort)
+                                                                EVENT_SERVER_CLIENT.initialize(onConnected)
+
+                                                                function onConnected() {
+                                                                    if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Connected to Websockets Server " + networkNode.name + "  -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webSocketsPort + '.'); }
+
+                                                                    callBackFunction(global.DEFAULT_OK_RESPONSE);
+                                                                    return
+                                                                }
+                                                                return
+                                                            }
+                                                        }
+                                                    }
+                                                } 
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                callBackFunction(global.DEFAULT_FAIL_RESPONSE);
+                logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the task that runs the Process Definition we are waiting for is located within the network. Check the logs for more info.");
+                return
             }
-        
-            logger.fileName = MODULE_NAME + "." + name;
             callBackFunction(global.DEFAULT_OK_RESPONSE);
 
         } catch (err) {
@@ -59,6 +119,16 @@
         }
     }
 
+    function finalize() {
+        EVENT_SERVER_CLIENT.finalize()
+        EVENT_SERVER_CLIENT = undefined
+        processThisDependsOn = undefined
+        currentProcessKey = undefined
+        thisObject.networkNode = undefined
+        bot = undefined
+        thisObject = undefined
+    }
+
     function start(callBackFunction) {
 
         try {
@@ -81,9 +151,9 @@
                 if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> waitForDependantProcess -> key = " + key); }
                 if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> waitForDependantProcess -> callerId = " + callerId); }
 
-                global.EVENT_SERVER_CLIENT.listenToEvent(key, 'Process Execution Finished', undefined, callerId, responseCallBack, eventsCallBack)
+                EVENT_SERVER_CLIENT.listenToEvent(key, 'Process Execution Finished', undefined, callerId, responseCallBack, eventsCallBack)
 
-                bot.processHeartBeat(undefined, undefined, "Waiting for " + processThisDependsOn.parentNode.parentNode.name + "->" + processThisDependsOn.parentNode.code.codeName + "->" + processThisDependsOn.code.codeName)  
+                bot.processHeartBeat(undefined, undefined, "Waiting for " + thisObject.networkNode.name + "->" + processThisDependsOn.parentNode.parentNode.name + "->" + processThisDependsOn.parentNode.code.codeName + "->" + processThisDependsOn.code.codeName)  
 
                 function responseCallBack(message) {
                     if (message.result !== global.DEFAULT_OK_RESPONSE.result) {
@@ -102,7 +172,7 @@
                     /* We continue the normal flow after we learn the dependent process has finished its execution. */
                     if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> eventsCallBack -> stopListening to Process Execution Finished. "); }
                     if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> eventsCallBack -> subscriptionId = " + subscriptionId); }
-                    global.EVENT_SERVER_CLIENT.stopListening(key, 'Process Execution Finished', subscriptionId)
+                    EVENT_SERVER_CLIENT.stopListening(key, 'Process Execution Finished', subscriptionId)
 
                     if (message.event.err.result === global.DEFAULT_OK_RESPONSE.result) {
                         /* We emit our own event that the process started. */

TaskServer/SensorBotProcessMainLoop.js

@@ -254,6 +254,12 @@
                                     switch (err.result) {
                                         case global.DEFAULT_OK_RESPONSE.result: {
                                             logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well.");
+
+                                            if (global.STOP_TASK_GRACEFULLY === true) {
+                                                loopControl()
+                                                return
+                                            }
+
                                             initializeStatusDependencies();
                                             return;
                                         }
@@ -584,11 +590,12 @@
                             processExecutionEvents.finish(onFinished);
 
                             function onFinished(err) {
-
                                 try {
-
                                     if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents ->  onFinished -> Entering function."); }
 
+                                    processExecutionEvents.finalize()
+                                    processExecutionEvents =  undefined
+
                                     switch (err.result) {
                                         case global.DEFAULT_OK_RESPONSE.result: {
                                             logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Execution finished well.");

TaskServer/StatusDependencies.js

@@ -11,6 +11,7 @@
         statusReports: new Map(),
         reportsByMainUtility: new Map(),
         initialize: initialize,
+        finalize: finalize,
         keys: []
     };
 
@@ -165,4 +166,13 @@
         }
     }
 
+    function finalize() {
+        thisObject.statusReports.forEach(forEachStatusDependency)
+        function forEachStatusDependency(statusDependency) {
+            statusDependency.finalize()
+        }
+        thisObject.statusReports = undefined
+        bot = undefined
+        thisObject = undefined
+    }
 };

TaskServer/StatusReport.js

@@ -13,9 +13,11 @@
     let bot = BOT;
 
     let thisObject = {
+        networkNode: undefined,
         mainUtility: undefined,
         file: undefined,                    // Here we have the JSON object representing the file content.
         initialize: initialize,
+        finalize: finalize,
         load: load,
         save: save,
         status: undefined
@@ -30,7 +32,7 @@
     /* Storage account to be used here. */
 
     const FILE_STORAGE = require('./FileStorage.js');
-    let fileStorage = FILE_STORAGE.newFileStorage(logger);
+    let fileStorage  
 
     let sessionPath = ''
 
@@ -93,17 +95,6 @@
             statusDependencyNode.bottype = statusDependencyNode.referenceParent.parentNode.parentNode.type
             statusDependencyNode.dataMine = statusDependencyNode.referenceParent.parentNode.parentNode.parentNode.code.codeName
 
-            /* Let's see if the process is run monthly or not. */
-            if (statusDependencyNode.referenceParent.parentNode.code.startMode !== undefined) {
-                if (statusDependencyNode.referenceParent.parentNode.code.startMode.allMonths !== undefined) {
-                    if (statusDependencyNode.referenceParent.parentNode.code.startMode.oneMonth !== undefined) {
-                        if (statusDependencyNode.referenceParent.parentNode.code.startMode.allMonths.run === "true" || statusDependencyNode.referenceParent.parentNode.code.startMode.oneMonth.run === "true") {
-                            statusDependencyNode.processRunMonthly = true
-                        }
-                    }
-                }
-            }
-
             /* We retrieve the report main utility */
             if (statusDependencyNode.code !== undefined) {
                 if (statusDependencyNode.code.mainUtility !== undefined) {
@@ -122,7 +113,55 @@
                 sessionPath = bot.SESSION.folderName + "/"
             }
 
-            callBackFunction(global.DEFAULT_OK_RESPONSE);
+            /* Now we will see where do we need to fetch this status report from. */
+            let network = global.TASK_NETWORK
+            let processThisDependsOn = statusDependencyNode.referenceParent.parentNode
+
+            for (let i = 0; i < network.networkNodes.length; i++) {
+                let networkNode = network.networkNodes[i]
+                if (networkNode.dataMining !== undefined) {
+                    for (let j = 0; j < networkNode.dataMining.exchangeTasks.length; j++) {
+                        let exchangeTasks = networkNode.dataMining.exchangeTasks[j]
+                        for (let k = 0; k < exchangeTasks.taskManagers.length; k++) {
+                            let taskManager = exchangeTasks.taskManagers[k]
+                            for (let m = 0; m < taskManager.tasks.length; m++) {
+                                let task = taskManager.tasks[m]
+                                if (task.bot !== undefined) {
+                                    for (let n = 0; n < task.bot.processes.length; n++) {
+                                        let process = task.bot.processes[n]
+                                        if (process.marketReference !== undefined) {
+                                            if (process.marketReference.referenceParent !== undefined) {
+                                                let market = process.marketReference.referenceParent
+                                                let currentProcessMarket = bot.processNode.marketReference.referenceParent
+                                                if (currentProcessMarket.id === market.id) {
+                                                    if (process.referenceParent !== undefined) {
+                                                        let processDefinition = process.referenceParent
+                                                        if (processThisDependsOn.id === processDefinition.id) {
+
+                                                            /* We found where the task that runs the process definition this status report depends on and where it is located on the network. */
+
+                                                            thisObject.networkNode = networkNode
+                                                            if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving status report from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); }
+
+                                                            fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort);
+                                                            callBackFunction(global.DEFAULT_OK_RESPONSE);
+                                                            return
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            callBackFunction(global.DEFAULT_FAIL_RESPONSE);
+            logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this status report is located within the network. Check the logs for more info.");
+
 
         } catch (err) {
             logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack);
@@ -130,6 +169,13 @@
         }
     }
 
+    function finalize() {
+        fileStorage = undefined
+        thisObject.networkNode = undefined
+        bot = undefined
+        thisObject = undefined
+    }
+
     function load(callBackFunction) {
 
         try {
@@ -212,6 +258,7 @@
                     */
 
                     logger.write(MODULE_NAME, "[ERROR] load -> onFileReceived -> Error Parsing the Status report. -> Err = " + err.message);
+                    logger.write(MODULE_NAME, "[WARN] load -> onFileReceived -> Error Parsing the Status report. -> text = " + text);
 
                     let customFail = {
                         result: global.CUSTOM_FAIL_RESPONSE.result,

TaskServer/TradingBotProcessMainLoop.js

@@ -288,6 +288,12 @@
                                     switch (err.result) {
                                         case global.DEFAULT_OK_RESPONSE.result: {
                                             logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well.");
+
+                                            if (global.STOP_TASK_GRACEFULLY === true) {
+                                                loopControl()
+                                                return
+                                            }
+
                                             initializeStatusDependencies();
                                             return;
                                         }
@@ -694,6 +700,10 @@
                                     if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> onFinished -> Entering function."); }
                                     processFramework.finalize()
                                     processFramework = undefined
+                                    dataDependencies.finalize()
+                                    dataDependencies = undefined
+                                    statusDependencies.finalize()
+                                    statusDependencies = undefined
 
                                     switch (err.result) {
                                         case global.DEFAULT_OK_RESPONSE.result: {
@@ -792,8 +802,9 @@
                             processExecutionEvents.finish(onFinished);
 
                             function onFinished(err) {
-
                                 try {
+                                    processExecutionEvents.finalize()
+                                    processExecutionEvents = undefined
 
                                     if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents ->  onFinished -> Entering function."); }
 

TaskServer/server.js

@@ -125,12 +125,13 @@ function preLoader() {
             global.EVENT_SERVER_CLIENT.raiseEvent('Task Manager - ' + taskId, 'Nodejs Process Ready for Task')
             function eventReceived(message) {
                 global.TASK_NODE = message
-                global.TASK_NODE = JSON.parse(message.event.definition)
+                global.TASK_NODE = JSON.parse(message.event.taskDefinition)
+                global.TASK_NETWORK = JSON.parse(message.event.networkDefinition)
                 bootLoader()
             }
         } catch (err) {
             console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE -> ' + err.stack)
-            console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE))
+            console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE).substring(0, 1000))
         }
     }
     else {  /* This process was started not by the Task Manager, but independently (most likely for debugging purposes). In this case we listen to an event with the Task Info that should be emitted at the UI */
@@ -139,12 +140,13 @@ function preLoader() {
             global.EVENT_SERVER_CLIENT.listenToEvent('Task Server', 'Debug Task Started', undefined, 'Task Server', undefined, startDebugging)
             function startDebugging(message) {
                 global.TASK_NODE = message
-                global.TASK_NODE = JSON.parse(message.event.definition) 
+                global.TASK_NODE = JSON.parse(message.event.taskDefinition)
+                global.TASK_NETWORK = JSON.parse(message.event.networkDefinition)
                 bootLoader()
             }
         } catch (err) {
             console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE -> ' + err.stack)
-            console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE))
+            console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE).substring(0, 1000))
         }
     }
 }