From a3efd3eaf07b7dcde42a7128a12646d7ad651e3d Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 24 Apr 2020 13:40:42 +0200 Subject: [PATCH] Home Network fully implemented --- BackendServers/taskManagerServer.js | 12 +-- BackendServers/webSocketsServer.js | 10 +-- .../FunctionLibraries/ProtocolNode.js | 16 ++-- .../FunctionLibraries/TaskFunctions.js | 42 +++++++++- CanvasApp/DesignSpace/Workspace/Workspace.js | 2 +- CanvasApp/EventsServerClient.js | 4 +- CanvasApp/FloatingSpace/UiObject.js | 2 +- .../FloatingSpace/UiObjectConstructor.js | 15 +++- TaskServer/DataDependencies.js | 14 +++- TaskServer/DataSet.js | 54 +++++++++++- TaskServer/EventServerClient.js | 13 ++- TaskServer/FileStorage.js | 76 +++++++++++++++-- TaskServer/IndicatorBotProcessMainLoop.js | 16 +++- TaskServer/ProcessExecutionEvents.js | 82 +++++++++++++++++-- TaskServer/SensorBotProcessMainLoop.js | 11 ++- TaskServer/StatusDependencies.js | 10 +++ TaskServer/StatusReport.js | 73 ++++++++++++++--- TaskServer/TradingBotProcessMainLoop.js | 13 ++- TaskServer/server.js | 10 ++- 19 files changed, 401 insertions(+), 74 deletions(-) diff --git a/BackendServers/taskManagerServer.js b/BackendServers/taskManagerServer.js index eb775ee38..7fa44ce01 100644 --- a/BackendServers/taskManagerServer.js +++ b/BackendServers/taskManagerServer.js @@ -51,17 +51,17 @@ //console.log('[INFO] BackendServers -> Task Manager Server -> runTask -> Entering function.') if (message.event === undefined) { - console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without Event -> message = ' + JSON.stringify(message)) + console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without Event -> message = ' + JSON.stringify(message).substring(0, 1000)) return } if (message.event.taskId === undefined) { - console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without taskId -> message = ' + JSON.stringify(message)) + console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without taskId -> message = ' + JSON.stringify(message).substring(0, 1000)) return } - if (message.event.definition === undefined) { - console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without Definition -> message = ' + JSON.stringify(message)) + if (message.event.taskDefinition === undefined) { + console.log('[WARN] BackendServers -> Task Manager Server -> runTask -> Message Received Without taskDefinition -> message = ' + JSON.stringify(message).substring(0, 1000)) return } @@ -116,12 +116,12 @@ //console.log('[INFO] BackendServers -> Task Manager Server -> stopTask -> Entering function.') if (message.event === undefined) { - console.log('[WARN] BackendServers -> Task Manager Server -> stopTask -> Message Received Without Event -> message = ' + JSON.stringify(message)) + console.log('[WARN] BackendServers -> Task Manager Server -> stopTask -> Message Received Without Event -> message = ' + JSON.stringify(message).substring(0, 1000)) return } if (message.event.taskId === undefined) { - console.log('[WARN] BackendServers -> Task Manager Server -> stopTask -> Message Received Without taskId -> message = ' + JSON.stringify(message)) + console.log('[WARN] BackendServers -> Task Manager Server -> stopTask -> Message Received Without taskId -> message = ' + JSON.stringify(message).substring(0, 1000)) return } diff --git a/BackendServers/webSocketsServer.js b/BackendServers/webSocketsServer.js index f4737adc8..537eb974f 100644 --- a/BackendServers/webSocketsServer.js +++ b/BackendServers/webSocketsServer.js @@ -47,7 +47,7 @@ try { if (LOG_INFO === true) { - console.log('Message Received: ' + message) + console.log('Message Received: ' + message.substring(0, 1000)) } let messageArray = message.split('|*|') @@ -58,7 +58,7 @@ if (origin === 'Web Browser') { if (isNaN(nonce) || nonce === "") { - console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce is not a Number. message = ' + message) + console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce is not a Number. message = ' + message.substring(0, 1000)) console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce is not a Number. nonce = ' + nonce) return } @@ -68,7 +68,7 @@ } if (Number(nonce) < Number(lastNonce)) { - console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce received is less than Last Nonce. message = ' + message) + console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce received is less than Last Nonce. message = ' + message.substring(0, 1000)) console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce received is less than Last Nonce. nonce = ' + nonce) console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Nonce received is less than Last Nonce. lastNonce = ' + lastNonce) return @@ -79,7 +79,7 @@ try { let jsonCheck = JSON.parse(messageToEventServer) } catch (err) { - console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Message received from the browser is not a valid JSON. message = ' + message) + console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Message received from the browser is not a valid JSON. message = ' + message.substring(0, 1000)) console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> Message received from the browser is not a valid JSON. messageToEventServer = ' + messageToEventServer) return } @@ -106,7 +106,7 @@ console.log('Web Sockets Server Started.') } catch (err) { - console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> err.message = ' + err.message) + console.log('[ERROR] BackendServers -> Web Sockets Server -> run -> setUpWebSocketServer -> err.message = ' + err.message.substring(0, 1000)) } } } \ No newline at end of file diff --git a/CanvasApp/DesignSpace/Workspace/FunctionLibraries/ProtocolNode.js b/CanvasApp/DesignSpace/Workspace/FunctionLibraries/ProtocolNode.js index 3fff812a4..73d626530 100644 --- a/CanvasApp/DesignSpace/Workspace/FunctionLibraries/ProtocolNode.js +++ b/CanvasApp/DesignSpace/Workspace/FunctionLibraries/ProtocolNode.js @@ -24,23 +24,19 @@ function newProtocolNode () { /* Children Nodes */ if (nodeDefinition.properties !== undefined) { - let previousPropertyName // Since there are cases where there are many properties with the same name,because they can hold nodes of different types but only one at the time, we have to avoind counting each property of those as individual children. for (let i = 0; i < nodeDefinition.properties.length; i++) { let property = nodeDefinition.properties[i] switch (property.type) { case 'node': { if (node[property.name] !== undefined) { - if (property.name !== previousPropertyName) { - if (lightingPath === undefined) { - object[property.name] = getProtocolNode(node[property.name], removePersonalData, parseCode, includeIds, includeReferencesInSavedPayload, includePayload) - } else { - let newLightingPath = getNewLightingPath(lightingPath, node.type, property.childType) - if (newLightingPath !== undefined) { - object[property.name] = getProtocolNode(node[property.name], removePersonalData, parseCode, includeIds, includePayload, includeReferencesInSavedPayload, newLightingPath) - } + if (lightingPath === undefined) { + object[property.name] = getProtocolNode(node[property.name], removePersonalData, parseCode, includeIds, includeReferencesInSavedPayload, includePayload) + } else { + let newLightingPath = getNewLightingPath(lightingPath, node.type, property.childType) + if (newLightingPath !== undefined) { + object[property.name] = getProtocolNode(node[property.name], removePersonalData, parseCode, includeIds, includePayload, includeReferencesInSavedPayload, newLightingPath) } - previousPropertyName = property.name } } break diff --git a/CanvasApp/DesignSpace/Workspace/FunctionLibraries/TaskFunctions.js b/CanvasApp/DesignSpace/Workspace/FunctionLibraries/TaskFunctions.js index 4a83f27db..54af3c9c7 100644 --- a/CanvasApp/DesignSpace/Workspace/FunctionLibraries/TaskFunctions.js +++ b/CanvasApp/DesignSpace/Workspace/FunctionLibraries/TaskFunctions.js @@ -26,7 +26,7 @@ function newTaskFunctions () { process.payload.uiObject.run(eventsServerClient) } - let lightingPath = '->Task->' + + let taskLightingPath = '->Task->' + 'Sensor Bot Instance->' + 'Indicator Bot Instance->' + 'Trading Bot Instance->' + @@ -56,12 +56,26 @@ function newTaskFunctions () { 'Trading Bot->' + 'Data Mine->' - let definition = functionLibraryProtocolNode.getProtocolNode(node, false, true, true, false, false, lightingPath) + let taskDefinition = functionLibraryProtocolNode.getProtocolNode(node, false, true, true, false, false, taskLightingPath) + + let networkLightingPath = '->Network->Network Node->' + + 'Data Storage->Session Independent Data->Exchange Data Products->' + + 'Single Market Data->Data Product->Product Definition->' + + 'Data Mining->Testing Environment->Production Environment->' + + 'Exchange Tasks->Crypto Exchange->' + + 'Task Manager->Task->' + + 'Indicator Bot Instance->Sensor Bot Instance->' + + 'Indicator Process Instance->Sensor Process Instance->' + + 'Market Reference->Market->' + + 'Process Definition->' + + let networkDefinition = functionLibraryProtocolNode.getProtocolNode(networkNode.payload.parentNode, false, true, true, false, false, networkLightingPath) let event = { taskId: node.id, taskName: node.name, - definition: JSON.stringify(definition) // <- We need to do this workaround in order no to send unescaped charactars to the taskManager. + taskDefinition: JSON.stringify(taskDefinition), + networkDefinition: JSON.stringify(networkDefinition) } if (isDebugging === true) { @@ -129,6 +143,28 @@ function newTaskFunctions () { node.payload.uiObject.setErrorMessage('Task needs to be inside a Network Node.') return } + + if (node.payload.parentNode.payload.parentNode.payload.parentNode.payload.parentNode.payload.parentNode === undefined) { + node.payload.uiObject.setErrorMessage('Task needs to be inside a Network.') + return + } + + let networkNode = node.payload.parentNode.payload.parentNode.payload.parentNode.payload.parentNode + if (loadPropertyFromNodeConfig(networkNode.payload, 'host') === undefined) { + node.payload.uiObject.setErrorMessage('Network Node needs to have a valid Host property at its config.') + return + } + + if (loadPropertyFromNodeConfig(networkNode.payload, 'webPort') === undefined) { + node.payload.uiObject.setErrorMessage('Network Node needs to have a valid webPort property at its config.') + return + } + + if (loadPropertyFromNodeConfig(networkNode.payload, 'webSocketsPort') === undefined) { + node.payload.uiObject.setErrorMessage('Network Node needs to have a valid webSocketsPort property at its config.') + return + } + return true } diff --git a/CanvasApp/DesignSpace/Workspace/Workspace.js b/CanvasApp/DesignSpace/Workspace/Workspace.js index a58fb46a1..b8e4062f4 100644 --- a/CanvasApp/DesignSpace/Workspace/Workspace.js +++ b/CanvasApp/DesignSpace/Workspace/Workspace.js @@ -126,7 +126,7 @@ function newWorkspace () { if (host === undefined) { host = 'localhost' } if (webSocketsPort === undefined) { webSocketsPort = '8080' } - let eventsServerClient = newEventsServerClient(host, webSocketsPort) + let eventsServerClient = newEventsServerClient(host, webSocketsPort, networkNode.name) eventsServerClient.initialize() thisObject.eventsServerClients.set(networkNode.id, eventsServerClient) diff --git a/CanvasApp/EventsServerClient.js b/CanvasApp/EventsServerClient.js index a94c1495a..1c5301e37 100644 --- a/CanvasApp/EventsServerClient.js +++ b/CanvasApp/EventsServerClient.js @@ -1,5 +1,5 @@ -function newEventsServerClient (host, port) { +function newEventsServerClient (host, port, hostName) { /* Web Sockets Connection */ const MODULE_NAME = 'System Event Handler' @@ -153,7 +153,7 @@ function newEventsServerClient (host, port) { } if (thisObject.isConnected() !== true) { - canvas.cockpitSpace.setStatus('Connecting with ' + host + '. Please hold on until the connection can be stablished.', 100, canvas.cockpitSpace.statusTypes.WARNING) + canvas.cockpitSpace.setStatus('Connecting to ' + hostName + ' Network Node. Please hold on until the connection can be stablished.', 100, canvas.cockpitSpace.statusTypes.WARNING) } else { retryCommandsPhysics() } diff --git a/CanvasApp/FloatingSpace/UiObject.js b/CanvasApp/FloatingSpace/UiObject.js index 136c3c9a0..574fd4a4c 100644 --- a/CanvasApp/FloatingSpace/UiObject.js +++ b/CanvasApp/FloatingSpace/UiObject.js @@ -1424,7 +1424,7 @@ function newUiObject () { let label if (radius > 6) { - const MAX_LABEL_LENGTH = 65 + const MAX_LABEL_LENGTH = 100 label = currentStatus diff --git a/CanvasApp/FloatingSpace/UiObjectConstructor.js b/CanvasApp/FloatingSpace/UiObjectConstructor.js index 03f80b2ac..6c02e1d2c 100644 --- a/CanvasApp/FloatingSpace/UiObjectConstructor.js +++ b/CanvasApp/FloatingSpace/UiObjectConstructor.js @@ -157,6 +157,17 @@ function newUiObjectConstructor () { } } + /* Default Values in case there was no way to set a value previous to this. */ + if (floatingObject.angleToParent === undefined) { + floatingObject.angleToParent = ANGLE_TO_PARENT.RANGE_360 + } + if (floatingObject.distanceToParent === undefined) { + floatingObject.distanceToParent = DISTANCE_TO_PARENT.PARENT_100X + } + if (floatingObject.arrangementStyle === undefined) { + floatingObject.arrangementStyle = ARRANGEMENT_STYLE.CONCAVE + } + let uiObject = newUiObject() payload.uiObject = uiObject uiObject.fitFunction = canvas.floatingSpace.fitIntoVisibleArea @@ -243,7 +254,7 @@ function newUiObjectConstructor () { ) menuItemsInitialValues.push( { - action: 'change Distance to Paarent', + action: 'Change Distance to Paarent', actionFunction: floatingObject.distanceToParentToggle, actionStatus: floatingObject.getDistanceToParent, currentStatus: true, @@ -258,7 +269,7 @@ function newUiObjectConstructor () { ) menuItemsInitialValues.push( { - action: 'change Arrangement Style', + action: 'Change Arrangement Style', actionFunction: floatingObject.arrangementStyleToggle, actionStatus: floatingObject.getArrangementStyle, currentStatus: true, diff --git a/TaskServer/DataDependencies.js b/TaskServer/DataDependencies.js index a833ad6a3..3ea9ea759 100644 --- a/TaskServer/DataDependencies.js +++ b/TaskServer/DataDependencies.js @@ -10,7 +10,8 @@ nodeArray: undefined, dataSetsModulesArray: [], isItADepenency: isItADepenency, - initialize: initialize + initialize: initialize, + finalize: finalize }; let filter = new Map() @@ -103,6 +104,17 @@ } } + function finalize() { + for (let i = 0; i < thisObject.dataSetsModulesArray.length; i++) { + let dataSetModule = thisObject.dataSetsModulesArray[i] + dataSetModule.finalize() + } + thisObject.dataSetsModulesArray = undefined + filter = undefined + bot = undefined + thisObject = undefined + } + function isItADepenency(timeFrame, product) { let key = timeFrame + '-' + product diff --git a/TaskServer/DataSet.js b/TaskServer/DataSet.js index 434a86ce1..066e6f7fc 100644 --- a/TaskServer/DataSet.js +++ b/TaskServer/DataSet.js @@ -6,7 +6,9 @@ exports.newDataSet = function newDataSet(BOT, logger) { let thisObject = { node: undefined, + networkNode: undefined, initialize: initialize, + finalize: finalize, getTextFile: getTextFile, createTextFile: createTextFile }; @@ -14,7 +16,7 @@ exports.newDataSet = function newDataSet(BOT, logger) { /* Storage account to be used here. */ const FILE_STORAGE = require('./FileStorage.js'); - let fileStorage = FILE_STORAGE.newFileStorage(logger); + let fileStorage return thisObject; @@ -76,7 +78,48 @@ exports.newDataSet = function newDataSet(BOT, logger) { return } - callBackFunction(global.DEFAULT_OK_RESPONSE); + /* Now we will see where do we need to fetch this data from. */ + let network = global.TASK_NETWORK + let datasetProductDefinition = thisObject.node.parentNode + + for (let i = 0; i < network.networkNodes.length; i++) { + let networkNode = network.networkNodes[i] + if (networkNode.dataStorage !== undefined) { + if (networkNode.dataStorage.sessionIndependentData !== undefined) { + for (let j = 0; j < networkNode.dataStorage.sessionIndependentData.exchangeDataProducts.length; j++) { + let exchangeDataProduct = networkNode.dataStorage.sessionIndependentData.exchangeDataProducts[j] + for (let k = 0; k < exchangeDataProduct.singleMarketData.length; k++) { + let singleMarketData = exchangeDataProduct.singleMarketData[k] + if (singleMarketData.referenceParent !== undefined) { + let market = singleMarketData.referenceParent + let currentProcessMarket = bot.processNode.marketReference.referenceParent + + if (currentProcessMarket.id === market.id) { + for (let m = 0; m < singleMarketData.dataProducts.length; m++) { + let dataProduct = singleMarketData.dataProducts[m] + if (dataProduct.referenceParent !== undefined) { + let productDefinition = dataProduct.referenceParent + if (datasetProductDefinition.id === productDefinition.id) { + + /* We found where the data is located on the network. */ + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving data from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); } + + fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return + } + } + } + } + } + } + } + } + } + } + + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this dataset is located within the network. Check the logs for more info."); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack); @@ -84,6 +127,13 @@ exports.newDataSet = function newDataSet(BOT, logger) { } } + function finalize() { + fileStorage = undefined + thisObject.networkNode = undefined + bot = undefined + thisObject = undefined + } + function getTextFile(pFolderPath, pFileName, callBackFunction) { try { diff --git a/TaskServer/EventServerClient.js b/TaskServer/EventServerClient.js index 3695225b2..a5de99f5a 100644 --- a/TaskServer/EventServerClient.js +++ b/TaskServer/EventServerClient.js @@ -1,4 +1,4 @@ -exports.newEventsServerClient = function newEventsServerClient() { +exports.newEventsServerClient = function newEventsServerClient(host, port) { const MODULE = "Task Server " const ERROR_LOG = true @@ -19,7 +19,14 @@ let WEB_SOCKETS_CLIENT const WEB_SOCKET = require('ws') - let port = process.env.WEB_SOCKETS_SERVER_PORT + + if (host === undefined) { + host = 'localhost' + } + if (port === undefined) { + port = process.env.WEB_SOCKETS_SERVER_PORT + } + return thisObject @@ -32,7 +39,7 @@ function setuptWebSockets(callBackFunction) { try { - WEB_SOCKETS_CLIENT = new WEB_SOCKET('ws://localhost:' + port ) + WEB_SOCKETS_CLIENT = new WEB_SOCKET('ws://' + host + ':' + port ) WEB_SOCKETS_CLIENT.onerror = error => { console.log('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> On connection error -> error = ' + JSON.stringify(error)) diff --git a/TaskServer/FileStorage.js b/TaskServer/FileStorage.js index 625c78fd5..199183aa3 100644 --- a/TaskServer/FileStorage.js +++ b/TaskServer/FileStorage.js @@ -1,7 +1,7 @@ const path = require('path') -exports.newFileStorage = function newFileStorage(logger) { +exports.newFileStorage = function newFileStorage(logger, host, port) { const MODULE_NAME = 'FileStorage' const MAX_RETRY = 30 @@ -51,9 +51,15 @@ exports.newFileStorage = function newFileStorage(logger) { logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> fileLocation: ' + fileLocation) - /* Here we actually write the file. */ - const fs = require('fs') - fs.readFile(fileLocation, onFileRead) + /* Here we actually reaad the file. */ + if (host === undefined || host === 'localhost') { + /* We read the file from the local file system. */ + const fs = require('fs') + fs.readFile(fileLocation, onFileRead) + } else { + /* We read the file via a web server over http */ + getFileViaHTTP(fileLocation, onFileRead) + } function onFileRead(err, text) { let retryTimeToUse = FAST_RETRY_TIME_IN_MILISECONDS @@ -118,7 +124,14 @@ exports.newFileStorage = function newFileStorage(logger) { if (canUsePrevious === true) { logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Could read the file, but could not parse it as it is not a valid JSON. Will try to read the PREVIOUS version instead. -> file = ' + fileLocation) - fs.readFile(fileLocation + '.Previous.json', onPreviousFileRead) + if (host === undefined || host === 'localhost') { + /* We read the file from the local file system. */ + const fs = require('fs') + fs.readFile(fileLocation + '.Previous.json', onPreviousFileRead) + } else { + /* We read the file via a web server over http */ + getFileViaHTTP(fileLocation + '.Previous.json', onPreviousFileRead) + } function onPreviousFileRead(err, text) { @@ -136,6 +149,7 @@ exports.newFileStorage = function newFileStorage(logger) { } else { logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Could read the file, but could not parse it as it is not a valid JSON.') logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> err = ' + err.stack) + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> text.toString() = ' + text.toString()) setTimeout(retry, retryTimeToUse) return } @@ -390,9 +404,8 @@ exports.newFileStorage = function newFileStorage(logger) { } } - /* Function to create folders of missing folders at any path. */ - - function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) { + /* Function to create folders of missing folders at any path. */ + function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) { const sep = '/'; const initDir = path.isAbsolute(targetDir) ? sep : ''; const baseDir = isRelativeToScript ? __dirname : '.'; @@ -420,5 +433,50 @@ exports.newFileStorage = function newFileStorage(logger) { return curDir; }, initDir); - } + } + + function getFileViaHTTP(filePath, callback) { + try { + + /* The filePath received is the one that is needed to fetch data from with fs. To do it via http we need to remove the prefix that includes this: ./Data-Storage/ */ + filePath = filePath.substring(15, filePath.length) + + let http = require('http'); + let url = 'http://' + host + + ':' + port + + '/Storage/' + + filePath + + http.get(url, onResponse); + + function onResponse(response) { + + const chunks = [] + + response.on('data', onMessegesArrived) + response.on('end', onEnd) + + function onMessegesArrived(chunk) { + chunks.push(chunk) + } + + function onEnd() { + let fileContent = Buffer.concat(chunks).toString('utf8') + let err = null + if (fileContent === 'The specified key does not exist.') { + err = { + code: "ENOENT" // This is how fs would have returned upon this situation. + } + fileContent = undefined + } + callback(err, fileContent) + } + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + abort = true + } + } } diff --git a/TaskServer/IndicatorBotProcessMainLoop.js b/TaskServer/IndicatorBotProcessMainLoop.js index bf61953d3..2c2401a15 100644 --- a/TaskServer/IndicatorBotProcessMainLoop.js +++ b/TaskServer/IndicatorBotProcessMainLoop.js @@ -252,6 +252,12 @@ switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well."); + + if (global.STOP_TASK_GRACEFULLY === true) { + loopControl() + return + } + initializeStatusDependencies(); return; } @@ -776,10 +782,13 @@ function onFinished(err) { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> onFinished -> Entering function."); } processFramework.finalize() processFramework = undefined + dataDependencies.finalize() + dataDependencies = undefined + statusDependencies.finalize() + statusDependencies = undefined switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { @@ -876,11 +885,12 @@ processExecutionEvents.finish(onFinished); function onFinished(err) { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Entering function."); } + processExecutionEvents.finalize() + processExecutionEvents = undefined + switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Execution finished well."); diff --git a/TaskServer/ProcessExecutionEvents.js b/TaskServer/ProcessExecutionEvents.js index 18ff8a70b..ba44bf7f2 100644 --- a/TaskServer/ProcessExecutionEvents.js +++ b/TaskServer/ProcessExecutionEvents.js @@ -8,27 +8,33 @@ */ const MODULE_NAME = "Process Execution Events"; + logger.fileName = MODULE_NAME let bot = BOT; let thisObject = { + networkNode: undefined, initialize, initialize, + finalize: finalize, start: start, finish: finish }; let processThisDependsOn; // This is the process this process needs to wait for it to finishes in order to start. - let currentProcessKey + let EVENT_SERVER_CLIENT = require('./EventServerClient.js'); + return thisObject; function initialize(callBackFunction) { try { + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } let name = 'Not Depends on any Process' + logger.fileName = MODULE_NAME + "." + name; if (bot.processNode.referenceParent !== undefined) { @@ -48,9 +54,63 @@ } } } + + /* We need to find at which network node is running the process that we need to hear from when it finished. */ + let network = global.TASK_NETWORK + + for (let i = 0; i < network.networkNodes.length; i++) { + let networkNode = network.networkNodes[i] + if (networkNode.dataMining !== undefined) { + for (let j = 0; j < networkNode.dataMining.exchangeTasks.length; j++) { + let exchangeTasks = networkNode.dataMining.exchangeTasks[j] + for (let k = 0; k < exchangeTasks.taskManagers.length; k++) { + let taskManager = exchangeTasks.taskManagers[k] + for (let m = 0; m < taskManager.tasks.length; m++) { + let task = taskManager.tasks[m] + if (task.bot !== undefined) { + for (let n = 0; n < task.bot.processes.length; n++) { + let process = task.bot.processes[n] + if (process.marketReference !== undefined) { + if (process.marketReference.referenceParent !== undefined) { + let market = process.marketReference.referenceParent + let currentProcessMarket = bot.processNode.marketReference.referenceParent + + if (currentProcessMarket.id === market.id) { + if (process.referenceParent !== undefined) { + let processDefinition = process.referenceParent + if (processThisDependsOn.id === processDefinition.id) { + + /* We found where the task that runs the process definition we are waiting for is located on the network. */ + + thisObject.networkNode = networkNode + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Connecting to Websockets Server " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webSocketsPort + '.'); } + + EVENT_SERVER_CLIENT = EVENT_SERVER_CLIENT.newEventsServerClient(networkNode.code.host, networkNode.code.webSocketsPort) + EVENT_SERVER_CLIENT.initialize(onConnected) + + function onConnected() { + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Connected to Websockets Server " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webSocketsPort + '.'); } + + callBackFunction(global.DEFAULT_OK_RESPONSE); + return + } + return + } + } + } + } + } + } + } + } + } + } + } + } + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the task that runs the Process Definition we are waiting for is located within the network. Check the logs for more info."); + return } - - logger.fileName = MODULE_NAME + "." + name; callBackFunction(global.DEFAULT_OK_RESPONSE); } catch (err) { @@ -59,6 +119,16 @@ } } + function finalize() { + EVENT_SERVER_CLIENT.finalize() + EVENT_SERVER_CLIENT = undefined + processThisDependsOn = undefined + currentProcessKey = undefined + thisObject.networkNode = undefined + bot = undefined + thisObject = undefined + } + function start(callBackFunction) { try { @@ -81,9 +151,9 @@ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> waitForDependantProcess -> key = " + key); } if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> waitForDependantProcess -> callerId = " + callerId); } - global.EVENT_SERVER_CLIENT.listenToEvent(key, 'Process Execution Finished', undefined, callerId, responseCallBack, eventsCallBack) + EVENT_SERVER_CLIENT.listenToEvent(key, 'Process Execution Finished', undefined, callerId, responseCallBack, eventsCallBack) - bot.processHeartBeat(undefined, undefined, "Waiting for " + processThisDependsOn.parentNode.parentNode.name + "->" + processThisDependsOn.parentNode.code.codeName + "->" + processThisDependsOn.code.codeName) + bot.processHeartBeat(undefined, undefined, "Waiting for " + thisObject.networkNode.name + "->" + processThisDependsOn.parentNode.parentNode.name + "->" + processThisDependsOn.parentNode.code.codeName + "->" + processThisDependsOn.code.codeName) function responseCallBack(message) { if (message.result !== global.DEFAULT_OK_RESPONSE.result) { @@ -102,7 +172,7 @@ /* We continue the normal flow after we learn the dependent process has finished its execution. */ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> eventsCallBack -> stopListening to Process Execution Finished. "); } if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> eventsCallBack -> subscriptionId = " + subscriptionId); } - global.EVENT_SERVER_CLIENT.stopListening(key, 'Process Execution Finished', subscriptionId) + EVENT_SERVER_CLIENT.stopListening(key, 'Process Execution Finished', subscriptionId) if (message.event.err.result === global.DEFAULT_OK_RESPONSE.result) { /* We emit our own event that the process started. */ diff --git a/TaskServer/SensorBotProcessMainLoop.js b/TaskServer/SensorBotProcessMainLoop.js index 5a5d55444..e90924148 100644 --- a/TaskServer/SensorBotProcessMainLoop.js +++ b/TaskServer/SensorBotProcessMainLoop.js @@ -254,6 +254,12 @@ switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well."); + + if (global.STOP_TASK_GRACEFULLY === true) { + loopControl() + return + } + initializeStatusDependencies(); return; } @@ -584,11 +590,12 @@ processExecutionEvents.finish(onFinished); function onFinished(err) { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Entering function."); } + processExecutionEvents.finalize() + processExecutionEvents = undefined + switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Execution finished well."); diff --git a/TaskServer/StatusDependencies.js b/TaskServer/StatusDependencies.js index 19e7485ca..48b5659a9 100644 --- a/TaskServer/StatusDependencies.js +++ b/TaskServer/StatusDependencies.js @@ -11,6 +11,7 @@ statusReports: new Map(), reportsByMainUtility: new Map(), initialize: initialize, + finalize: finalize, keys: [] }; @@ -165,4 +166,13 @@ } } + function finalize() { + thisObject.statusReports.forEach(forEachStatusDependency) + function forEachStatusDependency(statusDependency) { + statusDependency.finalize() + } + thisObject.statusReports = undefined + bot = undefined + thisObject = undefined + } }; diff --git a/TaskServer/StatusReport.js b/TaskServer/StatusReport.js index 66839dac9..4648c1a94 100644 --- a/TaskServer/StatusReport.js +++ b/TaskServer/StatusReport.js @@ -13,9 +13,11 @@ let bot = BOT; let thisObject = { + networkNode: undefined, mainUtility: undefined, file: undefined, // Here we have the JSON object representing the file content. initialize: initialize, + finalize: finalize, load: load, save: save, status: undefined @@ -30,7 +32,7 @@ /* Storage account to be used here. */ const FILE_STORAGE = require('./FileStorage.js'); - let fileStorage = FILE_STORAGE.newFileStorage(logger); + let fileStorage let sessionPath = '' @@ -93,17 +95,6 @@ statusDependencyNode.bottype = statusDependencyNode.referenceParent.parentNode.parentNode.type statusDependencyNode.dataMine = statusDependencyNode.referenceParent.parentNode.parentNode.parentNode.code.codeName - /* Let's see if the process is run monthly or not. */ - if (statusDependencyNode.referenceParent.parentNode.code.startMode !== undefined) { - if (statusDependencyNode.referenceParent.parentNode.code.startMode.allMonths !== undefined) { - if (statusDependencyNode.referenceParent.parentNode.code.startMode.oneMonth !== undefined) { - if (statusDependencyNode.referenceParent.parentNode.code.startMode.allMonths.run === "true" || statusDependencyNode.referenceParent.parentNode.code.startMode.oneMonth.run === "true") { - statusDependencyNode.processRunMonthly = true - } - } - } - } - /* We retrieve the report main utility */ if (statusDependencyNode.code !== undefined) { if (statusDependencyNode.code.mainUtility !== undefined) { @@ -122,7 +113,55 @@ sessionPath = bot.SESSION.folderName + "/" } - callBackFunction(global.DEFAULT_OK_RESPONSE); + /* Now we will see where do we need to fetch this status report from. */ + let network = global.TASK_NETWORK + let processThisDependsOn = statusDependencyNode.referenceParent.parentNode + + for (let i = 0; i < network.networkNodes.length; i++) { + let networkNode = network.networkNodes[i] + if (networkNode.dataMining !== undefined) { + for (let j = 0; j < networkNode.dataMining.exchangeTasks.length; j++) { + let exchangeTasks = networkNode.dataMining.exchangeTasks[j] + for (let k = 0; k < exchangeTasks.taskManagers.length; k++) { + let taskManager = exchangeTasks.taskManagers[k] + for (let m = 0; m < taskManager.tasks.length; m++) { + let task = taskManager.tasks[m] + if (task.bot !== undefined) { + for (let n = 0; n < task.bot.processes.length; n++) { + let process = task.bot.processes[n] + if (process.marketReference !== undefined) { + if (process.marketReference.referenceParent !== undefined) { + let market = process.marketReference.referenceParent + let currentProcessMarket = bot.processNode.marketReference.referenceParent + if (currentProcessMarket.id === market.id) { + if (process.referenceParent !== undefined) { + let processDefinition = process.referenceParent + if (processThisDependsOn.id === processDefinition.id) { + + /* We found where the task that runs the process definition this status report depends on and where it is located on the network. */ + + thisObject.networkNode = networkNode + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving status report from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); } + + fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return + } + } + } + } + } + } + } + } + } + } + } + } + + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this status report is located within the network. Check the logs for more info."); + } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack); @@ -130,6 +169,13 @@ } } + function finalize() { + fileStorage = undefined + thisObject.networkNode = undefined + bot = undefined + thisObject = undefined + } + function load(callBackFunction) { try { @@ -212,6 +258,7 @@ */ logger.write(MODULE_NAME, "[ERROR] load -> onFileReceived -> Error Parsing the Status report. -> Err = " + err.message); + logger.write(MODULE_NAME, "[WARN] load -> onFileReceived -> Error Parsing the Status report. -> text = " + text); let customFail = { result: global.CUSTOM_FAIL_RESPONSE.result, diff --git a/TaskServer/TradingBotProcessMainLoop.js b/TaskServer/TradingBotProcessMainLoop.js index 461b648a8..ab3bee5ad 100644 --- a/TaskServer/TradingBotProcessMainLoop.js +++ b/TaskServer/TradingBotProcessMainLoop.js @@ -288,6 +288,12 @@ switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well."); + + if (global.STOP_TASK_GRACEFULLY === true) { + loopControl() + return + } + initializeStatusDependencies(); return; } @@ -694,6 +700,10 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> onFinished -> Entering function."); } processFramework.finalize() processFramework = undefined + dataDependencies.finalize() + dataDependencies = undefined + statusDependencies.finalize() + statusDependencies = undefined switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { @@ -792,8 +802,9 @@ processExecutionEvents.finish(onFinished); function onFinished(err) { - try { + processExecutionEvents.finalize() + processExecutionEvents = undefined if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Entering function."); } diff --git a/TaskServer/server.js b/TaskServer/server.js index 631ed3edb..f32a3bc13 100644 --- a/TaskServer/server.js +++ b/TaskServer/server.js @@ -125,12 +125,13 @@ function preLoader() { global.EVENT_SERVER_CLIENT.raiseEvent('Task Manager - ' + taskId, 'Nodejs Process Ready for Task') function eventReceived(message) { global.TASK_NODE = message - global.TASK_NODE = JSON.parse(message.event.definition) + global.TASK_NODE = JSON.parse(message.event.taskDefinition) + global.TASK_NETWORK = JSON.parse(message.event.networkDefinition) bootLoader() } } catch (err) { console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE -> ' + err.stack) - console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE)) + console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE).substring(0, 1000)) } } else { /* This process was started not by the Task Manager, but independently (most likely for debugging purposes). In this case we listen to an event with the Task Info that should be emitted at the UI */ @@ -139,12 +140,13 @@ function preLoader() { global.EVENT_SERVER_CLIENT.listenToEvent('Task Server', 'Debug Task Started', undefined, 'Task Server', undefined, startDebugging) function startDebugging(message) { global.TASK_NODE = message - global.TASK_NODE = JSON.parse(message.event.definition) + global.TASK_NODE = JSON.parse(message.event.taskDefinition) + global.TASK_NETWORK = JSON.parse(message.event.networkDefinition) bootLoader() } } catch (err) { console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE -> ' + err.stack) - console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE)) + console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE).substring(0, 1000)) } } }