Data-Storage Structure Multi Project

TaskServer/.env

@@ -8,6 +8,6 @@ WRITE_LOGS_TO_FILES=true
 BOTS_PATH=../Legacy-Code
 LOG_PATH=../Log-Files
 
-WEB_SOCKETS_SERVER_PORT=18041
+WEB_SOCKETS_INTERFACE_PORT=18041
 WEB_SERVER_PORT=34248
 WEB_SERVER_URL=localhost

TaskServer/Commons.js

@@ -530,7 +530,7 @@
                 dateForPath = "/" + currentDay.getUTCFullYear() + '/' + utilities.pad(currentDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(currentDay.getUTCDate(), 2);
             }
 
-            let filePathRoot = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + contextSummary.dataMine + "/" + contextSummary.bot;
+            let filePathRoot = 'Project/' + contextSummary.project + "/" + contextSummary.mineType + "/" + contextSummary.dataMine + "/" + contextSummary.bot + '/' + bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset
             let filePath = filePathRoot + "/Output/" + contextSummary.product + "/" + contextSummary.dataset + "/" + timeFrameLabel + dateForPath;
             filePath += '/' + fileName
 

TaskServer/DataDependencies.js

@@ -1,7 +1,5 @@
 exports.newDataDependencies = function newDataDependencies(BOT, logger, DATA_SET) {
 
-    const FULL_LOG = true;
-    const LOG_FILE_CONTENT = false;
     const MODULE_NAME = "Data Dependencies";
 
     let bot = BOT

TaskServer/DataSet.js

@@ -117,10 +117,12 @@ exports.newDataSet = function newDataSet(BOT, logger) {
 
             if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getTextFile -> Entering function."); }
 
-            let dataMineCodeName = thisObject.node.parentNode.parentNode.parentNode.config.codeName
+            let mine = thisObject.node.parentNode.parentNode.parentNode.config.codeName
+            let mineType = thisObject.node.parentNode.parentNode.parentNode.type.replace(' ', '-')
+            let project = thisObject.node.parentNode.parentNode.parentNode.project
             let botCodeName = thisObject.node.parentNode.parentNode.config.codeName
 
-            let filePathRoot = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + dataMineCodeName + "/" + botCodeName;
+            let filePathRoot = 'Project/' + project + "/" + mineType + "/" + mine + "/" + botCodeName + '/' + bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset
             let filePath = filePathRoot + "/Output/" + pFolderPath;
             filePath += '/' + pFileName
 
@@ -162,7 +164,7 @@ exports.newDataSet = function newDataSet(BOT, logger) {
                 return;
             }
 
-            let filePathRoot = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + thisObject.node.dataMine + "/" + thisObject.node.bot;
+            let filePathRoot = 'Project/' + thisObject.node.project + "/" + thisObject.node.mineType + "/" + thisObject.node.dataMine + "/" + thisObject.node.bot + '/' + bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset
             let filePath = filePathRoot + "/Output/" + pFolderPath + '/' + pFileName;
 
             fileStorage.createTextFile(filePath, pFileContent, onFileCreated);

TaskServer/FileStorage.js

@@ -197,10 +197,9 @@ exports.newFileStorage = function newFileStorage(logger, host, port) {
             }
 
             function retry() {
-                console.log(new Date(), '[WARN] Read File Retry #: ' + currentRetryGetTextFile, fileLocation)
                 if (currentRetryGetTextFile < MAX_RETRY) {
                     currentRetryGetTextFile++
-                    logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> retry -> Will try to read the file again -> Retry #: ' + currentRetryGetTextFile)
+                    logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> retry -> Will try to read the file again -> Retry #: ' + currentRetryGetTextFile)
                     recursiveGetTextFile(filePath, callBackFunction, noRetry, canUsePrevious)
                 } else {
                     currentRetryGetTextFile = 0
@@ -368,7 +367,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) {
             function retry() {
                 if (currentRetryWriteTextFile < MAX_RETRY) {
                     currentRetryWriteTextFile++
-                    logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> retry -> Will try to write the file again -> Retry #: ' + currentRetryWriteTextFile)
+                    logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> retry -> Will try to write the file again -> Retry #: ' + currentRetryWriteTextFile)
                     recursiveCreateTextFile(filePath, fileContent, callBackFunction, keepPrevious)
                 } else {
                     currentRetryWriteTextFile = 0
@@ -428,7 +427,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) {
             function retry() {
                 if (currentRetryDeleteTextFile < MAX_RETRY) {
                     currentRetryDeleteTextFile++
-                    logger.write(MODULE_NAME, '[WARN] FileStorage -> deleteTextFile -> retry -> Will try to delete the file again -> Retry #: ' + currentRetryDeleteTextFile)
+                    logger.write(MODULE_NAME, '[INFO] FileStorage -> deleteTextFile -> retry -> Will try to delete the file again -> Retry #: ' + currentRetryDeleteTextFile)
                     recursiveDeleteTextFile(filePath, callBackFunction)
                 } else {
                     currentRetryDeleteTextFile = 0

TaskServer/IndicatorOutput.js

@@ -134,6 +134,8 @@
                 contextSummary.bot = botNode.config.codeName
                 let dataMineNode = global.FIND_NODE_IN_NODE_MESH(outputDatasetNode, 'Data Mine')
                 contextSummary.dataMine = dataMineNode.config.codeName
+                contextSummary.mineType = dataMineNode.type.replace(' ', '-')
+                contextSummary.project = dataMineNode.project
 
                 let fileContent = commons.generateFileContent(outputData, outputDatasetNode.referenceParent.parentNode.record, resultsWithIrregularPeriods, processingDailyFiles, currentDay, callBackFunction)
                 commons.writeFile(contextSummary, fileContent, anotherFileWritten, processingDailyFiles, timeFrameLabel, currentDay, callBackFunction)

TaskServer/LowFrequencyTrading/TradingProcess.js

@@ -81,7 +81,7 @@
                 dateEndOfMarket: undefined          // Datetime of the last file available to be used as an input of this process.
             };
 
-            getContextVariables()
+            if (getContextVariables() !== true) { return }
 
             if (bot.FIRST_EXECUTION === true && bot.RESUME === false) {
                 /* 
@@ -245,13 +245,13 @@
                     if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
                         logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
                         callBackFunction(global.DEFAULT_RETRY_RESPONSE);
-                        return
+                        return false
                     }
 
                     if (statusReport.status === "Status Report is corrupt.") {
                         logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
                         callBackFunction(global.DEFAULT_RETRY_RESPONSE);
-                        return
+                        return false
                     }
 
                     thisReport = statusReport.file;
@@ -265,7 +265,7 @@
                         }
                         logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
                         callBackFunction(customOK);
-                        return
+                        return false
                     }
 
                     contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf());
@@ -276,13 +276,13 @@
                     if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
                         logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
                         callBackFunction(global.DEFAULT_RETRY_RESPONSE);
-                        return
+                        return false
                     }
 
                     if (statusReport.status === "Status Report is corrupt.") {
                         logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process.");
                         callBackFunction(global.DEFAULT_FAIL_RESPONSE);
-                        return
+                        return false
                     }
 
                     thisReport = statusReport.file;
@@ -293,10 +293,10 @@
                         if (bot.RESUME !== true) {
                             if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Starting from the begining because bot has just started and resume execution was true."); }
                             startFromBegining();
-                            return
+                            return true
                         }
                         contextVariables.lastFile = new Date(thisReport.lastFile);
-                        return
+                        return true
 
                     } else {
                         /*
@@ -306,7 +306,7 @@
                         */
                         if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Starting from the begining of the market because own status report not found or lastFile was undefined."); }
                         startFromBegining();
-                        return
+                        return true
                     }
 
                     function startFromBegining() {

TaskServer/Root.js

@@ -183,6 +183,8 @@
 
                     /* Simplifying the access to basic info */
                     botConfig.dataMine = global.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName
+                    botConfig.mineType = global.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.type.replace(' ', '-')
+                    botConfig.project = global.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.project
                     botConfig.exchange = global.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.name
                     botConfig.exchangeNode = global.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode
                     botConfig.market = {
@@ -196,7 +198,7 @@
                     botConfig.loopCounter = 0;
 
                     /* File Path Root */
-                    botConfig.filePathRoot = botConfig.exchange + "/" + botConfig.market.baseAsset + "-" + botConfig.market.quotedAsset + "/" + botConfig.dataMine + "/" + botConfig.codeName;
+                    botConfig.filePathRoot = 'Project/' + botConfig.project + "/" + botConfig.mineType + "/" + botConfig.dataMine + "/" + botConfig.codeName + '/' + botConfig.exchange + "/" + botConfig.market.baseAsset + "-" + botConfig.market.quotedAsset
 
                     /* Process Key */
                     botConfig.processKey = global.TASK_NODE.bot.processes[processIndex].name + '-' + global.TASK_NODE.bot.processes[processIndex].type + '-' + global.TASK_NODE.bot.processes[processIndex].id

TaskServer/StatusReport.js

@@ -92,6 +92,8 @@
             statusDependencyNode.process = statusDependencyNode.referenceParent.parentNode.config.codeName
             statusDependencyNode.bottype = statusDependencyNode.referenceParent.parentNode.parentNode.type
             statusDependencyNode.dataMine = statusDependencyNode.referenceParent.parentNode.parentNode.parentNode.config.codeName
+            statusDependencyNode.mineType = statusDependencyNode.referenceParent.parentNode.parentNode.parentNode.type.replace(' ', '-')
+            statusDependencyNode.project = statusDependencyNode.referenceParent.parentNode.parentNode.parentNode.project
 
             /* We retrieve the report main utility */
             if (statusDependencyNode.config !== undefined) {
@@ -251,9 +253,8 @@
 
             if (ownerId !== botId) {
 
-                let rootPath = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + statusDependencyNode.dataMine + "/" + statusDependencyNode.bot
-
-                filePath = rootPath + "/Reports/" + sessionPath + statusDependencyNode.process;
+                let filePathRoot = 'Project/' + statusDependencyNode.project + "/" + statusDependencyNode.mineType + "/" + statusDependencyNode.dataMine + "/" + statusDependencyNode.bot + '/' + bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset
+                filePath = filePathRoot + "/Reports/" + sessionPath + statusDependencyNode.process;
             } else {
                 filePath = bot.filePathRoot + "/Reports/" + sessionPath + statusDependencyNode.process;
             }