From 7e51bf8e9b65c7e55368f5eb9b1a573f21f86a74 Mon Sep 17 00:00:00 2001 From: Matias Date: Tue, 4 Jun 2019 20:26:54 -0300 Subject: [PATCH] ALGO node progress --- .gitignore | 2 +- Assistant.js | 16 +- AzureBlobStorage.js | 329 ------------ AzureStorage.js | 234 --------- BlobStorage.js | 37 -- CloudRequire.js | 116 ---- CloudUtilities.js | 7 +- Context.js | 67 +-- DataDependencies.js | 9 +- DataSet.js | 48 +- DebugLog.js | 111 ++-- IndicatorBotProcessMainLoop.js | 168 ++---- Integrations/FileStorage.js | 105 ++++ Integrations/OperationsModuleIntegration.js | 55 -- Integrations/Strategy.js | 47 ++ MinioBlobStorage.js | 401 -------------- MultiPeriodDaily.js | 35 +- MultiPeriodMarket.js | 18 +- Root.js | 117 +---- Run.js | 75 +-- SensorBotProcessMainLoop.js | 159 ++---- StatusDependencies.js | 11 +- StatusReport.js | 86 ++- StorageAccessManager.js | 62 --- TradingBotProcessMainLoop.js | 272 +++------- docker-compose.prod.yml | 2 +- package-lock.json | 555 +++++--------------- package.json | 12 +- this.config.json | 20 - utils/auth.js | 35 -- web.config.json | 43 -- 31 files changed, 645 insertions(+), 2609 deletions(-) delete mode 100644 AzureBlobStorage.js delete mode 100644 AzureStorage.js delete mode 100644 BlobStorage.js delete mode 100644 CloudRequire.js create mode 100644 Integrations/FileStorage.js delete mode 100644 Integrations/OperationsModuleIntegration.js create mode 100644 Integrations/Strategy.js delete mode 100644 MinioBlobStorage.js delete mode 100644 StorageAccessManager.js delete mode 100644 this.config.json delete mode 100644 utils/auth.js delete mode 100644 web.config.json diff --git a/.gitignore b/.gitignore index c6e23b7ff..4d0f3bae0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ bin/* *.cache node_modules/* exchange/Auth0.js -.env +.env* \.vscode/ Project.njsproj diff --git a/Assistant.js b/Assistant.js index 7a482e279..65f40bed9 100644 --- a/Assistant.js +++ b/Assistant.js @@ -32,7 +32,7 @@ addExtraData: addExtraData }; - let utilities = UTILITIES.newCloudUtilities(bot, logger); + let utilities = UTILITIES.newCloudUtilities(logger); let exchangePositions = []; // These are the open positions at the exchange at the account the bot is authorized to use. let openPositions = []; // These are the open positions the bot knows it made by itself. @@ -134,11 +134,11 @@ /* Procedure to get the current market rate. */ let key = - global.PLATFORM_CONFIG.marketRateProvider.devTeam + "-" + - global.PLATFORM_CONFIG.marketRateProvider.bot + "-" + - global.PLATFORM_CONFIG.marketRateProvider.product + "-" + - global.PLATFORM_CONFIG.marketRateProvider.dataSet + "-" + - global.PLATFORM_CONFIG.marketRateProvider.dataSetVersion; + bot.marketRateProvider.devTeam + "-" + + bot.marketRateProvider.bot + "-" + + bot.marketRateProvider.product + "-" + + bot.marketRateProvider.dataSet + "-" + + bot.marketRateProvider.dataSetVersion; if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> getMarketRateFromIndicator -> key = " + key); } @@ -146,9 +146,9 @@ let dateForPath = bot.processDatetime.getUTCFullYear() + '/' + utilities.pad(bot.processDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.processDatetime.getUTCDate(), 2); let fileName = '' + global.MARKET.assetA + '_' + global.MARKET.assetB + '.json'; - let filePath = global.PLATFORM_CONFIG.marketRateProvider.product + "/" + global.PLATFORM_CONFIG.marketRateProvider.dataSet + "/" + dateForPath; + let filePath = bot.marketRateProvider.product + "/" + bot.marketRateProvider.dataSet + "/" + dateForPath; - dataSet.getTextFile(filePath, fileName, onFileReceived, true); + dataSet.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { diff --git a/AzureBlobStorage.js b/AzureBlobStorage.js deleted file mode 100644 index 5f4425a02..000000000 --- a/AzureBlobStorage.js +++ /dev/null @@ -1,329 +0,0 @@ - -exports.newAzureBlobBlobStorage = function newAzureBlobBlobStorage(BOT, logger) { - - let FULL_LOG = true; - let ERROR_LOG = true; - let LOG_FILE_CONTENT = true; - - let bot = BOT; - const ROOT_DIR = './'; - - const MODULE_NAME = "Azure Blob Storage"; - const RETRY_WAIT_TIME = 10000; - const MAX_RETRIES = 5; - - let AZURE_STORAGE = require('./AzureStorage') - let storage = AZURE_STORAGE.newAzureStorage(bot, logger); - - let thisObject = { - initialize: initialize, - createFolder: createFolder, - createTextFile: createTextFile, - getTextFile: getTextFile - }; - - let readOnlyBlobService; - let writeOnlyBlobService; - let containerName; - let environment = global.CURRENT_ENVIRONMENT; - - return thisObject; - - function initialize(pDataOwner, callBackFunction, disableLogging) { - - try { - - containerName = pDataOwner.toLowerCase(); - - if (pDataOwner.environment !== undefined) { // This is use for data migration from one environment to the other. - - environment = pDataOwner.environment; - - } - - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> environment = " + environment); } - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> containerName = " + containerName); } - - - - let readConnectionString = global.USER_PROFILE.storagePermissions.get(pDataOwner + "." + "READ"); - - if (readConnectionString !== undefined) { - readOnlyBlobService = storage.createBlobService(readConnectionString); - } - - let writeConnectionString = global.USER_PROFILE.storagePermissions.get(pDataOwner + "." + "WRITE"); - - if (writeConnectionString !== undefined) { - writeOnlyBlobService = storage.createBlobService(writeConnectionString); - } - - if (readOnlyBlobService !== undefined || writeOnlyBlobService !== undefined) { - - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - - } else { - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> No READ or WRITE permissions found."); } - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> pDataOwner = " + pDataOwner); } - - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - - } - } - catch (err) { - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function createFolder(pFolderPath, callBackFunction) { - - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] createFolder -> Entering function."); } - - try { - - /* Folders do not need to be created when using blobs. */ - - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createFolder -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function createTextFile(pFolderPath, pFileName, pFileContent, callBackFunction) { - - let retryCount = 0; - if (writeOnlyBlobService === undefined) { - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> initialize function not executed or failed. Can not process this request. Sorry."); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - try { - - if (FULL_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] createTextFile -> About to create a text file."); - logger.write(MODULE_NAME, "[INFO] createTextFile -> containerName = " + containerName); - logger.write(MODULE_NAME, "[INFO] createTextFile -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[INFO] createTextFile -> pFileName = " + pFileName); - } - - writeOnlyBlobService.createBlockBlobFromText(containerName, pFolderPath + "/" + pFileName, pFileContent, onFileCreated); - - function onFileCreated(err, result, response) { - - try { - - if (FULL_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> Response from Azure received."); - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> err = " + JSON.stringify(err)); - } - - if (LOG_FILE_CONTENT === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> result = " + JSON.stringify(result)); - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileReceived -> response = " + JSON.stringify(response)); - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileReceived -> pFileContent = " + pFileContent); - } - - if (err) { - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> err = " + JSON.stringify(err)); } - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> result = " + JSON.stringify(result)); } - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> response = " + JSON.stringify(response)); } - - if (ERROR_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> Error trying to create this file."); - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> containerName = " + containerName); - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> pFileName = " + pFileName); - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> err.code = " + err.code); - } - - if ( - err.code === 'ECONNRESET' || - err.code === 'ENOTFOUND' || - err.code === 'ESOCKETTIMEDOUT' || - err.code === 'ETIMEDOUT' || - err.code === 'ECONNREFUSED' || - err.code === 'EADDRINUSE' || - err.code === 'EAI_AGAIN' || - err.code === 'AuthenticationFailed' || - err.code === 'OperationTimedOut' || - err.code === 'ServerBusy' - ) { - - setTimeout(retryCreateFile, RETRY_WAIT_TIME); - return; - - function retryCreateFile() { - if(retryCount < MAX_RETRIES){ - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> Retrying to create the file. Retry count: " + retryCount); - - writeOnlyBlobService.createBlockBlobFromText(containerName, pFolderPath + "/" + pFileName, pFileContent, onFileCreated); - } - return; - } - } - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> Dont know what to do here. Cancelling operation. "); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - - } else { - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> File Created."); } - callBackFunction(global.DEFAULT_OK_RESPONSE); - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function getTextFile(pFolderPath, pFileName, callBackFunction) { - - let retryCount = 0; - if (readOnlyBlobService === undefined) { - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> initialize function not executed or failed. Can not process this request. Sorry."); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - try { - - if (FULL_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] getTextFile -> About to get a text file."); - logger.write(MODULE_NAME, "[INFO] getTextFile -> containerName = " + containerName); - logger.write(MODULE_NAME, "[INFO] getTextFile -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[INFO] getTextFile -> pFileName = " + pFileName); - } - - readOnlyBlobService.getBlobToText(containerName, pFolderPath + "/" + pFileName, onFileReceived); - - function onFileReceived(err, text, response) { - - try { - - if (FULL_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> Response from Azure received."); - logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> err = " + JSON.stringify(err)); - } - - if (LOG_FILE_CONTENT === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> response = " + JSON.stringify(response)); - logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> text = " + text); - } - - if (err) { - - if (ERROR_LOG === true && logger !== undefined && err.code !== 'BlobNotFound') { - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> Error trying to get this file."); - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> containerName = " + containerName); - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> pFileName = " + pFileName); - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> err.code = " + err.code); - } - - if ( - err.code === 'ECONNRESET' || - err.code === 'ENOTFOUND' || - err.code === 'ESOCKETTIMEDOUT' || - err.code === 'ETIMEDOUT' || - err.code === 'ECONNREFUSED' || - err.code === 'EADDRINUSE' || - err.code === 'EAI_AGAIN' || - err.code === 'AuthenticationFailed' || - err.code === 'OperationTimedOut' || - err.code === 'ServerBusy' - ) { - - setTimeout(retryCreateFile, RETRY_WAIT_TIME); - return; - - function retryCreateFile() { - if(retryCount < MAX_RETRIES){ - logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> Retrying to get the file. Retry count: " + retryCount); - readOnlyBlobService.getBlobToText(containerName, pFolderPath + "/" + pFileName, onFileReceived); - } - return; - } - } - - if (err.code === "BlobNotFound") { - - /* This is how Azure tell us the file does not exist. */ - - let customErr = { - result: global.CUSTOM_FAIL_RESPONSE.result, - message: "File does not exist." - }; - - logger.write(MODULE_NAME, "[WARN] getTextFile -> onFileReceived -> Custom Response -> message = " + customErr.message); - logger.write(MODULE_NAME, "[WARN] getTextFile -> containerName = " + containerName); - logger.write(MODULE_NAME, "[WARN] getTextFile -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[WARN] getTextFile -> pFileName = " + pFileName); - - callBackFunction(customErr); - return; - - } - - if (err.code === "ParentNotFound") { - - /* This is how Azure tell us the folder where the file was supposed to be does not exist. We map this to our own response. */ - - let customErr = { - result: global.CUSTOM_FAIL_RESPONSE.result, - message: "Folder does not exist." - }; - - logger.write(MODULE_NAME, "[WARN] getTextFile -> onFileReceived -> Custom Response -> message = " + customErr.message); - logger.write(MODULE_NAME, "[WARN] getTextFile -> containerName = " + containerName); - logger.write(MODULE_NAME, "[WARN] getTextFile -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[WARN] getTextFile -> pFileName = " + pFileName); - - callBackFunction(customErr); - return; - - } - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileCreated -> err = " + JSON.stringify(err)); } - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileCreated -> response = " + JSON.stringify(response)); } - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> Dont know what to do here. Cancelling operation. "); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - - } else { - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> File retrieved."); } - callBackFunction(global.DEFAULT_OK_RESPONSE, text); - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - -}; diff --git a/AzureStorage.js b/AzureStorage.js deleted file mode 100644 index 8e35fce13..000000000 --- a/AzureStorage.js +++ /dev/null @@ -1,234 +0,0 @@ -exports.newAzureStorage = function newAzureStorage(BOT, logger) { - - let FULL_LOG = true; - let LOG_FILE_CONTENT = false; - - let bot = BOT; - const ROOT_DIR = './'; - - const MODULE_NAME = "Azure Blob Storage"; - const REQUEST_TIMEOUT = 180; - - let thisObject = { - createBlobService: createBlobService - }; - - return thisObject; - - function createBlobService(pConnectionString) { - - return newBlobService(pConnectionString); - - } - - function newBlobService(pConnectionString) { - - let endPoint; - let SAS; - let https = require('https'); - - let thisObject = { - getBlobToText: getBlobToText, - createBlockBlobFromText: createBlockBlobFromText - }; - - let properties = pConnectionString.split(";"); - endPoint = properties[0].substring("BlobEndpoint=https://".length); - SAS = properties[1].substring("SharedAccessSignature=".length); - - return thisObject; - - function getBlobToText(pContainerName, pBlobPath, callBackFunction) { - - getBlob(endPoint, pContainerName, pBlobPath, SAS, onContentArrived); - - function onContentArrived(err, pContent) { - - let response = null; - - if (err === null) { - err = checkForStorageErrors(pContent) - } - - callBackFunction(err, pContent, response); - - } - } - - function createBlockBlobFromText(pContainerName, pBlobPath, pBlobContent, callBackFunction) { - - putBlob(endPoint, pContainerName, pBlobPath, pBlobContent, SAS, onFinished); - - function onFinished(err, pContent) { - - let result = null; - let response = null; - - if (err === null) { - err = checkForStorageErrors(pContent) - } - - callBackFunction(err, result, response); - - } - } - - function checkForStorageErrors(pData) { - - let err = null; - - if (pData !== undefined) { - - if (pData.indexOf("BlobNotFound") > 0) { - - err = { - code: 'BlobNotFound' - }; - } - - if (pData.indexOf("ParentNotFound") > 0) { - - err = { - code: 'ParentNotFound' - }; - } - - if (pData.indexOf("AuthenticationFailed") > 0) { - - err = { - code: 'AuthenticationFailed' - }; - } - - if (pData.indexOf("ServerBusy") > 0) { - - err = { - code: 'ServerBusy' - }; - } - - if (pData.indexOf("AuthorizationPermissionMismatch") > 0) { - - err = { - code: 'AuthorizationPermissionMismatch' - }; - } - } - - return err; - } - - function getBlob(pEndPoint, pContainer, pBlobPath, pSAS, callBack) { - - let options = { - host: pEndPoint, - port: 443, - path: '/' + pContainer + '/' + pBlobPath + '?' + pSAS + '&timeout=' + REQUEST_TIMEOUT, - method: 'GET', - headers: {}, - agent: false - } - - let responseData = ''; - let err = null; - - /* We add the headers. */ - - options.headers['x-ms-version'] = '2017-07-29'; - //options.headers['Connection'] = 'Keep-Alive'; - - /* Making the https get call. */ - - let request = https.request(options, onResponse); - request.end(); //This actually sends the request. - request.on('error', onError); - - function onError(err) { - - callBack(err); - - } - - function onResponse(response) { - response.on('data', onPieceOfDataArrived); - response.on('end', onAllDataArrived); - } - - function onPieceOfDataArrived(pData) { - - responseData = responseData + pData.toString(); - } - - function onAllDataArrived() { - - callBack(err, responseData) - - } - } - - function putBlob(pEndPoint, pContainer, pBlobPath, pBlobContent, pSAS, callBack) { - - let options = { - host: pEndPoint, - port: 443, - path: '/' + pContainer + '/' + pBlobPath + '?' + pSAS + '&timeout=' + REQUEST_TIMEOUT, - method: 'PUT', - headers: {}, - agent: false - } - - let responseData = ''; - let err = null; - let responseCode; - - /* We add the headers. */ - - options.headers['x-ms-version'] = '2017-07-29'; - options.headers['Content-Length'] = pBlobContent.length; - options.headers['x-ms-blob-type'] = 'BlockBlob'; - //options.headers['Connection'] = 'Keep-Alive'; - - /* Making the https get call. */ - - let request = https.request(options, onResponse); - request.body = pBlobContent; - request.write(pBlobContent); - request.end(); //This actually sends the request. - request.on('error', onError); - - function onError(err) { - - callBack(err); - - } - - function onResponse(response) { - - responseCode = response.statusCode; - - if (responseCode === 201) { - - callBack(err); // No problems. - - } - - response.on('data', onPieceOfDataArrived); - response.on('end', onAllDataArrived); - } - - function onPieceOfDataArrived(pData) { - responseData = responseData + pData.toString(); - } - - function onAllDataArrived() { - - if (responseCode !== 201) { - - callBack(err, responseData);; // Some problem found. - - } - } - } - } -} - diff --git a/BlobStorage.js b/BlobStorage.js deleted file mode 100644 index de0eb1e03..000000000 --- a/BlobStorage.js +++ /dev/null @@ -1,37 +0,0 @@ - -exports.newBlobStorage = function newBlobStorage(BOT, logger) { - - let FULL_LOG = false; - let ERROR_LOG = true; - let LOG_FILE_CONTENT = false; - - let bot = BOT; - const ROOT_DIR = './'; - - const MODULE_NAME = "Blob Storage"; - - let storage; - - let AZURE_STORAGE = require('./AzureBlobStorage') - let MINIO_STORAGE = require('./MinioBlobStorage') - - switch (process.env.STORAGE_PROVIDER) { - case 'Azure': { - storage = AZURE_STORAGE.newAzureBlobBlobStorage(bot, logger); - break; - } - case 'Minio': { - storage = MINIO_STORAGE.newMinioBlobBlobStorage(bot, logger); - break; - } - default: { - if (ERROR_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[ERROR] newBlobStorage -> Storage Provider not supported -> process.env.STORAGE_PROVIDER = " + process.env.STORAGE_PROVIDER); - } - return; - } - } - - return storage; - -}; \ No newline at end of file diff --git a/CloudRequire.js b/CloudRequire.js deleted file mode 100644 index 7bc874962..000000000 --- a/CloudRequire.js +++ /dev/null @@ -1,116 +0,0 @@ -exports.newCloudRequire = function newCloudRequire(BOT, logger) { - - const FULL_LOG = true; - const LOG_FILE_CONTENT = false; - - const MODULE_NAME = "Cloud Require"; - - let bot = BOT; - - let thisObject = { - downloadBot: downloadBot, - downloadCommons: downloadCommons - }; - - return thisObject; - - function downloadBot(pCloudStorage, pFilePath, callBackFunction) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] downloadBot -> Entering function."); } - - let fileName = "User.Bot.js"; - - pCloudStorage.getTextFile(pFilePath, fileName, onFileReceived); - - function onFileReceived(err, text) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] downloadBot -> onFileReceived -> err.message = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - try { - - let USER_BOT_MODULE = {}; - USER_BOT_MODULE.newUserBot = eval(text); - - /* - This following code is to load the bot code from a file on the file system. - It is useful for when you are activelly changing the code and dont wont to upload it to the storage at every run - */ - - /* - let fs = require('fs'); - try { - let fileName = "C:/Users/Luis/source/repos/Bots/AAMasters/AAChris-Indicator-Bot/Multi-Period-Daily/User.Bot.js"; - fs.readFile(fileName, onFileRead); - - function onFileRead(err, file) { - let fileString = file.toString(); - USER_BOT_MODULE.newUserBot = eval(fileString); - callBackFunction(global.DEFAULT_OK_RESPONSE, USER_BOT_MODULE); - } - } - catch(err) { - console.log("ERROR LOADING FILE FROM FILE SYSTEM"); - } - */ - callBackFunction(global.DEFAULT_OK_RESPONSE, USER_BOT_MODULE); // Coment this when loading from file system. - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] downloadBot -> onFileReceived -> err.message = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] downloadBot -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function downloadCommons(pCloudStorage, pFilePath, callBackFunction) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] downloadCommons -> Entering function."); } - - let fileName = "Commons.js"; - - pCloudStorage.getTextFile(pFilePath, fileName, onFileReceived); - - function onFileReceived(err, text) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - // Nothing happens since COMMONS modules are optional. - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } - - try { - - let COMMONS_MODULE = {}; - COMMONS_MODULE.newCommons = eval(text); - - callBackFunction(global.DEFAULT_OK_RESPONSE, COMMONS_MODULE); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] downloadCommons -> onFileReceived -> err.message = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] downloadCommons -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - -}; \ No newline at end of file diff --git a/CloudUtilities.js b/CloudUtilities.js index 1418020d7..18bbcf05f 100644 --- a/CloudUtilities.js +++ b/CloudUtilities.js @@ -1,11 +1,8 @@  -exports.newCloudUtilities = function newCloudUtilities(BOT, logger) { +exports.newCloudUtilities = function newCloudUtilities(logger) { const FULL_LOG = true; - let bot = BOT; - const ROOT_DIR = '../'; - const MODULE_NAME = "Cloud Utilities"; let utilities = { @@ -85,4 +82,4 @@ exports.newCloudUtilities = function newCloudUtilities(BOT, logger) { str = str.toString(); return str.length < max ? pad("0" + str, max) : str; } -} \ No newline at end of file +} diff --git a/Context.js b/Context.js index b10e96636..a7d4987d6 100644 --- a/Context.js +++ b/Context.js @@ -1,4 +1,4 @@ -exports.newContext = function newContext(BOT, logger, BLOB_STORAGE, UTILITIES, STATUS_REPORT) { +exports.newContext = function newContext(BOT, logger, UTILITIES) { let bot = BOT; @@ -61,18 +61,17 @@ /* Utilities needed. */ - let utilities = UTILITIES.newCloudUtilities(bot, logger); + let utilities = UTILITIES.newCloudUtilities(logger); /* Storage account to be used here. */ - let cloudStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + const FILE_STORAGE = require('./Integrations/FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(); let statusDependencies; let runIndex; // This is the index for this run and depends on the startMode. - let totalAlgobots; - return thisObject; function initialize(pStatusDependencies, pTotalAlgobots, callBackFunction) { @@ -82,7 +81,6 @@ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } statusDependencies = pStatusDependencies; - totalAlgobots = pTotalAlgobots; /* @@ -90,26 +88,9 @@ of the process. */ - initializeStorage(); + thisObject.newHistoryRecord.date = bot.processDatetime; - function initializeStorage() { - - cloudStorage.initialize(bot.devTeam, onInizialized); - - function onInizialized(err) { - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - thisObject.newHistoryRecord.date = bot.processDatetime; - - getStatusReport(onDone); - - } else { - logger.write(MODULE_NAME, "[ERROR] initialize -> initializeStorage -> onInizialized -> err = " + err.message); - callBackFunction(err); - } - } - } + getStatusReport(onDone); function onDone(err) { try { @@ -180,13 +161,10 @@ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> getExecutionHistory -> Entering function."); } - let fileName = "Execution.History." + bot.startMode + "." + runIndex + ".json"; - let filePath = bot.filePathRoot + "/Output/" + bot.process; + let fileName = "/Execution.History." + bot.startMode + "." + runIndex + ".json"; + let filePath = bot.filePathRoot + "/Output/" + bot.process + fileName; - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> getExecutionHistory -> fileName = " + fileName); } - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> getExecutionHistory -> filePath = " + filePath); } - - cloudStorage.getTextFile(filePath, fileName, onFileReceived); + fileStorage.getTextFile(global.DEV_TEAM, filePath, onFileReceived); function onFileReceived(err, text) { @@ -243,14 +221,11 @@ thisObject.statusReport.runs[runIndex].lastExecution = bot.processDatetime; thisObject.statusReport.runs[runIndex].endDatetime = bot.processDatetime; - let fileName = "Execution.Context." + bot.startMode + "." + runIndex + ".json"; + let fileName = "/Execution.Context." + bot.startMode + "." + runIndex + ".json"; let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2); - let filePath = bot.filePathRoot + "/Output/" + bot.process + '/' + dateForPath; + let filePath = bot.filePathRoot + "/Output/" + bot.process + '/' + dateForPath + fileName; - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> getExecutionContext -> fileName = " + fileName); } - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> getExecutionContext -> filePath = " + filePath); } - - cloudStorage.getTextFile(filePath, fileName, onFileReceived); + fileStorage.getTextFile(global.DEV_TEAM, filePath, onFileReceived); function onFileReceived(err, text) { @@ -416,9 +391,9 @@ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] saveThemAll -> writeExecutionContext -> Entering function."); } - let fileName = "Execution.Context." + bot.startMode + "." + runIndex +".json"; + let fileName = "/Execution.Context." + bot.startMode + "." + runIndex +".json"; let dateForPath = bot.processDatetime.getUTCFullYear() + '/' + utilities.pad(bot.processDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.processDatetime.getUTCDate(), 2) + '/' + utilities.pad(bot.processDatetime.getUTCHours(), 2) + '/' + utilities.pad(bot.processDatetime.getUTCMinutes(), 2); - let filePath = bot.filePathRoot + "/Output/" + bot.process + '/' + dateForPath; + let filePath = bot.filePathRoot + "/Output/" + bot.process + '/' + dateForPath + fileName; let fileContent = JSON.stringify(thisObject.executionContext); if(fileContent === undefined){ @@ -430,7 +405,7 @@ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] saveThemAll -> writeExecutionContext -> filePath = " + filePath); } if (global.LOG_CONTROL[MODULE_NAME].logContent === true) { logger.write(MODULE_NAME, "[INFO] saveThemAll -> writeExecutionContext -> fileContent = " + fileContent); } - cloudStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + fileStorage.createTextFile(global.DEV_TEAM, filePath, fileContent + '\n', onFileCreated); function onFileCreated(err) { @@ -459,8 +434,8 @@ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] saveThemAll -> writeExucutionHistory -> Entering function."); } - let fileName = "Execution.History." + bot.startMode + "." + runIndex + ".json"; - let filePath = bot.filePathRoot + "/Output/" + bot.process; + let fileName = "/Execution.History." + bot.startMode + "." + runIndex + ".json"; + let filePath = bot.filePathRoot + "/Output/" + bot.process + fileName; if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] saveThemAll -> writeExucutionHistory -> fileName = " + fileName); } if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] saveThemAll -> writeExucutionHistory -> filePath = " + filePath); } @@ -494,7 +469,7 @@ let fileContent = JSON.stringify(thisObject.executionHistory); - cloudStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + fileStorage.createTextFile(global.DEV_TEAM, filePath, fileContent + '\n', onFileCreated); function onFileCreated(err) { @@ -513,10 +488,10 @@ /* Here we will write the file containing the max sequence number. */ fileContent = runIndex; - fileName = "Execution.History." + bot.startMode + "." + "Sequence" + ".json"; - filePath = bot.filePathRoot + "/Output/" + bot.process; + fileName = "/Execution.History." + bot.startMode + "." + "Sequence" + ".json"; + filePath = bot.filePathRoot + "/Output/" + bot.process + fileName; - cloudStorage.createTextFile(filePath, fileName, fileContent + '\n', onSequenceFileCreated); + fileStorage.createTextFile(global.DEV_TEAM, filePath, fileContent + '\n', onSequenceFileCreated); function onSequenceFileCreated(err) { diff --git a/DataDependencies.js b/DataDependencies.js index df98108cc..5db7121d1 100644 --- a/DataDependencies.js +++ b/DataDependencies.js @@ -1,13 +1,10 @@ -exports.newDataDependencies = function newDataDependencies(BOT, logger, DATA_SET, BLOB_STORAGE, UTILITIES) { +exports.newDataDependencies = function newDataDependencies(BOT, logger, DATA_SET) { const FULL_LOG = true; const LOG_FILE_CONTENT = false; const MODULE_NAME = "Data Dependencies"; - let bot = BOT; - let ownerBot; // This is the bot owner of the Data Set. - let thisObject = { config: undefined, dataSets: new Map(), @@ -43,7 +40,7 @@ for (let i = 0; i < thisObject.config.length; i++) { - let dataSetModule = DATA_SET.newDataSet(BOT, logger, BLOB_STORAGE, UTILITIES); + let dataSetModule = DATA_SET.newDataSet(BOT, logger); dataSetModule.initialize(thisObject.config[i], onInitilized); @@ -89,4 +86,4 @@ callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } -}; \ No newline at end of file +}; diff --git a/DataSet.js b/DataSet.js index 215137939..7a3c48c3e 100644 --- a/DataSet.js +++ b/DataSet.js @@ -1,4 +1,4 @@ -exports.newDataSet = function newDataSet(BOT, logger, BLOB_STORAGE, UTILITIES) { +exports.newDataSet = function newDataSet(BOT, logger) { const MODULE_NAME = "Data Set"; @@ -12,13 +12,10 @@ let dependencyConfig; - /* Utilities needed. */ - - let utilities = UTILITIES.newCloudUtilities(bot, logger); - /* Storage account to be used here. */ - let cloudStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + const FILE_STORAGE = require('./Integrations/FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(); return thisObject; @@ -26,31 +23,15 @@ try { + + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } + logger.fileName = MODULE_NAME + "." + pDependencyConfig.bot + "." + pDependencyConfig.product + "." + pDependencyConfig.dataSet; dependencyConfig = pDependencyConfig; - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } + callBackFunction(global.DEFAULT_OK_RESPONSE); - initializeStorage(); - - function initializeStorage() { - - cloudStorage.initialize(dependencyConfig.devTeam, onInizialized); - - function onInizialized(err) { - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> initializeStorage -> onInizialized -> Entering function."); } - callBackFunction(global.DEFAULT_OK_RESPONSE); - - } else { - logger.write(MODULE_NAME, "[ERROR] initialize -> initializeStorage -> onInizialized -> err = " + err.message); - callBackFunction(err); - } - } - } } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); @@ -62,13 +43,12 @@ try { if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getTextFile -> Entering function."); } - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getTextFile -> pFolderPath = " + pFolderPath); } - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getTextFile -> pFileName = " + pFileName); } - let filePathRoot = dependencyConfig.devTeam + "/" + dependencyConfig.bot + "." + dependencyConfig.botVersion.major + "." + dependencyConfig.botVersion.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + dependencyConfig.dataSetVersion; + let filePathRoot = dependencyConfig.devTeam + "/" + dependencyConfig.bot + "." + dependencyConfig.botVersion.major + "." + dependencyConfig.botVersion.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + dependencyConfig.dataSetVersion; let filePath = filePathRoot + "/Output/" + pFolderPath; + filePath += '/' + pFileName - cloudStorage.getTextFile(filePath, pFileName, onFileReceived); + fileStorage.getTextFile(dependencyConfig.devTeam, filePath, onFileReceived); // TODO pass credentials? function onFileReceived(err, text) { @@ -106,10 +86,10 @@ return; } - let filePathRoot = dependencyConfig.devTeam + "/" + dependencyConfig.bot + "." + dependencyConfig.botVersion.major + "." + dependencyConfig.botVersion.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + dependencyConfig.dataSetVersion; - let filePath = filePathRoot + "/Output/" + pFolderPath; + let filePathRoot = dependencyConfig.devTeam + "/" + dependencyConfig.bot + "." + dependencyConfig.botVersion.major + "." + dependencyConfig.botVersion.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + dependencyConfig.dataSetVersion; + let filePath = filePathRoot + "/Output/" + pFolderPath + '/' + pFileName; - cloudStorage.createTextFile(filePath, pFileName, pFileContent, onFileCreated); + fileStorage.createTextFile(dependencyConfig.devTeam, filePath, pFileContent, onFileCreated); // TODO pass credentials? Why should this bot write on other devTeam folder? function onFileCreated(err) { @@ -125,4 +105,4 @@ } } -}; \ No newline at end of file +}; diff --git a/DebugLog.js b/DebugLog.js index 90bce53b3..6c78446ef 100644 --- a/DebugLog.js +++ b/DebugLog.js @@ -6,9 +6,9 @@ let executionDatetime = "D." + global.EXECUTION_DATETIME.getUTCFullYear() + "." + pad(global.EXECUTION_DATETIME.getUTCMonth() + 1, 2) + "." + pad(global.EXECUTION_DATETIME.getUTCDate(), 2) + - ".T." + pad(global.EXECUTION_DATETIME.getUTCHours(),2) + - "." + pad(global.EXECUTION_DATETIME.getUTCMinutes(),2) + - "." + pad(global.EXECUTION_DATETIME.getUTCSeconds(),2); + ".T." + pad(global.EXECUTION_DATETIME.getUTCHours(), 2) + + "." + pad(global.EXECUTION_DATETIME.getUTCMinutes(), 2) + + "." + pad(global.EXECUTION_DATETIME.getUTCSeconds(), 2); let messageId = 0; @@ -63,72 +63,49 @@ let contentToPersist = accumulatedLog; accumulatedLog = "["; - const BLOB_STORAGE = require(ROOT_DIR + 'BlobStorage'); - let cloudStorage = BLOB_STORAGE.newBlobStorage(thisObject.bot); + const FILE_STORAGE = require('./Integrations/FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(); - cloudStorage.initialize(thisObject.bot.devTeam, onInizialized, true); + let filePath = thisObject.bot.filePathRoot + "/Logs/" + thisObject.bot.process + "/" + executionDatetime; - function onInizialized(err) { + if (thisObject.bot.debug.year !== undefined) { - try { - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - let filePath = thisObject.bot.filePathRoot + "/Logs/" + thisObject.bot.process + "/" + executionDatetime; - - if (thisObject.bot.debug.year !== undefined) { - - filePath = filePath + "/" + thisObject.bot.debug.year + "/" + thisObject.bot.debug.month; - } - - let fileName; - - if (internalLoopCounter >= 0) { - - fileName = "Loop." + pad(thisObject.bot.loopCounter, 8) + "." + pad(internalLoopCounter, 4) + ".json"; - - } else { - - fileName = "Loop." + pad(thisObject.bot.loopCounter, 8) + ".json"; - - } - - writeLog(); - - function writeLog() { - - cloudStorage.createTextFile(filePath, fileName, contentToPersist + '\r\n' + "]", onFileCreated); - - function onFileCreated(err) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - console.log("[ERROR] DebugLog -> persist -> onInizialized -> onFileCreated -> err = " + err.message); - console.log("[ERROR] DebugLog -> persist -> onInizialized -> onFileCreated -> filePath = " + filePath); - console.log("[ERROR] DebugLog -> persist -> onInizialized -> onFileCreated -> fileName = " + fileName); - - setTimeout(writeLog, 10000); // Lets retry until we make it. - return; - } - - contentToPersist = ""; - //thisObject = {}; - } - } - - } else { - - console.log("[ERROR] DebugLog -> persist -> onInizialized -> cloud storge failed to initialize. "); - console.log("[ERROR] DebugLog -> persist -> onInizialized -> err.message = " + err.message); - console.log("[ERROR] DebugLog -> persist -> onInizialized -> contentToPersist = " + contentToPersist); - - } - - } catch (err) { - console.log("[ERROR] DebugLog -> persist -> onInizialized -> err = " + err.message); - console.log("[ERROR] DebugLog -> persist -> onInizialized -> contentToPersist = " + contentToPersist); - } + filePath = filePath + "/" + thisObject.bot.debug.year + "/" + thisObject.bot.debug.month; } + let fileName; + + if (internalLoopCounter >= 0) { + + fileName = "Loop." + pad(thisObject.bot.loopCounter, 8) + "." + pad(internalLoopCounter, 4) + ".json"; + + } else { + + fileName = "Loop." + pad(thisObject.bot.loopCounter, 8) + ".json"; + + } + + writeLog(); + + function writeLog() { + + fileStorage.createTextFile(global.DEV_TEAM, filePath + '/' + fileName, contentToPersist + '\r\n' + "]", onFileCreated); + + function onFileCreated(err) { + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + console.log("[ERROR] DebugLog -> persist -> onInizialized -> onFileCreated -> err = " + err.message); + console.log("[ERROR] DebugLog -> persist -> onInizialized -> onFileCreated -> filePath = " + filePath); + console.log("[ERROR] DebugLog -> persist -> onInizialized -> onFileCreated -> fileName = " + fileName); + + setTimeout(writeLog, 10000); // Lets retry until we make it. + return; + } + + contentToPersist = ""; + //thisObject = {}; + } + } } catch (err) { console.log("[ERROR] DebugLog -> persist -> err = " + err.message); console.log("[ERROR] DebugLog -> persist -> onInizialized -> contentToPersist = " + contentToPersist); @@ -140,12 +117,6 @@ try { - if (global.CURRENT_EXECUTION_AT === "IDE") { - - console.log("CloneExecutor" + spacePad(pModule, 50) + " : " + pMessage); - - } - if (disableLogging === true) { return; } if (thisObject.bot === undefined) { return; } diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 9d06c3150..24157ec81 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -8,9 +8,9 @@ let USER_BOT_MODULE; let COMMONS_MODULE; - const BLOB_STORAGE = require(ROOT_DIR + 'BlobStorage'); const MULTI_PERIOD_MARKET = require(ROOT_DIR + 'MultiPeriodMarket'); const MULTI_PERIOD_DAILY = require(ROOT_DIR + 'MultiPeriodDaily'); + const FILE_STORAGE = require('./Integrations/FileStorage.js'); const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); let logger; // We need this here in order for the loopHealth function to work and be able to rescue the loop when it gets in trouble. @@ -38,62 +38,47 @@ UI_COMMANDS = pUI_COMMANDS; processConfig = pProcessConfig; - cloudStorage = BLOB_STORAGE.newBlobStorage(bot, parentLogger); + let fileStorage = FILE_STORAGE.newFileStorage(); - cloudStorage.initialize("AAPlatform", onInizialized); + let filePath; - function onInizialized(err) { + if (global.RUN_AS_TEAM) { + filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo + "/" + pProcessConfig.name; // DevTeams bots only are run at the cloud. + } else { + filePath = global.DEV_TEAM + "/" + "members" + "/" + global.USER_LOGGED_IN + "/" + global.CURRENT_BOT_REPO + "/" + pProcessConfig.name; // DevTeam Members bots only are run at the browser. + } + filePath += "/User.Bot.js" - if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] initialize -> onInizialized -> Entering function."); } + fileStorage.getTextFile(global.DEV_TEAM, filePath, onBotDownloaded); - if (err.result === global.DEFAULT_OK_RESPONSE.result) { + function onBotDownloaded(err, text) { - let filePath; + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - if (global.RUN_AS_TEAM) { - filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo + "/" + pProcessConfig.name; // DevTeams bots only are run at the cloud. - } else { - filePath = global.DEV_TEAM + "/" + "members" + "/" + global.USER_LOGGED_IN + "/" + global.CURRENT_BOT_REPO + "/" + pProcessConfig.name; // DevTeam Members bots only are run at the browser. - } - - const CLOUD_REQUIRE = require(ROOT_DIR + 'CloudRequire'); - let cloudRequire = CLOUD_REQUIRE.newCloudRequire(bot, parentLogger); - - cloudRequire.downloadBot(cloudStorage, filePath, onBotDownloaded); - - function onBotDownloaded(err, pMODULE) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> err.message = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - USER_BOT_MODULE = pMODULE; - - filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; - - cloudRequire.downloadCommons(cloudStorage, filePath, onCommonsDownloaded); - - function onCommonsDownloaded(err, pMODULE) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> onCommonsDownloaded -> err.message = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - COMMONS_MODULE = pMODULE; - - callBackFunction(global.DEFAULT_OK_RESPONSE); - } - } - - } else { - parentLogger.write(MODULE_NAME, "[ERROR] Root -> start -> getBotConfig -> onInizialized -> err = " + err.message); + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> err.message = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + + USER_BOT_MODULE = {} + USER_BOT_MODULE.newUserBot = eval(text); // TODO This needs to be changed function + + filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; + filePath += "/Commons.js" + + fileStorage.getTextFile(global.DEV_TEAM, filePath, onCommonsDownloaded); + + function onCommonsDownloaded(err, text) { + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + parentLogger.write(MODULE_NAME, "[WARN] initialize -> onBotDownloaded -> onCommonsDownloaded -> Commons not found: " + err.code || err.message); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + } + COMMONS_MODULE = {} + COMMONS_MODULE.newCommons = eval(text); // TODO This needs to be changed function + + callBackFunction(global.DEFAULT_OK_RESPONSE); } } } catch (err) { @@ -176,7 +161,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeStatusDependencies -> Entering function."); } - statusDependencies = STATUS_DEPENDENCIES.newStatusDependencies(bot, logger, STATUS_REPORT, BLOB_STORAGE, UTILITIES); + statusDependencies = STATUS_DEPENDENCIES.newStatusDependencies(bot, logger, STATUS_REPORT, UTILITIES); statusDependencies.initialize(processConfig.statusDependencies, pMonth, pYear, onInizialized); @@ -251,7 +236,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> Entering function."); } - dataDependencies = DATA_DEPENDENCIES.newDataDependencies(bot, logger, DATA_SET, BLOB_STORAGE, UTILITIES); + dataDependencies = DATA_DEPENDENCIES.newDataDependencies(bot, logger, DATA_SET); dataDependencies.initialize(processConfig.dataDependencies, onInizialized); @@ -273,12 +258,12 @@ switch (processConfig.framework.name) { case 'Multi-Period-Market': { - processFramework = MULTI_PERIOD_MARKET.newMultiPeriodMarket(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE); + processFramework = MULTI_PERIOD_MARKET.newMultiPeriodMarket(bot, logger, COMMONS_MODULE, UTILITIES, USER_BOT_MODULE, COMMONS_MODULE); intitializeProcessFramework(); break; } case 'Multi-Period-Daily': { - processFramework = MULTI_PERIOD_DAILY.newMultiPeriodDaily(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE); + processFramework = MULTI_PERIOD_DAILY.newMultiPeriodDaily(bot, logger, COMMONS_MODULE, UTILITIES, USER_BOT_MODULE, COMMONS_MODULE); intitializeProcessFramework(); break; } @@ -354,7 +339,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeUserBot -> Entering function."); } - usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE); + usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES); usertBot.initialize(statusDependencies, pMonth, pYear, onInizialized); @@ -873,10 +858,6 @@ function checkLoopHealth(pLastLoop) { - if (global.CURRENT_EXECUTION_AT !== "Node") { - return; - } - if (bot.enableCheckLoopHealth === false) { logger.write(MODULE_NAME, "[WARN] run -> loop -> checkLoopHealth -> bot.enableCheckLoopHealth = " + bot.enableCheckLoopHealth); @@ -913,64 +894,16 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> shallWeStop -> Entering function. "); } - cloudStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + let stop = false + if (process.env.STOP_GRACEFULLY !== undefined) + stop = JSON.parse(process.env.STOP_GRACEFULLY) - cloudStorage.initialize("AAPlatform", onInizialized); - - function onInizialized(err) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> shallWeStop -> onInizialized -> Entering function."); } - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - let filePath = global.DEV_TEAM + "/" + "AACloud"; // DevTeams bots only are run at the cloud. - - let fileName = "this.config.json"; - - cloudStorage.getTextFile(filePath, fileName, onFileReceived); - - function onFileReceived(err, text) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - /* - If for any reason this config file cannot be read, we are not going to abort the loop for that. Instead we are going to assume - that there are no instructions to stop and we will keep the show running. - */ - - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> err.message = " + err.message); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> The config file cound not be read. "); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> We will ignore its content and continue running the loop. "); - - continueCallBack(); - return; - } - - try { - - let configRead = JSON.parse(text); - - if (configRead.stopGracefully === false && global.SHALL_BOT_STOP === false) { - continueCallBack(); - } else { - stopCallBack(); - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> err.message = " + err.message); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - } + if (!stop && global.SHALL_BOT_STOP === false) { + continueCallBack(); + } else { + stopCallBack(); } - } - catch (err) { + } catch (err) { logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> err.message = " + err.message); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -981,7 +914,6 @@ return; } } - } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] run -> loop -> err = " + err.message); clearInterval(fixedTimeLoopIntervalHandle); @@ -991,9 +923,7 @@ callBackFunction(err); } } - } - - catch (err) { + } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] run -> err = " + err.message); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); diff --git a/Integrations/FileStorage.js b/Integrations/FileStorage.js new file mode 100644 index 000000000..c8507ccae --- /dev/null +++ b/Integrations/FileStorage.js @@ -0,0 +1,105 @@ +const axios = require('axios') + +exports.newFileStorage = function newFileStorage() { + const INFO_LOG = true + const ERROR_LOG = true + + let thisObject = { + getTextFile, + createTextFile + } + + return thisObject + + function getTextFile(container, filePath, callBackFunction) { + try { + if (INFO_LOG === true) { console.log('[INFO] getTextFile -> getBlobToText -> Entering function: ' + filePath) } + + axios({ + url: process.env.HOST_URL + 'graphql', + method: 'post', + data: { + query: ` + query web_FileContent($file: web_FileInput){ + web_FileContent(file: $file) + } + `, + variables: { + file: { + container: container.toLowerCase(), + filePath, + storage: process.env.HOST_STORAGE, + accessKey: process.env.HOST_ACCESS_KEY + } + } + } + }).then(res => { + if (res.data.errors) { + let error = { + code: res.data.errors[0] + } + callBackFunction(error) + return + } + let response = res.data.data.web_FileContent + if (response) + callBackFunction(global.DEFAULT_OK_RESPONSE, response) + else + callBackFunction(global.CUSTOM_FAIL_RESPONSE) + }).catch(error => { + if (ERROR_LOG === true) { console.log('newFileStorage: [ERROR] getTextFile -> getBlobToText -> Invalid JSON received. ') } + if (ERROR_LOG === true) { console.log('newFileStorage: [ERROR] getTextFile -> getBlobToText -> response.text = ' + error.message) } + callBackFunction(global.DEFAULT_FAIL_RESPONSE) + }) + } catch (err) { + if (ERROR_LOG === true) { console.log('[ERROR] getTextFile -> getBlobToText -> err = ' + err.stack) } + } + } + + function createTextFile(container, filePath, fileContent, callBackFunction) { + try { + if (INFO_LOG === true) { console.log('[INFO] getTextFile -> createTextFile -> Entering function: ' + filePath) } + + axios({ + url: process.env.HOST_URL + 'graphql', + method: 'post', + data: { + query: ` + mutation web_CreateFile($file: web_FileInput){ + web_CreateFile(file: $file) + } + `, + variables: { + file: { + container: container.toLowerCase(), + filePath, + storage: process.env.HOST_STORAGE, + accessKey: process.env.HOST_ACCESS_KEY, + fileContent + } + } + } + }).then(res => { + if (res.data.errors) { + let error = { + code: res.data.errors[0] + } + callBackFunction(error) + return + } + let response = res.data.data.web_CreateFile + if (response) + callBackFunction(global.DEFAULT_OK_RESPONSE) + else + callBackFunction(global.CUSTOM_FAIL_RESPONSE) + }).catch(error => { + if (ERROR_LOG === true) { console.log('newFileStorage: [ERROR] getTextFile -> getBlobToText -> Invalid JSON received. ') } + if (ERROR_LOG === true) { console.log('newFileStorage: [ERROR] getTextFile -> getBlobToText -> response.text = ' + error.message) } + callBackFunction(global.DEFAULT_FAIL_RESPONSE) + }) + } catch (err) { + if (ERROR_LOG === true) { console.log('[ERROR] createTextFile -> getBlobToText -> err = ' + err.stack) } + } + } + +} diff --git a/Integrations/OperationsModuleIntegration.js b/Integrations/OperationsModuleIntegration.js deleted file mode 100644 index 43001d62c..000000000 --- a/Integrations/OperationsModuleIntegration.js +++ /dev/null @@ -1,55 +0,0 @@ -const axios = require('axios') -const auth = require('../utils/auth') - -exports.updateExecutionResults = async function (summaryDate, buyAverage, sellAverage, - marketRate, combinedProfitsA, combinedProfitsB) { - - try { - - const accessToken = await auth.authenticate() - - const operationsResponse = await axios({ - url: process.env.GATEWAY_ENDPOINT, - method: 'post', - data: { - query: ` - mutation($id: ID!, $summaryDate: Int!, $buyAverage: Float!, $sellAverage: Float!, - $marketRate: Float!, $combinedProfitsA: Float!, $combinedProfitsB: Float!, - $assetA: String!, $assetB: String!){ - operations_UpdateExecutionSummary( - id: $id, - summaryDate: $summaryDate, - buyAverage: $buyAverage, - sellAverage: $sellAverage, - marketRate: $marketRate, - combinedProfitsA: $combinedProfitsA, - combinedProfitsB: $combinedProfitsB, - assetA: $assetA, - assetB: $assetB - ) - } - `, - variables: { - id: process.env.CLONE_ID, - summaryDate: summaryDate, - buyAverage: buyAverage, - sellAverage: sellAverage, - marketRate: marketRate, - combinedProfitsA: combinedProfitsA, - combinedProfitsB: combinedProfitsB, - assetA: global.MARKET.assetA, - assetB: global.MARKET.assetB - }, - }, - headers: { - authorization: 'Bearer ' + accessToken - } - }) - - if (operationsResponse.data.errors) - throw new Error(operationsResponse.data.errors[0].message) - - } catch (error) { - throw new Error('There has been an error updating the execution on the clone: ' + error) - } -} \ No newline at end of file diff --git a/Integrations/Strategy.js b/Integrations/Strategy.js new file mode 100644 index 000000000..585c5aec8 --- /dev/null +++ b/Integrations/Strategy.js @@ -0,0 +1,47 @@ +const axios = require('axios') + +exports.getStrategy = async function () { + try { + + let strategyOnlyOnce = true + if (process.env.STRATEGY_ONLY_ONCE !== undefined) + strategyOnlyOnce = JSON.parse(process.env.STRATEGY_ONLY_ONCE) + + if (!global.STRATEGY || !strategyOnlyOnce) { + + let fbSlug = process.env.BOT + if (process.env.TEST_FB !== undefined) { + fbSlug = process.env.TEST_FB + } + + const strategizerResponse = await axios({ + url: process.env.STRATEGIZER_ENDPOINT || process.env.GATEWAY_ENDPOINT, + method: 'post', + data: { + query: ` + query($fbSlug: String!){ + strategizer_TradingSystemByFb(fbSlug: $fbSlug){ + data + } + } + `, + variables: { + fbSlug: fbSlug + }, + }, + headers: { + access_token: process.env.ACCESS_TOKEN + } + }) + + if (strategizerResponse.data.errors) + throw new Error(strategizerResponse.data.errors[0].message) + + global.STRATEGY = strategizerResponse.data.data.strategizer_TradingSystemByFb.data; + } + return global.STRATEGY + + } catch (error) { + throw new Error('There has been an error getting the strategy: ' + error) + } +} diff --git a/MinioBlobStorage.js b/MinioBlobStorage.js deleted file mode 100644 index 50f8a5968..000000000 --- a/MinioBlobStorage.js +++ /dev/null @@ -1,401 +0,0 @@ - -exports.newMinioBlobBlobStorage = function newMinioBlobBlobStorage(BOT, logger) { - - let FULL_LOG = false; - let ERROR_LOG = true; - let LOG_FILE_CONTENT = false; - - let bot = BOT; - const ROOT_DIR = './'; - - const MODULE_NAME = "Minio Blob Storage"; - const SECOND_TRY_WAIT_TIME = 10000; - - let thisObject = { - initialize: initialize, - createFolder: createFolder, - createTextFile: createTextFile, - getTextFile: getTextFile - }; - - let containerName; - let environment = global.CURRENT_ENVIRONMENT; - - let Minio = require('minio'); - let minioClient; - - return thisObject; - - function initialize(pDataOwner, callBackFunction, disableLogging) { - - try { - - minioClient = new Minio.Client({ - endPoint: process.env.MINIO_END_POINT, - port: JSON.parse(process.env.MINIO_PORT), - useSSL: JSON.parse(process.env.MINIO_USE_SSL), - accessKey: process.env.MINIO_ACCESS_KEY, - secretKey: process.env.MINIO_SECRET_KEY - }) - - containerName = pDataOwner.toLowerCase(); - - if (pDataOwner.environment !== undefined) { // This is use for data migration from one environment to the other. - - environment = pDataOwner.environment; - - } - - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> environment = " + environment); } - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> containerName = " + containerName); } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - - } - catch (err) { - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function createFolder(pFolderPath, callBackFunction) { - - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] createFolder -> Entering function."); } - - try { - - /* Folders do not need to be created when using blobs. */ - - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createFolder -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function createTextFile(pFolderPath, pFileName, pFileContent, callBackFunction) { - - try { - - if (FULL_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] createTextFile -> About to create a text file."); - logger.write(MODULE_NAME, "[INFO] createTextFile -> containerName = " + containerName); - logger.write(MODULE_NAME, "[INFO] createTextFile -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[INFO] createTextFile -> pFileName = " + pFileName); - } - - minioClient.putObject(containerName, pFolderPath + "/" + pFileName, pFileContent, 'text/plain', onFileCreated) - - function onFileCreated(err, result, response) { - - try { - - if (FULL_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> Response from Minio received."); - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> err = " + JSON.stringify(err)); - } - - if (LOG_FILE_CONTENT === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> result = " + JSON.stringify(result)); - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileReceived -> response = " + JSON.stringify(response)); - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileReceived -> pFileContent = " + pFileContent); - } - - /* Console logging */ - console.log(MODULE_NAME, "[INFO] 'createTextFile' -> onFileReceived -> File created at MINIO -> File = " + pFolderPath + "/" + pFileName); - - if (err) { - console.log(MODULE_NAME, "[WARN] 'createTextFile' -> onFileReceived -> File created at MINIO -> err = " + err.message); - } - - if (err) { - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> err = " + JSON.stringify(err)); } - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> result = " + JSON.stringify(result)); } - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> response = " + JSON.stringify(response)); } - - if (ERROR_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> Error trying to create this file."); - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> containerName = " + containerName); - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> pFileName = " + pFileName); - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> err.code = " + err.code); - } - - if ( - err.code === 'ECONNRESET' || - err.code === 'ENOTFOUND' || - err.code === 'ESOCKETTIMEDOUT' || - err.code === 'ETIMEDOUT' || - err.code === 'ECONNREFUSED' || - err.code === 'EADDRINUSE' || - err.code === 'EAI_AGAIN' || - err.code === 'AuthenticationFailed' || - err.code === 'OperationTimedOut' || - err.code === 'ServerBusy' - ) { - - setTimeout(secondTry, SECOND_TRY_WAIT_TIME); - return; - - function secondTry() { - - try { - - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> secondTry -> Retrying to create the file."); - - minioClient.putObject(containerName, pFolderPath + "/" + pFileName, pFileContent, 'text/plain', onSecondTry) - - function onSecondTry(err, result, response) { - - try { - - if (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> File not created. Giving Up."); } - - if (ERROR_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> containerName = " + containerName); - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> pFileName = " + pFileName); - logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> err.code = " + err.code); - } - - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - } else { - logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> secondTry -> File succesfully created on second try."); - callBackFunction(global.DEFAULT_OK_RESPONSE); - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> secondTry -> onSecondTry -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> secondTry -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> Dont know what to do here. Cancelling operation. "); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - - } else { - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> File Created."); } - callBackFunction(global.DEFAULT_OK_RESPONSE); - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function getTextFile(pFolderPath, pFileName, callBackFunction) { - - try { - - if (FULL_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] getTextFile -> About to get a text file."); - logger.write(MODULE_NAME, "[INFO] getTextFile -> containerName = " + containerName); - logger.write(MODULE_NAME, "[INFO] getTextFile -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[INFO] getTextFile -> pFileName = " + pFileName); - } - - minioClient.getObject(containerName, pFolderPath + "/" + pFileName, function (err, dataStream) { - let data = ''; - - if (err) { - onFileReceived(err, null, "Error retrieving file " + pFolderPath + "/" + pFileName + " from bucket " + containerName); - return - } - dataStream.on('data', function (chunk) { - data += chunk - }) - dataStream.on('end', function () { - onFileReceived(null, data, "Data retrieved."); - }) - dataStream.on('error', function (err) { - onFileReceived(err, null, "Error on data stream while retrieving file " + pFolderPath + "/" + pFileName + " from bucket " + containerName); - }) - }) - - function onFileReceived(err, text, response) { - - try { - - if (FULL_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> Response from Minio received."); - logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> err = " + JSON.stringify(err)); - } - - if (LOG_FILE_CONTENT === true && logger !== undefined) { - logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> response = " + JSON.stringify(response)); - logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> text = " + text); - } - - /* Console logging */ - console.log(MODULE_NAME, "[INFO] 'getTextFile' -> onFileReceived -> File read from MINIO -> File = " + pFolderPath + "/" + pFileName); - - if (err) { - console.log(MODULE_NAME, "[WARN] 'getTextFile' -> onFileReceived -> File read from MINIO -> err = " + err.message); - } - - if (err) { - - if (ERROR_LOG === true && logger !== undefined && err.code !== 'BlobNotFound') { - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> Error trying to get this file."); - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> containerName = " + containerName); - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> pFileName = " + pFileName); - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> err.code = " + err.code); - } - - if ( - err.code === 'ECONNRESET' || - err.code === 'ENOTFOUND' || - err.code === 'ESOCKETTIMEDOUT' || - err.code === 'ETIMEDOUT' || - err.code === 'ECONNREFUSED' || - err.code === 'EADDRINUSE' || - err.code === 'EAI_AGAIN' || - err.code === 'AuthenticationFailed' || - err.code === 'OperationTimedOut' || - err.code === 'ServerBusy' - ) { - - setTimeout(secondTry, SECOND_TRY_WAIT_TIME); - return; - - function secondTry() { - - try { - - logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> secondTry -> Retrying to get the file."); - - minioClient.getObject(containerName, pFolderPath + "/" + pFileName, function (err, dataStream) { - let data = ''; - - if (err) { - onSecondTry(err, null, "Error retrieving file " + pFolderPath + "/" + pFileName + " from bucket " + containerName); - return - } - dataStream.on('data', function (chunk) { - data += chunk - }) - dataStream.on('end', function () { - onSecondTry(null, data, "Data retrieved."); - }) - dataStream.on('error', function (err) { - onSecondTry(err, null, "Error on data stream while retrieving file " + pFolderPath + "/" + pFileName + " from bucket " + containerName); - }) - }) - - function onSecondTry(err, text, response) { - - try { - - if (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> File not retrieved. Giving Up."); } - - if (ERROR_LOG === true && logger !== undefined) { - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> containerName = " + containerName); - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> pFileName = " + pFileName); - logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> err.code = " + err.code); - } - - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - } else { - logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> secondTry -> File succesfully retrieved on second try."); - callBackFunction(global.DEFAULT_OK_RESPONSE, text); - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> secondTry -> onSecondTry -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> secondTry -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - - if (err.code === "NoSuchKey") { - - /* This is how Minio tell us the file does not exist. */ - - let customErr = { - result: global.CUSTOM_FAIL_RESPONSE.result, - message: "File does not exist." - }; - - logger.write(MODULE_NAME, "[WARN] getTextFile -> onFileReceived -> Custom Response -> message = " + customErr.message); - logger.write(MODULE_NAME, "[WARN] getTextFile -> containerName = " + containerName); - logger.write(MODULE_NAME, "[WARN] getTextFile -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[WARN] getTextFile -> pFileName = " + pFileName); - - callBackFunction(customErr); - return; - - } - - if (err.code === "ParentNotFound") { - - /* This is how Minio tell us the folder where the file was supposed to be does not exist. We map this to our own response. */ - - let customErr = { - result: global.CUSTOM_FAIL_RESPONSE.result, - message: "Folder does not exist." - }; - - logger.write(MODULE_NAME, "[WARN] getTextFile -> onFileReceived -> Custom Response -> message = " + customErr.message); - logger.write(MODULE_NAME, "[WARN] getTextFile -> containerName = " + containerName); - logger.write(MODULE_NAME, "[WARN] getTextFile -> pFolderPath = " + pFolderPath); - logger.write(MODULE_NAME, "[WARN] getTextFile -> pFileName = " + pFileName); - - callBackFunction(customErr); - return; - - } - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileCreated -> err = " + JSON.stringify(err)); } - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileCreated -> response = " + JSON.stringify(response)); } - - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> Dont know what to do here. Cancelling operation. "); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - - } else { - if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> File retrieved."); } - callBackFunction(global.DEFAULT_OK_RESPONSE, text); - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> err = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - -}; diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index bceab2532..08cae3da7 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -1,9 +1,8 @@ -exports.newMultiPeriodDaily = function newMultiPeriodDaily(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE) { +exports.newMultiPeriodDaily = function newMultiPeriodDaily(bot, logger, COMMONS, UTILITIES, USER_BOT_MODULE, COMMONS_MODULE) { const FULL_LOG = true; const LOG_FILE_CONTENT = false; const MODULE_NAME = "Multi Period Daily"; - const commons = COMMONS.newCommons(bot, logger, UTILITIES); const GMT_SECONDS = ':00.000 GMT+0000'; const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; @@ -12,7 +11,7 @@ start: start }; - let utilities = UTILITIES.newCloudUtilities(bot, logger); + let utilities = UTILITIES.newCloudUtilities(logger); let statusDependencies; let dataDependencies; @@ -20,7 +19,9 @@ let dataFiles = []; let usertBot; - let currentBotStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + + const FILE_STORAGE = require('./Integrations/FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(); let processConfig; @@ -57,20 +58,8 @@ } - currentBotStorage.initialize(bot.devTeam, onStorageInizialized); - - function onStorageInizialized(err) { - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE); - usertBot.initialize(dataDependencies, callBackFunction); - - } else { - logger.write(MODULE_NAME, "[ERROR] initialize -> onStorageInizialized -> err = " + err.message); - callBackFunction(err); - } - } + usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES); + usertBot.initialize(dataDependencies, callBackFunction); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); @@ -423,7 +412,7 @@ let filePath = dependency.product + '/' + "Multi-Period-Daily" + "/" + outputPeriodLabel + "/" + dateForPath; let fileName = market.assetA + '_' + market.assetB + ".json"; - storage.getTextFile(filePath, fileName, onFileReceived, true); + storage.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { @@ -492,7 +481,7 @@ let filePath = dependency.product + '/' + "Multi-Period-Daily" + "/" + outputPeriodLabel + "/" + dateForPath; let fileName = market.assetA + '_' + market.assetB + ".json"; - storage.getTextFile(filePath, fileName, onFileReceived, true); + storage.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { @@ -720,13 +709,13 @@ let fileContent = JSON.stringify(dataRange); - let fileName = 'Data.Range.' + market.assetA + '_' + market.assetB + '.json'; - let filePath = bot.filePathRoot + "/Output/" + pProductFolder + "/" + bot.process; + let fileName = '/Data.Range.' + market.assetA + '_' + market.assetB + '.json'; + let filePath = bot.filePathRoot + "/Output/" + pProductFolder + "/" + bot.process + fileName; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> fileName = " + fileName); } if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> filePath = " + filePath); } - currentBotStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + fileStorage.createTextFile(global.DEV_TEAM, filePath, fileContent + '\n', onFileCreated); function onFileCreated(err) { diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index 37c7603d1..fdd08f1d5 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -1,18 +1,14 @@ -exports.newMultiPeriodMarket = function newMultiPeriodMarket(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE) { +exports.newMultiPeriodMarket = function newMultiPeriodMarket(bot, logger, COMMONS, UTILITIES, USER_BOT_MODULE, COMMONS_MODULE) { const FULL_LOG = true; const LOG_FILE_CONTENT = false; const MODULE_NAME = "Multi Period Market"; - const commons = COMMONS.newCommons(bot, logger, UTILITIES); - thisObject = { initialize: initialize, start: start }; - let utilities = UTILITIES.newCloudUtilities(bot, logger); - let statusDependencies; let dataDependencies; let storages = []; @@ -47,7 +43,7 @@ dependency.bot + "-" + dependency.product + "-" + dependency.dataSet + "-" + - dependency.dataSetVersion + dependency.dataSetVersion storage = dataDependencies.dataSets.get(key); @@ -55,7 +51,7 @@ } - usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE); + usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES); usertBot.initialize(dataDependencies, callBackFunction); } catch (err) { @@ -114,9 +110,9 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoop -> Entering function."); } /* - + We will iterate through all posible periods. - + */ n = 0 // loop Variable representing each possible period as defined at the periods array. @@ -173,7 +169,7 @@ let filePath = dependency.product + "/" + "Multi-Period-Market" + "/" + outputPeriodLabel; - storage.getTextFile(filePath, fileName, onFileReceived, true); + storage.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { @@ -239,7 +235,7 @@ callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } - + function callTheBot() { try { diff --git a/Root.js b/Root.js index e440c9870..a159898d4 100644 --- a/Root.js +++ b/Root.js @@ -72,8 +72,6 @@ const FULL_LOG = true; - let cloudStorage; - let logDisplace = "CloneExecutor" + " "; let UI_COMMANDS; @@ -126,46 +124,8 @@ UI_COMMANDS = pUI_COMMANDS; - const BLOB_STORAGE = require(ROOT_DIR + 'BlobStorage'); - cloudStorage = BLOB_STORAGE.newBlobStorage(); + callBackFunction(); - cloudStorage.initialize("AAPlatform", onInizialized, true); - - function onInizialized(err) { - - console.log(logDisplace + "Root : [INFO] initialize -> onInizialized -> Entering function. "); - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - let filePath = global.DEV_TEAM + "/" + "AACloud"; // DevTeams bots only are run at the cloud. - - let fileName = "this.config.json"; - - cloudStorage.getTextFile(filePath, fileName, onFileReceived); - - function onFileReceived(err, text) { - - console.log(logDisplace + "Root : [INFO] initialize -> onInizialized -> onFileReceived -> Entering function. "); - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - console.log(logDisplace + "Root : [INFO] initialize -> onInizialized -> onFileReceived -> err = " + err.message); - return; - } - - try { - global.PLATFORM_CONFIG = JSON.parse(text); - callBackFunction(); - } catch (err) { - console.log(logDisplace + "Root : [ERROR] initialize -> onInizialized -> onFileReceived -> err = " + err.message); - return; - } - } - - } else { - console.log(logDisplace + "Root : [ERROR] initialize -> onInizialized -> err = " + err.message); - return; - } - } } catch (err) { console.log(logDisplace + "Root : [ERROR] initialize -> err.message = " + err.message); @@ -205,43 +165,29 @@ try { console.log(logDisplace + "Root : [INFO] start -> getBotConfig -> Entering function. "); - const BLOB_STORAGE = require(ROOT_DIR + 'BlobStorage'); - let cloudStorage = BLOB_STORAGE.newBlobStorage(cloneToExecute); + const FILE_STORAGE = require('./Integrations/FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(); - cloudStorage.initialize("AAPlatform", onInizialized, true); + let filePath = global.DEV_TEAM + '/bots/' + global.CURRENT_BOT_REPO + '/this.bot.config.json'; - function onInizialized(err) { + fileStorage.getTextFile(global.DEV_TEAM, filePath, onFileReceived); - if (err.result === global.DEFAULT_OK_RESPONSE.result) { + function onFileReceived(err, text) { - let filePath = global.DEV_TEAM + "/" + "bots" + "/" + cloneToExecute.repo; // DevTeams bots only are run at the cloud. + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> err = " + err.message); + console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> filePath = " + filePath); + console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> fileName = " + fileName); - let fileName = "this.bot.config.json"; + return; + } - cloudStorage.getTextFile(filePath, fileName, onFileReceived); - - function onFileReceived(err, text) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> err = " + err.message); - console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> filePath = " + filePath); - console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> fileName = " + fileName); - - return; - } - - try { - botConfig = JSON.parse(text); - botConfig.repo = cloneToExecute.repo; - findProcess(); - } catch (err) { - console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> err = " + err.message); - return; - } - } - - } else { - console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> err = " + err.message); + try { + botConfig = JSON.parse(text); + botConfig.repo = cloneToExecute.repo; + findProcess(); + } catch (err) { + console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> err = " + err.message); return; } } @@ -266,9 +212,9 @@ /* File Path Root */ - botConfig.filePathRoot = botConfig.devTeam + "/" + botConfig.codeName + "." + botConfig.version.major + "." + botConfig.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + botConfig.dataSetVersion; + botConfig.filePathRoot = botConfig.devTeam + "/" + botConfig.codeName + "." + botConfig.version.major + "." + botConfig.version.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + botConfig.dataSetVersion; - if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> cloneToExecute.process = " + cloneToExecute.process); } + if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> filePathRoot = " + botConfig.filePathRoot); } /* Now we loop throug all the configured processes at each bots configuration until we find the one we are supposed to run at this Node.js process. */ @@ -288,6 +234,7 @@ processConfig.timePeriod = global.EXECUTION_CONFIG.timePeriod } + // TODO Pending. Move this if inside IndicatorBotProcessMainLoop line 270 if (processConfig.framework !== undefined) { if (processConfig.framework.name === "Multi-Period-Daily" || processConfig.framework.name === "Multi-Period-Market") { if (processConfig.startMode.noTime.beginDatetime !== undefined) { @@ -472,12 +419,7 @@ botConfig.startMode = "Backtest"; console.log(logDisplace + "Root : [INFO] start -> findProcess -> Process found at the bot configuration file. -> Start Mode = " + botConfig.startMode); - botConfig.backtest = processConfig.startMode.backtest; - - /* We override these waitTimes to the one specified at the backtest configuration. */ - - processConfig.normalWaitTime = processConfig.startMode.backtest.waitTime; - processConfig.retryWaitTime = processConfig.startMode.backtest.waitTime; + botConfig.backtest = processConfig.startMode.backtest /* Backtest Mode does not support Resume Execution, so this is the only way. */ @@ -755,12 +697,9 @@ let clonName = botConfig.codeName + "-" + "Clon" + clonKey; clonName += ".1.0"; + clonName += "-" + process.env.CLONE_ID; - if (global.CURRENT_EXECUTION_AT === "Node") { - clonName += "-" + process.env.CLONE_ID; - } - - botConfig.filePathRoot = botConfig.devTeam + "/" + clonName + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + botConfig.dataSetVersion; + botConfig.filePathRoot = botConfig.devTeam + "/" + clonName + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + botConfig.dataSetVersion; botConfig.instance = clonName; botConfig.instanceIndex = i; @@ -779,14 +718,10 @@ let genes = {}; let clonName = botConfig.codeName; - - if (global.CURRENT_EXECUTION_AT === "Node") { - clonName += "-" + process.env.CLONE_ID; - } - + clonName += "-" + process.env.CLONE_ID; clonName += ".1.0"; - botConfig.filePathRoot = botConfig.devTeam + "/" + clonName + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + botConfig.dataSetVersion; + botConfig.filePathRoot = botConfig.devTeam + "/" + clonName + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + botConfig.dataSetVersion; pBotConfig.instance = clonName; pBotConfig.instanceIndex = 0; diff --git a/Run.js b/Run.js index b2374168c..c8003e294 100644 --- a/Run.js +++ b/Run.js @@ -1,7 +1,5 @@ require('dotenv').config() -global.CURRENT_ENVIRONMENT = "Develop"; -global.CURRENT_EXECUTION_AT = "Node"; global.SHALL_BOT_STOP = false; global.AT_BREAKPOINT = false; // This is used only when running at the browser. global.RUN_AS_TEAM = false; @@ -38,7 +36,6 @@ function readExecutionConfiguration() { let startMode // Environment Configuration - global.CURRENT_ENVIRONMENT = process.env.PLATFORM_ENVIRONMENT global.GATEWAY_ENDPOINT = process.env.GATEWAY_ENDPOINT // General Financial Being Configuration @@ -57,8 +54,7 @@ function readExecutionConfiguration() { run: "false", resumeExecution: process.env.RESUME_EXECUTION, beginDatetime: process.env.BEGIN_DATE_TIME, - endDatetime: process.env.END_DATE_TIME, - waitTime: process.env.WAIT_TIME + endDatetime: process.env.END_DATE_TIME } let competition = { @@ -123,7 +119,19 @@ function readExecutionConfiguration() { dataSet: process.env.DATA_SET }; - readStoragePermissions(); + // var cloneExecutorConfig = require('./package.json'); + // global.CLONE_EXECUTOR = { + // codeName: 'clone-executor', + // codeName: 'clone-executor', + // version: cloneExecutorConfig.version + // } + + global.CLONE_EXECUTOR = { + codeName: 'AACloud', + version: '1.1' + } + + startRoot(); } catch (err) { @@ -165,61 +173,6 @@ function getTimePeriod(timePeriod) { } } -function readStoragePermissions() { - try { - console.log("[INFO] Run -> readStoragePermissions -> Entering function. "); - - /* Dinamically generating the azure storage permissions for the bot to run */ - global.STORAGE_BASE_URL = process.env.STORAGE_BASE_URL - global.USER_PROFILE = {} - const MAX_STORAGE_PERMISSION_DAYS = 10; - const STORAGE_ACCESS_MANAGER = require('./StorageAccessManager'); - storageAccessManager = STORAGE_ACCESS_MANAGER.newStorageAccessManager(); - - storageAccessManager.initialize(process.env.STORAGE_CONNECTION_STRING, onInitialized); - - function onInitialized() { - - /* Here we will rearrange the storage permissions array into a map, so that it can be easily consumed when needed. */ - - let permissionsMap = new Map; - - let containers = ["AAPlatform", "AAMasters", "AAVikings", global.DEV_TEAM] - let container; - let key; - let value; - - for (let i = 0; i < containers.length; i++) { - - container = containers[i]; - let readPermission = storageAccessManager.getPermission(container.toLowerCase(), "READ", MAX_STORAGE_PERMISSION_DAYS); - - key = container + ".READ"; - value = readPermission; - - permissionsMap.set(key, value); - - } - - let writePermission = storageAccessManager.getPermission(container.toLowerCase(), "WRITE", MAX_STORAGE_PERMISSION_DAYS); - - key = container + ".WRITE"; - value = writePermission; - - permissionsMap.set(key, value); - - global.USER_PROFILE.storagePermissions = permissionsMap; - global.USER_PROFILE.connectionString = ""; - process.env.STORAGE_CONNECTION_STRING = ""; - - startRoot(); - - } - } catch (err) { - console.log("[ERROR] Run -> readStoragePermissions -> err = " + err.message); - } -} - function startRoot() { console.log("[INFO] Run -> startRoot -> Entering function. "); diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index 0efe5db11..610165797 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -8,7 +8,8 @@ let USER_BOT_MODULE; let COMMONS_MODULE; - const BLOB_STORAGE = require(ROOT_DIR + 'BlobStorage'); + const FILE_STORAGE = require('./Integrations/FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(); const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); let logger; // We need this here in order for the loopHealth function to work and be able to rescue the loop when it gets in trouble. @@ -36,64 +37,46 @@ UI_COMMANDS = pUI_COMMANDS; processConfig = pProcessConfig; - cloudStorage = BLOB_STORAGE.newBlobStorage(bot, parentLogger); + let filePath; + if (global.RUN_AS_TEAM) { + filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo + "/" + pProcessConfig.name; // DevTeams bots only are run at the cloud. + } else { + filePath = global.DEV_TEAM + "/" + "members" + "/" + global.USER_LOGGED_IN + "/" + global.CURRENT_BOT_REPO + "/" + pProcessConfig.name; // DevTeam Members bots only are run at the browser. + } + filePath += "/User.Bot.js" - cloudStorage.initialize("AAPlatform", onInizialized); + fileStorage.getTextFile(global.DEV_TEAM, filePath, onBotDownloaded); - function onInizialized(err) { + function onBotDownloaded(err, text) { - if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] initialize -> onInizialized -> Entering function."); } + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - let filePath; - - if (global.RUN_AS_TEAM) { - filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo + "/" + pProcessConfig.name; // DevTeams bots only are run at the cloud. - } else { - filePath = global.DEV_TEAM + "/" + "members" + "/" + global.USER_LOGGED_IN + "/" + global.CURRENT_BOT_REPO + "/" + pProcessConfig.name; // DevTeam Members bots only are run at the browser. - } - - const CLOUD_REQUIRE = require(ROOT_DIR + 'CloudRequire'); - let cloudRequire = CLOUD_REQUIRE.newCloudRequire(bot, parentLogger); - - cloudRequire.downloadBot(cloudStorage, filePath, onBotDownloaded); - - function onBotDownloaded(err, pMODULE) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> err.message = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - USER_BOT_MODULE = pMODULE; - - filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; - - cloudRequire.downloadCommons(cloudStorage, filePath, onCommonsDownloaded); - - function onCommonsDownloaded(err, pMODULE) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> onCommonsDownloaded -> err.message = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - COMMONS_MODULE = pMODULE; - - callBackFunction(global.DEFAULT_OK_RESPONSE); - } - } - - } else { - parentLogger.write(MODULE_NAME, "[ERROR] Root -> start -> getBotConfig -> onInizialized -> err = " + err.message); + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> err.message = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + USER_BOT_MODULE = {} + USER_BOT_MODULE.newUserBot = eval(text); // TODO This needs to be changed function + + filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; + filePath += "/Commons.js" + + fileStorage.getTextFile(global.DEV_TEAM, filePath, onCommonsDownloaded); + + function onCommonsDownloaded(err, text) { + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + parentLogger.write(MODULE_NAME, "[WARN] initialize -> onBotDownloaded -> onCommonsDownloaded -> Commons not found: " + err.code || err.message); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + } + COMMONS_MODULE = {} + COMMONS_MODULE.newCommons = eval(text); // TODO This needs to be changed function + + callBackFunction(global.DEFAULT_OK_RESPONSE); } } + } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); @@ -172,7 +155,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeStatusDependencies -> Entering function."); } - statusDependencies = STATUS_DEPENDENCIES.newStatusDependencies(bot, logger, STATUS_REPORT, BLOB_STORAGE, UTILITIES); + statusDependencies = STATUS_DEPENDENCIES.newStatusDependencies(bot, logger, STATUS_REPORT, UTILITIES); statusDependencies.initialize(processConfig.statusDependencies, pMonth, pYear, onInizialized); @@ -320,7 +303,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeUserBot -> Entering function."); } - usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE, STATUS_REPORT, exchangeAPI); + usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, STATUS_REPORT, exchangeAPI); usertBot.initialize(statusDependencies, pMonth, pYear, onInizialized); @@ -643,10 +626,6 @@ function checkLoopHealth(pLastLoop) { - if (global.CURRENT_EXECUTION_AT !== "Node") { - return; - } - if (bot.enableCheckLoopHealth === false) { logger.write(MODULE_NAME, "[WARN] run -> loop -> checkLoopHealth -> bot.enableCheckLoopHealth = " + bot.enableCheckLoopHealth); @@ -683,64 +662,16 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> shallWeStop -> Entering function. "); } - cloudStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + let stop = false + if (process.env.STOP_GRACEFULLY !== undefined) + stop = JSON.parse(process.env.STOP_GRACEFULLY) - cloudStorage.initialize("AAPlatform", onInizialized); - - function onInizialized(err) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> shallWeStop -> onInizialized -> Entering function."); } - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - let filePath = global.DEV_TEAM + "/" + "AACloud"; // DevTeams bots only are run at the cloud. - - let fileName = "this.config.json"; - - cloudStorage.getTextFile(filePath, fileName, onFileReceived); - - function onFileReceived(err, text) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - /* - If for any reason this config file cannot be read, we are not going to abort the loop for that. Instead we are going to assume - that there are no instructions to stop and we will keep the show running. - */ - - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> err.message = " + err.message); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> The config file cound not be read. "); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> We will ignore its content and continue running the loop. "); - - continueCallBack(); - return; - } - - try { - - let configRead = JSON.parse(text); - - if (configRead.stopGracefully === false && global.SHALL_BOT_STOP === false) { - continueCallBack(); - } else { - stopCallBack(); - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> err.message = " + err.message); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - } + if (!stop && global.SHALL_BOT_STOP === false) { + continueCallBack(); + } else { + stopCallBack(); } - } - catch (err) { + } catch (err) { logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> err.message = " + err.message); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); diff --git a/StatusDependencies.js b/StatusDependencies.js index 2ce421ef8..5159c4e24 100644 --- a/StatusDependencies.js +++ b/StatusDependencies.js @@ -1,4 +1,4 @@ -exports.newStatusDependencies = function newStatusDependencies(BOT, logger, STATUS_REPORT, BLOB_STORAGE, UTILITIES) { +exports.newStatusDependencies = function newStatusDependencies(BOT, logger, STATUS_REPORT, UTILITIES) { const FULL_LOG = true; const LOG_FILE_CONTENT = false; @@ -7,14 +7,11 @@ let thisObject = { config: undefined, - statusReports: new Map(), + statusReports: new Map(), initialize: initialize, keys: [] }; - let bot = BOT; - let ownerBot; // This is the bot owner of the Status Report. Only owners can save the report and override the existing content. - return thisObject; function initialize(pStatusDependenciesConfig, pMonth, pYear, callBackFunction) { @@ -35,7 +32,7 @@ for (let i = 0; i < pStatusDependenciesConfig.length; i++) { - let statusReportModule = STATUS_REPORT.newStatusReport(BOT, logger, BLOB_STORAGE, UTILITIES); + let statusReportModule = STATUS_REPORT.newStatusReport(BOT, logger, UTILITIES); statusReportModule.initialize(pStatusDependenciesConfig[i], pMonth, pYear, onInitilized); @@ -136,4 +133,4 @@ } } -}; \ No newline at end of file +}; diff --git a/StatusReport.js b/StatusReport.js index d92dfc2cc..b731f4a60 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -1,6 +1,6 @@ -exports.newStatusReport = function newStatusReport(BOT, logger, BLOB_STORAGE, UTILITIES) { +exports.newStatusReport = function newStatusReport(BOT, logger, UTILITIES) { - /* + /* A Status Report is a file that every bot reads at the begining of its execution and saves after it finishes its job. The purpose of the file is to record checkpoint information of what was the last thing done by the bot and helpfull enough to start the next execution. @@ -25,11 +25,12 @@ /* Utilities needed. */ - let utilities = UTILITIES.newCloudUtilities(bot, logger); + let utilities = UTILITIES.newCloudUtilities(logger); /* Storage account to be used here. */ - let cloudStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + const FILE_STORAGE = require('./Integrations/FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(); let month; let year; @@ -51,31 +52,9 @@ logger.fileName = MODULE_NAME + "." + pOwner.bot + "." + pOwner.process + "." + year + "." + month; - timePath = "/" + year + "/" + month; - } - - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } - - - initializeStorage(); - - function initializeStorage() { - - cloudStorage.initialize(owner.devTeam, onInizialized); - - function onInizialized(err) { - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> initializeStorage -> onInizialized -> Entering function."); } - callBackFunction(global.DEFAULT_OK_RESPONSE); - - } else { - logger.write(MODULE_NAME, "[ERROR] initialize -> initializeStorage -> onInizialized -> err = " + err.message); - callBackFunction(err); - } - } + timePath = "/" + year + "/" + month; } + callBackFunction(global.DEFAULT_OK_RESPONSE); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); @@ -86,7 +65,7 @@ try { - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> load -> Entering function."); } + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] load -> Entering function."); } let fileName = "Status.Report." + global.MARKET.assetA + '_' + global.MARKET.assetB + ".json"; let filePath; @@ -96,7 +75,7 @@ if (ownerId !== botId) { - let rootPath = owner.devTeam + "/" + owner.bot + "." + owner.botVersion.major + "." + owner.botVersion.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + owner.dataSetVersion; + let rootPath = owner.devTeam + "/" + owner.bot + "." + owner.botVersion.major + "." + owner.botVersion.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + owner.dataSetVersion; filePath = rootPath + "/Reports/" + owner.process + timePath; @@ -110,18 +89,16 @@ fileName = bot.startMode + "." + fileName; } - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> load -> fileName = " + fileName); } - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> load -> filePath = " + filePath); } + filePath += '/' + fileName - cloudStorage.getTextFile(filePath, fileName, onFileReceived); + fileStorage.getTextFile(global.DEV_TEAM, filePath, onFileReceived); function onFileReceived(err, text) { - if ( - err.result === global.CUSTOM_FAIL_RESPONSE.result && - (err.message === 'Folder does not exist.' || err.message === 'File does not exist.') - ) { - logger.write(MODULE_NAME, "[INFO] initialize -> load -> onFileReceived -> err = " + err.message); + if ( err.result === global.CUSTOM_FAIL_RESPONSE.result && (err.message === 'Folder does not exist.' || err.message === 'File does not exist.') + || err.code === "The specified key does not exist.") { + + logger.write(MODULE_NAME, "[INFO] load -> onFileReceived -> err = " + err.message); /* In this case we can assume that this is the first execution ever of this bot.*/ @@ -131,19 +108,19 @@ result: global.CUSTOM_OK_RESPONSE.result, message: "Status Report was never created." }; - logger.write(MODULE_NAME, "[WARN] initialize -> load -> onFileReceived -> customOK = " + customOK.message); + logger.write(MODULE_NAME, "[WARN] load -> onFileReceived -> customOK = " + customOK.message); callBackFunction(customOK); return; } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] initialize -> load -> onFileReceived -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] load -> onFileReceived -> err = " + err.message); callBackFunction(err); return; } if (global.LOG_CONTROL[MODULE_NAME].logContent === true) { - logger.write(MODULE_NAME, "[INFO] initialize -> load -> onFileReceived -> Content received = " + text); + logger.write(MODULE_NAME, "[INFO] load -> onFileReceived -> Content received = " + text); } try { @@ -158,20 +135,20 @@ is being updated at the moment of the read. The bot can not run without a valid Status Report but we can request the platform to retry later. */ - logger.write(MODULE_NAME, "[ERROR] initialize -> load -> onFileReceived -> Error Parsing the Status report. -> Err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] load -> onFileReceived -> Error Parsing the Status report. -> Err = " + err.message); let customFail = { result: global.CUSTOM_FAIL_RESPONSE.result, message: "Status Report is corrupt." }; - logger.write(MODULE_NAME, "[WARN] initialize -> load -> onFileReceived -> customFail = " + customFail.message); + logger.write(MODULE_NAME, "[WARN] load -> onFileReceived -> customFail = " + customFail.message); callBackFunction(customFail); return; } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] initialize -> load -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] load -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -203,12 +180,14 @@ fileName = bot.startMode + "." + fileName; } + filePath += '/' + fileName + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] save -> fileName = " + fileName); } if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] save -> filePath = " + filePath); } let fileContent = JSON.stringify(thisObject.file); - cloudStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + fileStorage.createTextFile(global.DEV_TEAM, filePath, fileContent + '\n', onFileCreated); function onFileCreated(err) { @@ -237,15 +216,15 @@ function verifyMarketComplete(callBackFunction) { - /* - The mission of this function is to update the main status report on a scheme where there are monthly sub reports. - This report contains the date of the last file sucessfully checked but in a consecutive way. + /* + The mission of this function is to update the main status report on a scheme where there are monthly sub reports. + This report contains the date of the last file sucessfully checked but in a consecutive way. - For example: if the market starts in March, and March, April and June are checked, then the file will be the last of June even if September is also checked. + For example: if the market starts in March, and March, April and June are checked, then the file will be the last of June even if September is also checked. - IMPORTANT NOTE: Once executed, this function will leave thisObject in an inconsistant state, so you need to re-initialize it if you wish to use it again. + IMPORTANT NOTE: Once executed, this function will leave thisObject in an inconsistant state, so you need to re-initialize it if you wish to use it again. - */ + */ try { @@ -478,11 +457,10 @@ } } } - catch (err) - { + catch (err) { logger.write(MODULE_NAME, "[ERROR] verifyMarketComplete -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); return; } } -}; \ No newline at end of file +}; diff --git a/StorageAccessManager.js b/StorageAccessManager.js deleted file mode 100644 index 9a36d9d25..000000000 --- a/StorageAccessManager.js +++ /dev/null @@ -1,62 +0,0 @@ -exports.newStorageAccessManager = function newStorageAccessManager() { - - let thisObject = { - getPermission: getPermission, - initialize: initialize - } - - let connectionString; - - return thisObject; - - function initialize(pConnectionString, callBackFunction) { - - connectionString = pConnectionString; - - callBackFunction(); - } - - function getPermission(pContainer, pType, pDays) { - - let azure = require('azure-storage'); - let blobService = azure.createBlobService(connectionString); - - let startDate = new Date(); - startDate.setDate(startDate.getDate() - 1); - - let expiryDate = new Date(startDate); - expiryDate.setDate(startDate.getDate() + pDays); - - let azurePermissions; - - switch (pType) { - case "READ": { - azurePermissions = azure.BlobUtilities.SharedAccessPermissions.READ; - break; - } - case "WRITE": { - azurePermissions = azure.BlobUtilities.SharedAccessPermissions.WRITE; - break; - } - case "DELETE": { - azurePermissions = azure.BlobUtilities.SharedAccessPermissions.DELETE; - break; - } - } - - let blobName = undefined; - - let sharedAccessPolicy = { - AccessPolicy: { - Permissions: azurePermissions, - Start: startDate, - Expiry: expiryDate - } - }; - - let sasToken = blobService.generateSharedAccessSignature(pContainer, blobName, sharedAccessPolicy); - - return "BlobEndpoint=" + global.STORAGE_BASE_URL + ";SharedAccessSignature=" + sasToken + ""; - } -} - diff --git a/TradingBotProcessMainLoop.js b/TradingBotProcessMainLoop.js index 97e2a0629..cf21ee6f4 100644 --- a/TradingBotProcessMainLoop.js +++ b/TradingBotProcessMainLoop.js @@ -8,10 +8,9 @@ let USER_BOT_MODULE; let COMMONS_MODULE; - const operationsIntegration = require('./Integrations/OperationsModuleIntegration') - - const BLOB_STORAGE = require(ROOT_DIR + 'BlobStorage'); const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); + const FILE_STORAGE = require('./Integrations/FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(); let logger; // We need this here in order for the loopHealth function to work and be able to rescue the loop when it gets in trouble. let nextLoopTimeoutHandle; @@ -24,7 +23,6 @@ bot["botCache"] = new Map(); let processConfig; - let cloudStorage; let UI_COMMANDS; return thisObject; @@ -39,78 +37,43 @@ UI_COMMANDS = pUI_COMMANDS; processConfig = pProcessConfig; - cloudStorage = BLOB_STORAGE.newBlobStorage(bot, parentLogger); + let filePath; + if (global.RUN_AS_TEAM) { + filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo + "/" + pProcessConfig.name; // DevTeams bots only are run at the cloud. + } else { + filePath = global.DEV_TEAM + "/" + "members" + "/" + global.USER_LOGGED_IN + "/" + global.CURRENT_BOT_REPO + "/" + pProcessConfig.name; // DevTeam Members bots only are run at the browser. + } + filePath += "/User.Bot.js" - cloudStorage.initialize("AAPlatform", onInizialized); + fileStorage.getTextFile(global.DEV_TEAM, filePath, onBotDownloaded); - function onInizialized(err) { + function onBotDownloaded(err, text) { - if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] initialize -> onInizialized -> Entering function."); } + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - let filePath; - - switch (global.CURRENT_EXECUTION_AT) { // This is what determines if the bot is loaded from the devTeam or an endUser copy. - case "Node": { - if (global.RUN_AS_TEAM) { - filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo + "/" + pProcessConfig.name; // DevTeams bots only are run at the cloud. - } else { - filePath = global.DEV_TEAM + "/" + "members" + "/" + global.USER_LOGGED_IN + "/" + global.CURRENT_BOT_REPO + "/" + pProcessConfig.name; // DevTeam Members bots only are run at the browser. - } - - break; - } - case "IDE": { - filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo + "/" + pProcessConfig.name; // This works like Cloud except of a few things. - break; - } - default: { - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> CURRENT_EXECUTION_AT must be either 'Node' or 'IDE'."); - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> global.CURRENT_EXECUTION_AT = " + global.CURRENT_EXECUTION_AT); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - const CLOUD_REQUIRE = require(ROOT_DIR + 'CloudRequire'); - let cloudRequire = CLOUD_REQUIRE.newCloudRequire(bot, parentLogger); - - cloudRequire.downloadBot(cloudStorage, filePath, onBotDownloaded); - - function onBotDownloaded(err, pMODULE) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> err.message = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - USER_BOT_MODULE = pMODULE; - - filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; - - cloudRequire.downloadCommons(cloudStorage, filePath, onCommonsDownloaded); - - function onCommonsDownloaded(err, pMODULE) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> onCommonsDownloaded -> err.message = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - COMMONS_MODULE = pMODULE; - - callBackFunction(global.DEFAULT_OK_RESPONSE); - } - } - - } else { - parentLogger.write(MODULE_NAME, "[ERROR] Root -> start -> getBotConfig -> onInizialized -> err = " + err.message); + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> err.message = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + USER_BOT_MODULE = {} + USER_BOT_MODULE.newUserBot = eval(text); // TODO This needs to be changed function + + filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; + filePath += "/Commons.js" + + fileStorage.getTextFile(global.DEV_TEAM, filePath, onCommonsDownloaded); + + function onCommonsDownloaded(err, text) { + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + parentLogger.write(MODULE_NAME, "[WARN] initialize -> onBotDownloaded -> onCommonsDownloaded -> Commons not found: " + err.code || err.message); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + } + COMMONS_MODULE = {} + COMMONS_MODULE.newCommons = eval(text); // TODO This needs to be changed function + + callBackFunction(global.DEFAULT_OK_RESPONSE); } } } catch (err) { @@ -182,6 +145,9 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Live Mode detected."); } + if(process.env.NORMAL_WAIT_TIME !== undefined) + processConfig.normalWaitTime = process.env.NORMAL_WAIT_TIME + // This will be considered the process date and time, so as to have it consistenly all over the execution. let localDate = new Date(); @@ -207,6 +173,12 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Backtesting Mode detected."); } + if(process.env.NORMAL_WAIT_TIME !== undefined) { + processConfig.normalWaitTime = process.env.NORMAL_WAIT_TIME + } else { + processConfig.normalWaitTime = 1 + } + let timePeriod; if (UI_COMMANDS.timePeriod !== undefined) { @@ -265,20 +237,17 @@ } - if (global.CURRENT_EXECUTION_AT === "Node") { + if (bot.processDatetime.valueOf() > endDatetime.valueOf()) { - if (bot.processDatetime.valueOf() > endDatetime.valueOf()) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> End of Backtesting Period reached. Exiting Bot Process Loop."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> End of Backtesting Period reached. Exiting Bot Process Loop."); } - - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; } } break; @@ -287,6 +256,10 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Competition Mode detected."); } + if(process.env.NORMAL_WAIT_TIME !== undefined) { + processConfig.normalWaitTime = process.env.NORMAL_WAIT_TIME + } + let localDate = new Date(); bot.processDatetime = new Date(Date.UTC( @@ -304,33 +277,30 @@ } - if (global.CURRENT_EXECUTION_AT === "Node") { + let beginDatetime = new Date(bot.competition.beginDatetime); - let beginDatetime = new Date(bot.competition.beginDatetime); + if (bot.processDatetime.valueOf() < beginDatetime.valueOf()) { - if (bot.processDatetime.valueOf() < beginDatetime.valueOf()) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Competition not started yet. Wainting for the competition to start."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Competition not started yet. Wainting for the competition to start."); } + nextWaitTime = 'Normal'; + loopControl(nextWaitTime); + return; + } - nextWaitTime = 'Normal'; - loopControl(nextWaitTime); - return; - } + let endDatetime = new Date(bot.competition.endDatetime); - let endDatetime = new Date(bot.competition.endDatetime); + if (bot.processDatetime.valueOf() > endDatetime.valueOf()) { - if (bot.processDatetime.valueOf() > endDatetime.valueOf()) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> End of Competition Period reached. Exiting Bot Process Loop."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> End of Competition Period reached. Exiting Bot Process Loop."); } - - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; } break; @@ -377,7 +347,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeStatusDependencies -> Entering function."); } - statusDependencies = STATUS_DEPENDENCIES.newStatusDependencies(bot, logger, STATUS_REPORT, BLOB_STORAGE, UTILITIES); + statusDependencies = STATUS_DEPENDENCIES.newStatusDependencies(bot, logger, STATUS_REPORT, UTILITIES); statusDependencies.initialize(processConfig.statusDependencies, undefined, undefined, onInizialized); @@ -451,7 +421,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> Entering function."); } - dataDependencies = DATA_DEPENDENCIES.newDataDependencies(bot, logger, DATA_SET, BLOB_STORAGE, UTILITIES); + dataDependencies = DATA_DEPENDENCIES.newDataDependencies(bot, logger, DATA_SET); dataDependencies.initialize(processConfig.dataDependencies, onInizialized); @@ -525,7 +495,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeContext -> Entering function."); } - context = CONTEXT.newContext(bot, logger, BLOB_STORAGE, UTILITIES, STATUS_REPORT); + context = CONTEXT.newContext(bot, logger, UTILITIES); context.initialize(statusDependencies, pTotalAlgobots, onInizialized); function onInizialized(err) { @@ -968,24 +938,6 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> updateClonesModule -> Entering function."); } - if (global.CURRENT_EXECUTION_AT === "Node") { - - // Update clones module with latest context informations - - let lastExecution = context.newHistoryRecord - let date = lastExecution.date / 1000 | 0 - let buyAvgRate = lastExecution.buyAvgRate - let sellAvgRate = lastExecution.sellAvgRate - let marketRate = lastExecution.marketRate - let combinedProfitsA = lastExecution.combinedProfitsA - let combinedProfitsB = lastExecution.combinedProfitsB - - await operationsIntegration.updateExecutionResults(date, buyAvgRate, sellAvgRate, - marketRate, combinedProfitsA, combinedProfitsB) - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> updateClonesModule -> Clones Module Updated."); } - } - nextWaitTime = 'Normal'; loopControl(nextWaitTime); @@ -1012,7 +964,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> onStop -> Stopping the Loop Gracefully. See you next time!"); } - if(global.FULL_LOG === 'true' || bot.startMode === 'Competition'){ + if (global.FULL_LOG === 'true' || bot.startMode === 'Competition') { logger.persist(); } @@ -1036,7 +988,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Normal -> Restarting Loop in " + (processConfig.normalWaitTime / 1000) + " seconds."); } checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.normalWaitTime * 5, bot.loopCounter); nextLoopTimeoutHandle = setTimeout(loop, processConfig.normalWaitTime); - if(global.FULL_LOG === 'true' || bot.startMode === 'Competition'){ + if (global.FULL_LOG === 'true' || bot.startMode === 'Competition') { logger.persist(); } } @@ -1066,10 +1018,6 @@ function checkLoopHealth(pLastLoop) { - if (global.CURRENT_EXECUTION_AT !== "Node") { - return; - } - if (bot.startMode === 'Backtest') { return; } if (bot.enableCheckLoopHealth === false) { @@ -1108,64 +1056,16 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> shallWeStop -> Entering function. "); } - cloudStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + let stop = false + if (process.env.STOP_GRACEFULLY !== undefined) + stop = JSON.parse(process.env.STOP_GRACEFULLY) - cloudStorage.initialize("AAPlatform", onInizialized); - - function onInizialized(err) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> shallWeStop -> onInizialized -> Entering function."); } - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - let filePath = global.DEV_TEAM + "/" + "AACloud"; // DevTeams bots only are run at the cloud. - - let fileName = "this.config.json"; - - cloudStorage.getTextFile(filePath, fileName, onFileReceived); - - function onFileReceived(err, text) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - /* - If for any reason this config file cannot be read, we are not going to abort the loop for that. Instead we are going to assume - that there are no instructions to stop and we will keep the show running. - */ - - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> err.message = " + err.message); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> The config file cound not be read. "); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> We will ignore its content and continue running the loop. "); - - continueCallBack(); - return; - } - - try { - - let configRead = JSON.parse(text); - - if (configRead.stopGracefully === false && global.SHALL_BOT_STOP === false) { - continueCallBack(); - } else { - stopCallBack(); - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> onFileReceived -> err.message = " + err.message); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - } + if (!stop && global.SHALL_BOT_STOP === false) { + continueCallBack(); + } else { + stopCallBack(); } - } - catch (err) { + } catch (err) { logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> err.message = " + err.message); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index df072f8a5..8ac5cc350 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,7 +1,7 @@ version: '3' services: node: - image: superalgos/clone-executor:1.0.11 + image: superalgos/clone-executor:1.0.12 build: context: . dockerfile: ./Dockerfile diff --git a/package-lock.json b/package-lock.json index 401089f16..93b9e56c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,16 @@ { "name": "@superalgos/clone-executor", - "version": "1.0.11", + "version": "1.0.12", "lockfileVersion": 1, "requires": true, "dependencies": { "@superalgos/exchange-gateway": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/@superalgos/exchange-gateway/-/exchange-gateway-0.0.3.tgz", - "integrity": "sha512-Yi0kTUv3Bixh6wv/I+SsbGS47lLn5327tZtqr5I+n3Nm0yoUsbkkEEd/mdfiXYijdmd5e2ovEQYY48HoB/ik4Q==", + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@superalgos/exchange-gateway/-/exchange-gateway-0.0.4.tgz", + "integrity": "sha512-ebDiup8Op7zBNSoRv+wbfCgk9e8VOl9RCiYHFUq6p8AJ1Cu+zG2SOEXOT6eFG6ANq0HJASIl6oXi7CfdAekL6A==", "requires": { - "axios": "^0.18.0", - "dotenv": "^6.0.0", + "axios": "0.19.0", + "dotenv": "8.0.0", "lodash": "^4.17.11", "nonce": "^1.0.4", "querystring": "^0.2.0", @@ -30,14 +30,14 @@ } }, "ajv": { - "version": "5.5.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", - "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", + "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", "requires": { - "co": "^4.6.0", - "fast-deep-equal": "^1.0.0", + "fast-deep-equal": "^2.0.1", "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.3.0" + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" } }, "ansi-colors": { @@ -85,11 +85,11 @@ "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==" }, "async": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz", - "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz", + "integrity": "sha512-H1qVYh1MYhEEFLsP97cVKqCGo7KfCyTt6uEWqsTBr9SO84oK9Uwbyd/yCW+6rKJLHksBNUVWZDAjfS+Ccx0Bbg==", "requires": { - "lodash": "^4.17.10" + "lodash": "^4.17.11" } }, "asynckit": { @@ -108,37 +108,12 @@ "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, "axios": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", - "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.0.tgz", + "integrity": "sha512-1uvKqKQta3KBxIz14F2v06AEHZ/dIoeKfbTRkK1E5oqjDnuEerLmYTgJB5AiQZHJcljpg1TuRzdjDR06qNk0DQ==", "requires": { - "follow-redirects": "^1.3.0", - "is-buffer": "^1.1.5" - } - }, - "azure-storage": { - "version": "2.10.2", - "resolved": "https://registry.npmjs.org/azure-storage/-/azure-storage-2.10.2.tgz", - "integrity": "sha512-pOyGPya9+NDpAfm5YcFfklo57HfjDbYLXxs4lomPwvRxmb0Di/A+a+RkUmEFzaQ8S13CqxK40bRRB0sjj2ZQxA==", - "requires": { - "browserify-mime": "~1.2.9", - "extend": "^3.0.2", - "json-edm-parser": "0.1.2", - "md5.js": "1.3.4", - "readable-stream": "~2.0.0", - "request": "^2.86.0", - "underscore": "~1.8.3", - "uuid": "^3.0.0", - "validator": "~9.4.1", - "xml2js": "0.2.8", - "xmlbuilder": "^9.0.7" - }, - "dependencies": { - "underscore": { - "version": "1.8.3", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", - "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=" - } + "follow-redirects": "1.5.10", + "is-buffer": "^2.0.2" } }, "balanced-match": { @@ -150,21 +125,10 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "optional": true, "requires": { "tweetnacl": "^0.14.3" } }, - "block-stream2": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/block-stream2/-/block-stream2-1.1.0.tgz", - "integrity": "sha1-xzjjqRupd+u14f70MeE8oR2GOeI=", - "requires": { - "defined": "^1.0.0", - "inherits": "^2.0.1", - "readable-stream": "^2.0.4" - } - }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -179,16 +143,6 @@ "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==" }, - "browserify-mime": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/browserify-mime/-/browserify-mime-1.2.9.tgz", - "integrity": "sha1-rrGvKN5sDXpqLOQK22j/GEIq8x8=" - }, - "buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" - }, "camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", @@ -247,11 +201,6 @@ "wrap-ansi": "^2.0.0" } }, - "co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" - }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -299,18 +248,18 @@ "integrity": "sha512-mmGt/1pZqYRjMxB1axhTo16/snVZ5krrKkcmMeVKxzECMMXoCgnvTPp10QgHfcbQZw8Dq2jMNG6je4JlWU0gWg==" }, "colorspace": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.1.tgz", - "integrity": "sha512-pI3btWyiuz7Ken0BWh9Elzsmv2bM9AhA7psXib4anUXy/orfZ/E0MbQwhSOG/9L8hLlalqrU0UhOuqxW1YjmVw==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz", + "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==", "requires": { "color": "3.0.x", "text-hex": "1.0.x" } }, "combined-stream": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", - "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", "requires": { "delayed-stream": "~1.0.0" } @@ -325,51 +274,6 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, - "concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - }, - "dependencies": { - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, - "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" - }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - } - } - }, "cookiejar": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.2.tgz", @@ -436,11 +340,6 @@ "object-keys": "^1.0.12" } }, - "defined": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz", - "integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM=" - }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -462,15 +361,14 @@ "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==" }, "dotenv": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.2.0.tgz", - "integrity": "sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w==" + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.0.0.tgz", + "integrity": "sha512-30xVGqjLjiUOArT4+M5q9sYdvuR4riM6yK9wMcas9Vbp6zZa+ocC9dp6QoftuhTPhFAiLK/0C5Ni2nou/Bk8lg==" }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "optional": true, "requires": { "jsbn": "~0.1.0", "safer-buffer": "^2.1.0" @@ -525,11 +423,6 @@ "is-symbol": "^1.0.2" } }, - "es6-error": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-2.1.1.tgz", - "integrity": "sha1-kThDAexe0cmnJH0RKCRyFvA1R80=" - }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -565,9 +458,9 @@ "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, "fast-deep-equal": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", - "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" }, "fast-json-stable-stringify": { "version": "2.0.0", @@ -608,9 +501,9 @@ } }, "follow-redirects": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.1.tgz", - "integrity": "sha512-t2JCjbzxQpWvbhts3l6SH1DKzSrx8a+SsaVf4h6bG4kOXUuPYS/kg2Lr4gQSb7eemaHqJkOThF1BGyjlUkO1GQ==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", "requires": { "debug": "=3.1.0" } @@ -621,23 +514,13 @@ "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz", - "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", "requires": { "asynckit": "^0.4.0", - "combined-stream": "1.0.6", + "combined-stream": "^1.0.6", "mime-types": "^2.1.12" - }, - "dependencies": { - "combined-stream": { - "version": "1.0.6", - "resolved": "http://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz", - "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=", - "requires": { - "delayed-stream": "~1.0.0" - } - } } }, "formidable": { @@ -705,11 +588,11 @@ "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.0.tgz", - "integrity": "sha512-+qnmNjI4OfH2ipQ9VQOw23bBd/ibtfbVdK2fYbY4acTDqKTW/YDp9McimZdDbG8iV9fZizUqQMD5xvriB146TA==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", "requires": { - "ajv": "^5.3.0", + "ajv": "^6.5.5", "har-schema": "^2.0.0" } }, @@ -731,15 +614,6 @@ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=" }, - "hash-base": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.0.4.tgz", - "integrity": "sha1-X8hoaEfs1zSZQDMZprCj8/auSRg=", - "requires": { - "inherits": "^2.0.1", - "safe-buffer": "^5.0.1" - } - }, "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -780,9 +654,9 @@ "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, "is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz", + "integrity": "sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw==" }, "is-callable": { "version": "1.1.4", @@ -852,16 +726,7 @@ "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", - "optional": true - }, - "json-edm-parser": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/json-edm-parser/-/json-edm-parser-0.1.2.tgz", - "integrity": "sha1-HmCw/vG8CvZ7wNFG393lSGzWFbQ=", - "requires": { - "jsonparse": "~1.2.0" - } + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" }, "json-schema": { "version": "0.2.3", @@ -869,25 +734,15 @@ "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" }, "json-schema-traverse": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", - "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" - }, - "json-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-stream/-/json-stream-1.0.0.tgz", - "integrity": "sha1-GjhU4o0rvuqzHMfd9oPS3cVlJwg=" + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, - "jsonparse": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.2.0.tgz", - "integrity": "sha1-XAxWhRBxYOcv50ib3eoLRMK8Z70=" - }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -938,15 +793,15 @@ } }, "logform": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-1.10.0.tgz", - "integrity": "sha512-em5ojIhU18fIMOw/333mD+ZLE2fis0EzXl1ZwHx4iQzmpQi6odNiY/t+ITNr33JZhT9/KEaH+UPIipr6a9EjWg==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.1.2.tgz", + "integrity": "sha512-+lZh4OpERDBLqjiwDLpAWNQu6KMjnlXH2ByZwCuSqVPJletw0kTWJf5CgSNAUKn1KUkv3m2cUz/LK8zyEy7wzQ==", "requires": { "colors": "^1.2.1", "fast-safe-stringify": "^2.0.4", "fecha": "^2.3.3", "ms": "^2.1.1", - "triple-beam": "^1.2.0" + "triple-beam": "^1.3.0" }, "dependencies": { "ms": { @@ -964,15 +819,6 @@ "p-defer": "^1.0.0" } }, - "md5.js": { - "version": "1.3.4", - "resolved": "http://registry.npmjs.org/md5.js/-/md5.js-1.3.4.tgz", - "integrity": "sha1-6b296UogpawYsENA/Fdk1bCdkB0=", - "requires": { - "hash-base": "^3.0.0", - "inherits": "^2.0.1" - } - }, "mem": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/mem/-/mem-4.3.0.tgz", @@ -989,21 +835,21 @@ "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" }, "mime": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.2.tgz", - "integrity": "sha512-zJBfZDkwRu+j3Pdd2aHsR5GfH2jIWhmL1ZzBoc+X+3JEti2hbArWcyJ+1laC1D2/U/W1a/+Cegj0/OnEU2ybjg==" + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.3.tgz", + "integrity": "sha512-QgrPRJfE+riq5TPZMcHZOtm8c6K/yYrMbKIoRfapfiGLxS8OTeIfRhUGW5LU7MlRa52KOAGCfUNruqLrIBvWZw==" }, "mime-db": { - "version": "1.36.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.36.0.tgz", - "integrity": "sha512-L+xvyD9MkoYMXb1jAmzI/lWYAxAMCPvIBSWur0PZ5nOf5euahRLVqH//FKW9mWp2lkqUgYiXPgkzfMUFi4zVDw==" + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", + "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==" }, "mime-types": { - "version": "2.1.20", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.20.tgz", - "integrity": "sha512-HrkrPaP9vGuWbLK1B1FfgAkbqNjIuy4eHlIYnFi7kamZyLLrGlo2mpcx0bBmNpKqBtYtAfGbodDddIgddSJC2A==", + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", + "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", "requires": { - "mime-db": "~1.36.0" + "mime-db": "1.40.0" } }, "mimic-fn": { @@ -1024,47 +870,6 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, - "minio": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/minio/-/minio-7.0.5.tgz", - "integrity": "sha512-8ElURLlaiV1sGCE5fn9V7zhqYnvfUsntwg6lCR3OPzShak35dpCRKZFY3kdA7B1faTMGdN9jFMMIJ5jdWGEw4g==", - "requires": { - "async": "^1.5.2", - "block-stream2": "^1.0.0", - "concat-stream": "^1.4.8", - "es6-error": "^2.0.2", - "json-stream": "^1.0.0", - "lodash": "^4.14.2", - "mime-types": "^2.1.14", - "mkdirp": "^0.5.1", - "querystring": "0.2.0", - "through2": "^0.6.5", - "uuid": "^3.1.0", - "xml": "^1.0.0", - "xml2js": "^0.4.15" - }, - "dependencies": { - "async": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", - "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" - }, - "sax": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", - "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" - }, - "xml2js": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", - "requires": { - "sax": ">=0.6.0", - "xmlbuilder": "~9.0.1" - } - } - } - }, "mkdirp": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", @@ -1277,14 +1082,14 @@ "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, "process-nextick-args": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", - "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" }, "psl": { - "version": "1.1.29", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz", - "integrity": "sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ==" + "version": "1.1.32", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.32.tgz", + "integrity": "sha512-MHACAkHpihU/REGGPLj4sEfc/XKW2bheigvHO1dUqjaKigMp1C8+WLQYRGgeKFMsw5PMfegZcaN8IDXK/cD0+g==" }, "pump": { "version": "3.0.0", @@ -1296,9 +1101,9 @@ } }, "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "qs": { "version": "6.5.2", @@ -1311,16 +1116,20 @@ "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" }, "readable-stream": { - "version": "2.0.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", - "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", + "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==", "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "~1.0.0", - "process-nextick-args": "~1.0.6", - "string_decoder": "~0.10.x", - "util-deprecate": "~1.0.1" + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "dependencies": { + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + } } }, "request": { @@ -1375,15 +1184,10 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "sax": { - "version": "0.5.8", - "resolved": "http://registry.npmjs.org/sax/-/sax-0.5.8.tgz", - "integrity": "sha1-1HLbIo6zMcJQaw6MFVJK25OdEsE=" - }, "semver": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.0.0.tgz", - "integrity": "sha512-0UewU+9rFapKFnlbirLi3byoOuhrSsli/z/ihNnvM24vgF+8sNBiI1LZPBSH9wJKUwaUbw+s3hToDLCXkrghrQ==" + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz", + "integrity": "sha512-rWYq2e5iYW+fFe/oPPtYJxYgjBm8sC4rmoGdUOgBB7VnwKt6HrL793l2voH1UlsyYZpJ4g0wfjnTEO1s1NP2eQ==" }, "set-blocking": { "version": "2.0.0", @@ -1422,9 +1226,9 @@ "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, "sshpk": { - "version": "1.14.2", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.2.tgz", - "integrity": "sha1-xvxhZIo9nE52T9P8306hBeSSupg=", + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", "requires": { "asn1": "~0.2.3", "assert-plus": "^1.0.0", @@ -1452,9 +1256,12 @@ } }, "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.2.0.tgz", + "integrity": "sha512-6YqyX6ZWEYguAxgZzHGL7SsCeGx3V2TtOTqZz1xSTSWnqsbWwbptafNyvf/ACquZUXV3DANr5BDIwNYe1mN42w==", + "requires": { + "safe-buffer": "~5.1.0" + } }, "strip-ansi": { "version": "4.0.0", @@ -1499,21 +1306,6 @@ "ms": "^2.1.1" } }, - "form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - }, - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", @@ -1523,24 +1315,6 @@ "version": "6.7.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" - }, - "readable-stream": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.3.0.tgz", - "integrity": "sha512-EsI+s3k3XsW+fU8fQACLN59ky34AZ14LoeVZpYwmZvldCFo0r0gnelwF2TcMjLor/BTL5aDJVBMkss0dthToPw==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - }, - "string_decoder": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.2.0.tgz", - "integrity": "sha512-6YqyX6ZWEYguAxgZzHGL7SsCeGx3V2TtOTqZz1xSTSWnqsbWwbptafNyvf/ACquZUXV3DANr5BDIwNYe1mN42w==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -1557,33 +1331,6 @@ "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" }, - "through2": { - "version": "0.6.5", - "resolved": "https://registry.npmjs.org/through2/-/through2-0.6.5.tgz", - "integrity": "sha1-QaucZ7KdVyCQcUEOHXp6lozTrUg=", - "requires": { - "readable-stream": ">=1.0.33-1 <1.1.0-0", - "xtend": ">=4.0.0 <4.1.0-0" - }, - "dependencies": { - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "readable-stream": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", - "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "0.0.1", - "string_decoder": "~0.10.x" - } - } - } - }, "tough-cookie": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", @@ -1591,6 +1338,13 @@ "requires": { "psl": "^1.1.24", "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + } } }, "triple-beam": { @@ -1609,24 +1363,26 @@ "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", - "optional": true + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==" }, - "typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" - }, "underscore": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.9.1.tgz", "integrity": "sha512-5/4etnCkd9c8gwgowi5/om/mYO5ajCaOgdzj/oW+0eQV9WxKBDZw5+ycmKmeaTXjInS/W0BzpGLo2xR2aBwZdg==" }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "requires": { + "punycode": "^2.1.0" + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -1637,11 +1393,6 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" }, - "validator": { - "version": "9.4.1", - "resolved": "http://registry.npmjs.org/validator/-/validator-9.4.1.tgz", - "integrity": "sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA==" - }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", @@ -1674,53 +1425,19 @@ } }, "winston": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.1.0.tgz", - "integrity": "sha512-FsQfEE+8YIEeuZEYhHDk5cILo1HOcWkGwvoidLrDgPog0r4bser1lEIOco2dN9zpDJ1M88hfDgZvxe5z4xNcwg==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.2.1.tgz", + "integrity": "sha512-zU6vgnS9dAWCEKg/QYigd6cgMVVNwyTzKs81XZtTFuRwJOcDdBg7AU0mXVyNbs7O5RH2zdv+BdNZUlx7mXPuOw==", "requires": { - "async": "^2.6.0", + "async": "^2.6.1", "diagnostics": "^1.1.1", "is-stream": "^1.1.0", - "logform": "^1.9.1", + "logform": "^2.1.1", "one-time": "0.0.4", - "readable-stream": "^2.3.6", + "readable-stream": "^3.1.1", "stack-trace": "0.0.x", "triple-beam": "^1.3.0", - "winston-transport": "^4.2.0" - }, - "dependencies": { - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, - "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" - }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - } + "winston-transport": "^4.3.0" } }, "winston-transport": { @@ -1737,11 +1454,6 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, - "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" - }, "readable-stream": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", @@ -1813,29 +1525,6 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, - "xml": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", - "integrity": "sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=" - }, - "xml2js": { - "version": "0.2.8", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz", - "integrity": "sha1-m4FpCTFjH/CdGVdUn69U9PmAs8I=", - "requires": { - "sax": "0.5.x" - } - }, - "xmlbuilder": { - "version": "9.0.7", - "resolved": "http://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" - }, - "xtend": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" - }, "y18n": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", diff --git a/package.json b/package.json index 7586d63d6..6cf6f0627 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@superalgos/clone-executor", - "version": "1.0.11", + "version": "1.0.12", "description": "Module responsible for executing the bots on the cloud.", "author": "Superalgos, ltd.", "license": "Apache-2.0", @@ -13,12 +13,10 @@ "test": "echo \"Error: no test specified\" && exit 1" }, "dependencies": { - "@superalgos/exchange-gateway": "0.0.3", + "@superalgos/exchange-gateway": "0.0.4", "@superalgos/mqservice": "^0.0.14", - "axios": "^0.18.0", - "azure-storage": "^2.8.2", - "dotenv": "^6.0.0", - "minio": "^7.0.5", - "winston": "^3.1.0" + "axios": "0.19.0", + "dotenv": "8.0.0", + "winston": "^3.2.1" } } diff --git a/this.config.json b/this.config.json deleted file mode 100644 index fa18a5c76..000000000 --- a/this.config.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "codeName": "clone-executor", - "version": { - "major": 1, - "minor": 1, - "patch": 0 - }, - "stopGracefully": false, - "marketRateProvider": { - "devTeam": "AAMasters", - "bot": "AABruce", - "botVersion": { - "major": 1, - "minor": 0 - }, - "product": "Candles", - "dataSet": "One-Min", - "dataSetVersion": "dataSet.V1" - } -} diff --git a/utils/auth.js b/utils/auth.js deleted file mode 100644 index be0145a1f..000000000 --- a/utils/auth.js +++ /dev/null @@ -1,35 +0,0 @@ -const axios = require('axios') - -exports.authenticate = async function () { - try { - - if (!global.ACCESS_TOKEN) { - - const authBody = '{"client_id":"' + process.env.AUTH_CLIENT_ID - + '","client_secret": "' + process.env.AUTH_CLIENT_SECRET - + '","audience":"' + process.env.AUTH_AUDIENCE - + '","grant_type":"client_credentials"}' - - let authResponse = await axios({ - url: process.env.AUTH_URL, - method: 'post', - data: authBody, - headers: { 'content-type': 'application/json' } - }) - - if (!authResponse.data.errors) - global.ACCESS_TOKEN = authResponse.data.access_token - else throw new Error(operationsResponse.data.errors[0].message) - - process.env.AUTH_CLIENT_ID = "" - process.env.AUTH_CLIENT_SECRET = "" - process.env.AUTH_AUDIENCE = "" - process.env.AUTH_URL = "" - } - - return global.ACCESS_TOKEN - - } catch (error) { - throw new Error('There has been an error on the authentication: ' + error) - } -} \ No newline at end of file diff --git a/web.config.json b/web.config.json deleted file mode 100644 index 6575697d3..000000000 --- a/web.config.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "webModules": [ - { - "name": "Root.js" - }, - { - "name": "Assistant.js" - }, - { - "name": "BlobStorage.js" - }, - { - "name": "DebugLog.js" - }, - { - "name": "Context.js" - }, - { - "name": "DataDependencies.js" - }, - { - "name": "StatusDependencies.js" - }, - { - "name": "StatusReport.js" - }, - { - "name": "CloudUtilities.js" - }, - { - "name": "ExtractionBotProcessMainLoop.js" - }, - { - "name": "IndicatorBotProcessMainLoop.js" - }, - { - "name": "TradingBotProcessMainLoop.js" - }, - { - "name": "DataSet.js" - } - ] -} \ No newline at end of file