diff --git a/Context.js b/Context.js index c3e7a9b6a..6b0c32b0e 100644 --- a/Context.js +++ b/Context.js @@ -65,7 +65,7 @@ /* Storage account to be used here. */ - const FILE_STORAGE = require('./Integrations/FileStorage.js'); + const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(logger); let statusDependencies; diff --git a/DataSet.js b/DataSet.js index 351822890..17b5d8ee4 100644 --- a/DataSet.js +++ b/DataSet.js @@ -14,7 +14,7 @@ exports.newDataSet = function newDataSet(BOT, logger) { /* Storage account to be used here. */ - const FILE_STORAGE = require('./Integrations/FileStorage.js'); + const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(logger); return thisObject; diff --git a/DebugLog.js b/DebugLog.js index baa9fc203..616c1b4dd 100644 --- a/DebugLog.js +++ b/DebugLog.js @@ -63,7 +63,7 @@ exports.newDebugLog = function newDebugLog() { let contentToPersist = accumulatedLog; accumulatedLog = "["; - const FILE_STORAGE = require('./Integrations/FileStorage.js'); + const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(); let filePath = thisObject.bot.filePathRoot + "/Logs/" + thisObject.bot.process + "/" + executionDatetime; diff --git a/FileStorage.js b/FileStorage.js new file mode 100644 index 000000000..8857b87a4 --- /dev/null +++ b/FileStorage.js @@ -0,0 +1,183 @@ + +const path = require('path') +const fs = require('fs') + +exports.newFileStorage = function newFileStorage(logger) { + + const MODULE_NAME = 'FileStorage' + const MAX_RETRY = 30 + let currentRetryGetTextFile = 0 + let currentRetryWriteTextFile = 0 + + if (logger === undefined) { // Dummy logger + logger = {} + function write() { + + } + logger.write = write + } + + let thisObject = { + getTextFile: getTextFile, + createTextFile: createTextFile + } + + return thisObject + + function getTextFile(container, filePath, callBackFunction) { + + logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> Entering Function.') + + container = container.toLowerCase() + + /* Choose path for either bots or data */ + let fileLocation + if (filePath.indexOf("/bots/") > 0) { + fileLocation = process.env.BOTS_PATH + '/' + container + '/' + filePath + } else { + fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath + } + + try { + + logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> fileLocation: ' + fileLocation) + + /* Here we actually write the file. */ + fs.readFile(fileLocation, onFileRead) + + function onFileRead(err, text) { + + if (err) { + + if (err.code === 'ENOENT') { + let customResponse = { + result: global.CUSTOM_FAIL_RESPONSE.result, + message: 'File does not exist.' + } + callBackFunction(customResponse) + return + } + + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> err = ' + err.stack) + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> container = ' + container) + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> filePath = ' + filePath) + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> err = ' + err.stack) + setTimeout(retry, 100) + return + } + + if (text.toString() === "") { + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Read and Empty File -> filePath = ' + filePath) + setTimeout(retry, 100) + return + } + + callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString()) + } + + } catch (err) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> Error writing file -> file = ' + fileLocation) + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> Error writing file -> err = ' + err.stack) + retry() + } + + function retry() { + if (currentRetryGetTextFile < MAX_RETRY) { + currentRetryGetTextFile++ + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> retry -> Will try to read the file again -> Retry #: ' + currentRetryGetTextFile) + getTextFile(container, filePath, callBackFunction) + } else { + currentRetryGetTextFile = 0 + logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> Max retries reached reading a file. Giving up.') + logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> file = ' + fileLocation) + callBackFunction(global.DEFAULT_FAIL_RESPONSE) + } + } + } + + function createTextFile(container, filePath, fileContent, callBackFunction) { + + logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> Entering Function.') + + container = container.toLowerCase() + + /* Choose path for either logs or data */ + let fileLocation + if (filePath.indexOf("/Logs/") > 0) { + fileLocation = process.env.LOG_PATH + '/' + container + '/' + filePath + } else { + fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath + } + + try { + + logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> fileLocation: ' + fileLocation) + + /* If necesary a folder or folders are created before writing the file to disk. */ + let directoryPath = fileLocation.substring(0, fileLocation.lastIndexOf('/') + 1); + mkDirByPathSync(directoryPath) + + /* Here we actually write the file. */ + fs.writeFile(fileLocation, fileContent, onFileWriten) + + function onFileWriten(err) { + if (err) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> file = ' + fileLocation) + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> err = ' + err.stack) + setTimeout(retry, 100) + } else { + callBackFunction(global.DEFAULT_OK_RESPONSE) + } + } + + } catch (err) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> Error writing file -> file = ' + fileLocation) + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> Error writing file -> err = ' + err.stack) + retry() + } + + function retry() { + if (currentRetryWriteTextFile < MAX_RETRY) { + currentRetryWriteTextFile++ + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> retry -> Will try to write the file again -> Retry #: ' + currentRetryWriteTextFile) + createTextFile(container, filePath, fileContent, callBackFunction) + } else { + currentRetryWriteTextFile = 0 + logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> retry -> Max retries reached writting a file. Giving up.') + logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> retry -> file = ' + fileLocation) + callBackFunction(global.DEFAULT_FAIL_RESPONSE) + } + } + } + + /* Function to create folders of missing folders at any path. */ + + function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) { + const sep = '/'; + const initDir = path.isAbsolute(targetDir) ? sep : ''; + const baseDir = isRelativeToScript ? __dirname : '.'; + + return targetDir.split(sep).reduce((parentDir, childDir) => { + const curDir = path.resolve(baseDir, parentDir, childDir); + try { + fs.mkdirSync(curDir); + } catch (err) { + if (err.code === 'EEXIST') { // curDir already exists! + return curDir; + } + + // To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows. + if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure. + throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`); + } + + const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1; + if (!caughtErr || caughtErr && curDir === path.resolve(targetDir)) { + throw err; // Throw if it's just the last created dir. + } + } + + return curDir; + }, initDir); + } +} diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 30f83007b..b0bb3508c 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -10,7 +10,7 @@ const MULTI_PERIOD_MARKET = require(ROOT_DIR + 'MultiPeriodMarket'); const MULTI_PERIOD_DAILY = require(ROOT_DIR + 'MultiPeriodDaily'); - const FILE_STORAGE = require('./Integrations/FileStorage.js'); + const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(parentLogger); const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); diff --git a/Integrations/FileStorage.js b/Integrations/FileStorage.js deleted file mode 100644 index 481f10264..000000000 --- a/Integrations/FileStorage.js +++ /dev/null @@ -1,224 +0,0 @@ -const axios = require('axios') -const Ecosystem = require('./Ecosystem') -let fs = require('fs') -const { promisify } = require('util') -const readFileAsync = promisify(fs.readFile) -const writeFileAsync = promisify(fs.writeFile) -const path = require('path') - -exports.newFileStorage = function newFileStorage(logger) { - const MODULE_NAME = 'FileStorage' - const MAX_RETRY = 30 - let currentRetryGetTextFile = 0 - let currentRetryWriteTextFile = 0 - - const recoverableErrors = [ - 'SOCKETTIMEDOUT', - 'TIMEDOUT', - 'CONNRESET', - 'CONNREFUSED', - 'NOTFOUND', - 'ENOTFOUND', - 'ECONNREFUSED', - 'CONNREFUSED', - 'NOTFOUND', - 'ESOCKETTIMEDOUT', - 'ECONNRESET', - 'ETIMEDOUT', - 'EAI_AGAIN' - ] - - if (logger === undefined) { - logger = {} - function write() { - - } - logger.write = write - } - - return { - getTextFile, - createTextFile - } - - function logInfo(message) { - log('[INFO] ' + message) - } - function log(message) { - if (logger) { - logger.write(MODULE_NAME, message); - } else { - //console.log("['" + new Date().toISOString() + "', 0,'" + MODULE_NAME + "','" + message + "']") - } - } - - function logError(message) { - log('[ERROR] ' + message) - } - - async function getTextFile(container, filePath, callBackFunction) { - - try { - let host = await getDevTeamHost(container) - - if (host.url.indexOf('localhost') !== -1) { - - let fileLocation - - if (filePath.indexOf("/bots/") > 0) { - fileLocation = process.env.BOTS_PATH + '/' + container + '/' + filePath - } else { - fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath - } - - logInfo('getTextFile: ' + fileLocation) - //console.log('getTextFile: ' + fileLocation) - - let fileContent = await readFileAsync(fileLocation) - callBackFunction(global.DEFAULT_OK_RESPONSE, fileContent.toString()) - } else { - let response = await axios({ - url: host.url + 'graphql', - method: 'post', - data: { - query: ` - query web_FileContent($file: web_FileInput){ - web_FileContent(file: $file) - } - `, - variables: { - file: { - container: container.toLowerCase(), - filePath, - storage: host.storage, - accessKey: host.accessKey - } - } - } - }) - - if (response.data.errors) { - callBackFunction({ code: response.data.errors[0] }) - return - } - - if (response.data.data.web_FileContent) { - callBackFunction(global.DEFAULT_OK_RESPONSE, response.data.data.web_FileContent) - } else { - callBackFunction({ code: 'The specified key does not exist.' }) - } - } - } catch (err) { - if (verifyRetry(err.code) && currentRetryGetTextFile < MAX_RETRY) { - currentRetryGetTextFile++ - logInfo('getTextFile -> Retrying connection to the server because received error: ' + err.code + '. Retry #: ' + currentRetryGetTextFile) - getTextFile(container, filePath, callBackFunction) - } else if (err.code === 'ENOENT') { - callBackFunction({ code: 'The specified key does not exist.' }) - } else { - currentRetryGetTextFile = 0 - logError('getTextFile -> error = ' + err.message) - callBackFunction(global.DEFAULT_FAIL_RESPONSE) - } - } - } - - function createTextFile(container, filePath, fileContent, callBackFunction) { - - /* Choose path for either logs or data */ - let fileLocation - if (filePath.indexOf("/Logs/") > 0) { - fileLocation = process.env.LOG_PATH + '/' + container + '/' + filePath - } else { - fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath - } - - try { - - logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> fileLocation: ' + fileLocation) - - /* If necesary a folder or folders are created before writing the file to disk. */ - let directoryPath = fileLocation.substring(0, fileLocation.lastIndexOf('/') + 1); - mkDirByPathSync(directoryPath) - - /* Here we actually write the file. */ - fs.writeFile(fileLocation, fileContent, onFileWriten) - - function onFileWriten(err) { - if (err) { - logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> file = ' + fileLocation) - logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> err = ' + err.stack) - retry() - } else { - callBackFunction(global.DEFAULT_OK_RESPONSE) - } - } - - } catch (err) { - logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> Error writing file -> file = ' + fileLocation) - logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> Error writing file -> err = ' + err.stack) - retry() - } - - function retry() { - if (currentRetryWriteTextFile < MAX_RETRY) { - currentRetryWriteTextFile++ - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> retry -> Will try to write the file again -> Retry #: ' + currentRetryWriteTextFile) - createTextFile(container, filePath, fileContent, callBackFunction) - } else { - currentRetryWriteTextFile = 0 - logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> retry -> Max retries reached. Giving up.') - callBackFunction(global.DEFAULT_FAIL_RESPONSE) - } - } - } - - async function getDevTeamHost(devTeamName) { - let ecosystem = await Ecosystem.getEcosystem() - - for (var i = 0; i < ecosystem.devTeams.length; i++) { - if (ecosystem.devTeams[i].codeName.toLowerCase() === devTeamName.toLowerCase()) { - return ecosystem.devTeams[i].host - } - } - } - - function verifyRetry(errorCode) { - for (let i = 0; i < recoverableErrors.length; i++) { - const error = recoverableErrors[i]; - if (error === errorCode) { - return true - } - } - return false - } - - function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) { - const sep = '/'; - const initDir = path.isAbsolute(targetDir) ? sep : ''; - const baseDir = isRelativeToScript ? __dirname : '.'; - - return targetDir.split(sep).reduce((parentDir, childDir) => { - const curDir = path.resolve(baseDir, parentDir, childDir); - try { - fs.mkdirSync(curDir); - } catch (err) { - if (err.code === 'EEXIST') { // curDir already exists! - return curDir; - } - - // To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows. - if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure. - throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`); - } - - const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1; - if (!caughtErr || caughtErr && curDir === path.resolve(targetDir)) { - throw err; // Throw if it's just the last created dir. - } - } - - return curDir; - }, initDir); - } -} diff --git a/MultiPeriod.js b/MultiPeriod.js index 6a6d44f2e..1c10bad65 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -22,7 +22,7 @@ let usertBot; - const FILE_STORAGE = require('./Integrations/FileStorage.js'); + const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(logger); let processConfig; diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index cbe350560..7b4d9f14d 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -21,7 +21,7 @@ let usertBot; - const FILE_STORAGE = require('./Integrations/FileStorage.js'); + const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(logger); let processConfig; diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index d3dfe3ac1..99870774e 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -19,7 +19,7 @@ let usertBot; - const FILE_STORAGE = require('./Integrations/FileStorage.js'); + const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(logger); let processConfig; diff --git a/Project.njsproj b/Project.njsproj index 50156dc5f..207b21485 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -26,7 +26,7 @@ Code - + Code diff --git a/Root.js b/Root.js index 0a06b1132..49e504b88 100644 --- a/Root.js +++ b/Root.js @@ -134,7 +134,7 @@ try { console.log(logDisplace + "Root : [INFO] start -> getBotConfig -> Entering function. "); - const FILE_STORAGE = require('./Integrations/FileStorage.js'); + const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(); let filePath = global.TASK_NODE.bot.processes[processIndex].code.team + '/bots/' + global.TASK_NODE.bot.processes[processIndex].code.repo + '/this.bot.config.json'; diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index 4fd528f22..a839a0d3e 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -8,7 +8,7 @@ let USER_BOT_MODULE; let COMMONS_MODULE; - const FILE_STORAGE = require('./Integrations/FileStorage.js'); + const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(parentLogger); const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); diff --git a/StatusReport.js b/StatusReport.js index 7f6fcc120..fee0a71da 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -29,7 +29,7 @@ /* Storage account to be used here. */ - const FILE_STORAGE = require('./Integrations/FileStorage.js'); + const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(logger); let month; diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index da2c27630..13c751550 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -11,7 +11,7 @@ const MULTI_PERIOD_MARKET = require(ROOT_DIR + 'MultiPeriodMarket'); const MULTI_PERIOD_DAILY = require(ROOT_DIR + 'MultiPeriodDaily'); const MULTI_PERIOD = require(ROOT_DIR + 'MultiPeriod'); - const FILE_STORAGE = require('./Integrations/FileStorage.js'); + const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(parentLogger); @@ -277,10 +277,10 @@ /* Checking if we should process this loop or not.*/ if (global.STOP_PROCESSING === true) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> We are going to skip this Loop bacause we were requested to stop or never asked to start."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for Backtesting or Live Trade to be started."); } console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) - + " We are going to skip this Loop bacause we were requested to stop or never asked to start. "); + + " Waiting for Backtesting or Live Trade to be started. "); global.SYSTEM_EVENT_HANDLER.raiseEvent('Jason-Multi-Period', 'Process Stopped') diff --git a/server.js b/server.js index 1c4b6daf6..92853153d 100644 --- a/server.js +++ b/server.js @@ -72,6 +72,7 @@ if (global.TASK_NODE !== undefined) { } else { // I use this section to debug in standalone mode. let argument = ' {"type":"Task","name":"Runs Backtests, Fordwardtests & Live Trades ","bot":{"type":"Trading Engine","processes":[{"type":"Process","name":"Multi Period","code":{"team":"AAMasters","bot":"AAJason","process":"Multi-Period","repo":"AAJason-Trading-Engine-Bot"},"id":"4748c8c4-4d19-4076-96b2-e9c06524fbb3"}]},"id":"561bac18-fc78-464a-90c5-79fd821fc633"}' + try { global.TASK_NODE = JSON.parse(argument) } catch (err) {