diff --git a/Context.js b/Context.js
index c3e7a9b6a..6b0c32b0e 100644
--- a/Context.js
+++ b/Context.js
@@ -65,7 +65,7 @@
/* Storage account to be used here. */
- const FILE_STORAGE = require('./Integrations/FileStorage.js');
+ const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let statusDependencies;
diff --git a/DataSet.js b/DataSet.js
index 351822890..17b5d8ee4 100644
--- a/DataSet.js
+++ b/DataSet.js
@@ -14,7 +14,7 @@ exports.newDataSet = function newDataSet(BOT, logger) {
/* Storage account to be used here. */
- const FILE_STORAGE = require('./Integrations/FileStorage.js');
+ const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
return thisObject;
diff --git a/DebugLog.js b/DebugLog.js
index baa9fc203..616c1b4dd 100644
--- a/DebugLog.js
+++ b/DebugLog.js
@@ -63,7 +63,7 @@ exports.newDebugLog = function newDebugLog() {
let contentToPersist = accumulatedLog;
accumulatedLog = "[";
- const FILE_STORAGE = require('./Integrations/FileStorage.js');
+ const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage();
let filePath = thisObject.bot.filePathRoot + "/Logs/" + thisObject.bot.process + "/" + executionDatetime;
diff --git a/FileStorage.js b/FileStorage.js
new file mode 100644
index 000000000..8857b87a4
--- /dev/null
+++ b/FileStorage.js
@@ -0,0 +1,183 @@
+
+const path = require('path')
+const fs = require('fs')
+
+exports.newFileStorage = function newFileStorage(logger) {
+
+ const MODULE_NAME = 'FileStorage'
+ const MAX_RETRY = 30
+ let currentRetryGetTextFile = 0
+ let currentRetryWriteTextFile = 0
+
+ if (logger === undefined) { // Dummy logger
+ logger = {}
+ function write() {
+
+ }
+ logger.write = write
+ }
+
+ let thisObject = {
+ getTextFile: getTextFile,
+ createTextFile: createTextFile
+ }
+
+ return thisObject
+
+ function getTextFile(container, filePath, callBackFunction) {
+
+ logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> Entering Function.')
+
+ container = container.toLowerCase()
+
+ /* Choose path for either bots or data */
+ let fileLocation
+ if (filePath.indexOf("/bots/") > 0) {
+ fileLocation = process.env.BOTS_PATH + '/' + container + '/' + filePath
+ } else {
+ fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath
+ }
+
+ try {
+
+ logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> fileLocation: ' + fileLocation)
+
+ /* Here we actually write the file. */
+ fs.readFile(fileLocation, onFileRead)
+
+ function onFileRead(err, text) {
+
+ if (err) {
+
+ if (err.code === 'ENOENT') {
+ let customResponse = {
+ result: global.CUSTOM_FAIL_RESPONSE.result,
+ message: 'File does not exist.'
+ }
+ callBackFunction(customResponse)
+ return
+ }
+
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> err = ' + err.stack)
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> container = ' + container)
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> filePath = ' + filePath)
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> err = ' + err.stack)
+ setTimeout(retry, 100)
+ return
+ }
+
+ if (text.toString() === "") {
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Read and Empty File -> filePath = ' + filePath)
+ setTimeout(retry, 100)
+ return
+ }
+
+ callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString())
+ }
+
+ } catch (err) {
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> Error writing file -> file = ' + fileLocation)
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> Error writing file -> err = ' + err.stack)
+ retry()
+ }
+
+ function retry() {
+ if (currentRetryGetTextFile < MAX_RETRY) {
+ currentRetryGetTextFile++
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> retry -> Will try to read the file again -> Retry #: ' + currentRetryGetTextFile)
+ getTextFile(container, filePath, callBackFunction)
+ } else {
+ currentRetryGetTextFile = 0
+ logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> Max retries reached reading a file. Giving up.')
+ logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> file = ' + fileLocation)
+ callBackFunction(global.DEFAULT_FAIL_RESPONSE)
+ }
+ }
+ }
+
+ function createTextFile(container, filePath, fileContent, callBackFunction) {
+
+ logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> Entering Function.')
+
+ container = container.toLowerCase()
+
+ /* Choose path for either logs or data */
+ let fileLocation
+ if (filePath.indexOf("/Logs/") > 0) {
+ fileLocation = process.env.LOG_PATH + '/' + container + '/' + filePath
+ } else {
+ fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath
+ }
+
+ try {
+
+ logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> fileLocation: ' + fileLocation)
+
+ /* If necesary a folder or folders are created before writing the file to disk. */
+ let directoryPath = fileLocation.substring(0, fileLocation.lastIndexOf('/') + 1);
+ mkDirByPathSync(directoryPath)
+
+ /* Here we actually write the file. */
+ fs.writeFile(fileLocation, fileContent, onFileWriten)
+
+ function onFileWriten(err) {
+ if (err) {
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> file = ' + fileLocation)
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> err = ' + err.stack)
+ setTimeout(retry, 100)
+ } else {
+ callBackFunction(global.DEFAULT_OK_RESPONSE)
+ }
+ }
+
+ } catch (err) {
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> Error writing file -> file = ' + fileLocation)
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> Error writing file -> err = ' + err.stack)
+ retry()
+ }
+
+ function retry() {
+ if (currentRetryWriteTextFile < MAX_RETRY) {
+ currentRetryWriteTextFile++
+ logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> retry -> Will try to write the file again -> Retry #: ' + currentRetryWriteTextFile)
+ createTextFile(container, filePath, fileContent, callBackFunction)
+ } else {
+ currentRetryWriteTextFile = 0
+ logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> retry -> Max retries reached writting a file. Giving up.')
+ logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> retry -> file = ' + fileLocation)
+ callBackFunction(global.DEFAULT_FAIL_RESPONSE)
+ }
+ }
+ }
+
+ /* Function to create folders of missing folders at any path. */
+
+ function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
+ const sep = '/';
+ const initDir = path.isAbsolute(targetDir) ? sep : '';
+ const baseDir = isRelativeToScript ? __dirname : '.';
+
+ return targetDir.split(sep).reduce((parentDir, childDir) => {
+ const curDir = path.resolve(baseDir, parentDir, childDir);
+ try {
+ fs.mkdirSync(curDir);
+ } catch (err) {
+ if (err.code === 'EEXIST') { // curDir already exists!
+ return curDir;
+ }
+
+ // To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows.
+ if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure.
+ throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`);
+ }
+
+ const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1;
+ if (!caughtErr || caughtErr && curDir === path.resolve(targetDir)) {
+ throw err; // Throw if it's just the last created dir.
+ }
+ }
+
+ return curDir;
+ }, initDir);
+ }
+}
diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js
index 30f83007b..b0bb3508c 100644
--- a/IndicatorBotProcessMainLoop.js
+++ b/IndicatorBotProcessMainLoop.js
@@ -10,7 +10,7 @@
const MULTI_PERIOD_MARKET = require(ROOT_DIR + 'MultiPeriodMarket');
const MULTI_PERIOD_DAILY = require(ROOT_DIR + 'MultiPeriodDaily');
- const FILE_STORAGE = require('./Integrations/FileStorage.js');
+ const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(parentLogger);
const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog');
diff --git a/Integrations/FileStorage.js b/Integrations/FileStorage.js
deleted file mode 100644
index 481f10264..000000000
--- a/Integrations/FileStorage.js
+++ /dev/null
@@ -1,224 +0,0 @@
-const axios = require('axios')
-const Ecosystem = require('./Ecosystem')
-let fs = require('fs')
-const { promisify } = require('util')
-const readFileAsync = promisify(fs.readFile)
-const writeFileAsync = promisify(fs.writeFile)
-const path = require('path')
-
-exports.newFileStorage = function newFileStorage(logger) {
- const MODULE_NAME = 'FileStorage'
- const MAX_RETRY = 30
- let currentRetryGetTextFile = 0
- let currentRetryWriteTextFile = 0
-
- const recoverableErrors = [
- 'SOCKETTIMEDOUT',
- 'TIMEDOUT',
- 'CONNRESET',
- 'CONNREFUSED',
- 'NOTFOUND',
- 'ENOTFOUND',
- 'ECONNREFUSED',
- 'CONNREFUSED',
- 'NOTFOUND',
- 'ESOCKETTIMEDOUT',
- 'ECONNRESET',
- 'ETIMEDOUT',
- 'EAI_AGAIN'
- ]
-
- if (logger === undefined) {
- logger = {}
- function write() {
-
- }
- logger.write = write
- }
-
- return {
- getTextFile,
- createTextFile
- }
-
- function logInfo(message) {
- log('[INFO] ' + message)
- }
- function log(message) {
- if (logger) {
- logger.write(MODULE_NAME, message);
- } else {
- //console.log("['" + new Date().toISOString() + "', 0,'" + MODULE_NAME + "','" + message + "']")
- }
- }
-
- function logError(message) {
- log('[ERROR] ' + message)
- }
-
- async function getTextFile(container, filePath, callBackFunction) {
-
- try {
- let host = await getDevTeamHost(container)
-
- if (host.url.indexOf('localhost') !== -1) {
-
- let fileLocation
-
- if (filePath.indexOf("/bots/") > 0) {
- fileLocation = process.env.BOTS_PATH + '/' + container + '/' + filePath
- } else {
- fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath
- }
-
- logInfo('getTextFile: ' + fileLocation)
- //console.log('getTextFile: ' + fileLocation)
-
- let fileContent = await readFileAsync(fileLocation)
- callBackFunction(global.DEFAULT_OK_RESPONSE, fileContent.toString())
- } else {
- let response = await axios({
- url: host.url + 'graphql',
- method: 'post',
- data: {
- query: `
- query web_FileContent($file: web_FileInput){
- web_FileContent(file: $file)
- }
- `,
- variables: {
- file: {
- container: container.toLowerCase(),
- filePath,
- storage: host.storage,
- accessKey: host.accessKey
- }
- }
- }
- })
-
- if (response.data.errors) {
- callBackFunction({ code: response.data.errors[0] })
- return
- }
-
- if (response.data.data.web_FileContent) {
- callBackFunction(global.DEFAULT_OK_RESPONSE, response.data.data.web_FileContent)
- } else {
- callBackFunction({ code: 'The specified key does not exist.' })
- }
- }
- } catch (err) {
- if (verifyRetry(err.code) && currentRetryGetTextFile < MAX_RETRY) {
- currentRetryGetTextFile++
- logInfo('getTextFile -> Retrying connection to the server because received error: ' + err.code + '. Retry #: ' + currentRetryGetTextFile)
- getTextFile(container, filePath, callBackFunction)
- } else if (err.code === 'ENOENT') {
- callBackFunction({ code: 'The specified key does not exist.' })
- } else {
- currentRetryGetTextFile = 0
- logError('getTextFile -> error = ' + err.message)
- callBackFunction(global.DEFAULT_FAIL_RESPONSE)
- }
- }
- }
-
- function createTextFile(container, filePath, fileContent, callBackFunction) {
-
- /* Choose path for either logs or data */
- let fileLocation
- if (filePath.indexOf("/Logs/") > 0) {
- fileLocation = process.env.LOG_PATH + '/' + container + '/' + filePath
- } else {
- fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath
- }
-
- try {
-
- logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> fileLocation: ' + fileLocation)
-
- /* If necesary a folder or folders are created before writing the file to disk. */
- let directoryPath = fileLocation.substring(0, fileLocation.lastIndexOf('/') + 1);
- mkDirByPathSync(directoryPath)
-
- /* Here we actually write the file. */
- fs.writeFile(fileLocation, fileContent, onFileWriten)
-
- function onFileWriten(err) {
- if (err) {
- logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> file = ' + fileLocation)
- logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> err = ' + err.stack)
- retry()
- } else {
- callBackFunction(global.DEFAULT_OK_RESPONSE)
- }
- }
-
- } catch (err) {
- logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> Error writing file -> file = ' + fileLocation)
- logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> Error writing file -> err = ' + err.stack)
- retry()
- }
-
- function retry() {
- if (currentRetryWriteTextFile < MAX_RETRY) {
- currentRetryWriteTextFile++
- logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> retry -> Will try to write the file again -> Retry #: ' + currentRetryWriteTextFile)
- createTextFile(container, filePath, fileContent, callBackFunction)
- } else {
- currentRetryWriteTextFile = 0
- logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> retry -> Max retries reached. Giving up.')
- callBackFunction(global.DEFAULT_FAIL_RESPONSE)
- }
- }
- }
-
- async function getDevTeamHost(devTeamName) {
- let ecosystem = await Ecosystem.getEcosystem()
-
- for (var i = 0; i < ecosystem.devTeams.length; i++) {
- if (ecosystem.devTeams[i].codeName.toLowerCase() === devTeamName.toLowerCase()) {
- return ecosystem.devTeams[i].host
- }
- }
- }
-
- function verifyRetry(errorCode) {
- for (let i = 0; i < recoverableErrors.length; i++) {
- const error = recoverableErrors[i];
- if (error === errorCode) {
- return true
- }
- }
- return false
- }
-
- function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
- const sep = '/';
- const initDir = path.isAbsolute(targetDir) ? sep : '';
- const baseDir = isRelativeToScript ? __dirname : '.';
-
- return targetDir.split(sep).reduce((parentDir, childDir) => {
- const curDir = path.resolve(baseDir, parentDir, childDir);
- try {
- fs.mkdirSync(curDir);
- } catch (err) {
- if (err.code === 'EEXIST') { // curDir already exists!
- return curDir;
- }
-
- // To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows.
- if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure.
- throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`);
- }
-
- const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1;
- if (!caughtErr || caughtErr && curDir === path.resolve(targetDir)) {
- throw err; // Throw if it's just the last created dir.
- }
- }
-
- return curDir;
- }, initDir);
- }
-}
diff --git a/MultiPeriod.js b/MultiPeriod.js
index 6a6d44f2e..1c10bad65 100644
--- a/MultiPeriod.js
+++ b/MultiPeriod.js
@@ -22,7 +22,7 @@
let usertBot;
- const FILE_STORAGE = require('./Integrations/FileStorage.js');
+ const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let processConfig;
diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js
index cbe350560..7b4d9f14d 100644
--- a/MultiPeriodDaily.js
+++ b/MultiPeriodDaily.js
@@ -21,7 +21,7 @@
let usertBot;
- const FILE_STORAGE = require('./Integrations/FileStorage.js');
+ const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let processConfig;
diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js
index d3dfe3ac1..99870774e 100644
--- a/MultiPeriodMarket.js
+++ b/MultiPeriodMarket.js
@@ -19,7 +19,7 @@
let usertBot;
- const FILE_STORAGE = require('./Integrations/FileStorage.js');
+ const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let processConfig;
diff --git a/Project.njsproj b/Project.njsproj
index 50156dc5f..207b21485 100644
--- a/Project.njsproj
+++ b/Project.njsproj
@@ -26,7 +26,7 @@
Code
-
+
Code
diff --git a/Root.js b/Root.js
index 0a06b1132..49e504b88 100644
--- a/Root.js
+++ b/Root.js
@@ -134,7 +134,7 @@
try {
console.log(logDisplace + "Root : [INFO] start -> getBotConfig -> Entering function. ");
- const FILE_STORAGE = require('./Integrations/FileStorage.js');
+ const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage();
let filePath = global.TASK_NODE.bot.processes[processIndex].code.team + '/bots/' + global.TASK_NODE.bot.processes[processIndex].code.repo + '/this.bot.config.json';
diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js
index 4fd528f22..a839a0d3e 100644
--- a/SensorBotProcessMainLoop.js
+++ b/SensorBotProcessMainLoop.js
@@ -8,7 +8,7 @@
let USER_BOT_MODULE;
let COMMONS_MODULE;
- const FILE_STORAGE = require('./Integrations/FileStorage.js');
+ const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(parentLogger);
const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog');
diff --git a/StatusReport.js b/StatusReport.js
index 7f6fcc120..fee0a71da 100644
--- a/StatusReport.js
+++ b/StatusReport.js
@@ -29,7 +29,7 @@
/* Storage account to be used here. */
- const FILE_STORAGE = require('./Integrations/FileStorage.js');
+ const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let month;
diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js
index da2c27630..13c751550 100644
--- a/TradingEngineProcessMainLoop.js
+++ b/TradingEngineProcessMainLoop.js
@@ -11,7 +11,7 @@
const MULTI_PERIOD_MARKET = require(ROOT_DIR + 'MultiPeriodMarket');
const MULTI_PERIOD_DAILY = require(ROOT_DIR + 'MultiPeriodDaily');
const MULTI_PERIOD = require(ROOT_DIR + 'MultiPeriod');
- const FILE_STORAGE = require('./Integrations/FileStorage.js');
+ const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(parentLogger);
@@ -277,10 +277,10 @@
/* Checking if we should process this loop or not.*/
if (global.STOP_PROCESSING === true) {
- if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> We are going to skip this Loop bacause we were requested to stop or never asked to start."); }
+ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for Backtesting or Live Trade to be started."); }
console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30)
- + " We are going to skip this Loop bacause we were requested to stop or never asked to start. ");
+ + " Waiting for Backtesting or Live Trade to be started. ");
global.SYSTEM_EVENT_HANDLER.raiseEvent('Jason-Multi-Period', 'Process Stopped')
diff --git a/server.js b/server.js
index 1c4b6daf6..92853153d 100644
--- a/server.js
+++ b/server.js
@@ -72,6 +72,7 @@ if (global.TASK_NODE !== undefined) {
}
else { // I use this section to debug in standalone mode.
let argument = ' {"type":"Task","name":"Runs Backtests, Fordwardtests & Live Trades ","bot":{"type":"Trading Engine","processes":[{"type":"Process","name":"Multi Period","code":{"team":"AAMasters","bot":"AAJason","process":"Multi-Period","repo":"AAJason-Trading-Engine-Bot"},"id":"4748c8c4-4d19-4076-96b2-e9c06524fbb3"}]},"id":"561bac18-fc78-464a-90c5-79fd821fc633"}'
+
try {
global.TASK_NODE = JSON.parse(argument)
} catch (err) {