Merge pull request #135 from AAMasters/develop

Bug fixed
This commit is contained in:
Luis Fernando Molina authored and GitHub committed 2019-09-23 13:05:59 +02:00
commit b28429589f
15 files changed
+222 -241

No files matched your search

+1 -1
View File
@@ -65,7 +65,7 @@
/* Storage account to be used here. */
const FILE_STORAGE = require('./Integrations/FileStorage.js');
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let statusDependencies;
+1 -1
View File
@@ -14,7 +14,7 @@ exports.newDataSet = function newDataSet(BOT, logger) {
/* Storage account to be used here. */
const FILE_STORAGE = require('./Integrations/FileStorage.js');
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
return thisObject;
+1 -1
View File
@@ -63,7 +63,7 @@ exports.newDebugLog = function newDebugLog() {
let contentToPersist = accumulatedLog;
accumulatedLog = "[";
const FILE_STORAGE = require('./Integrations/FileStorage.js');
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage();
let filePath = thisObject.bot.filePathRoot + "/Logs/" + thisObject.bot.process + "/" + executionDatetime;
+183
View File
@@ -0,0 +1,183 @@
const path = require('path')
const fs = require('fs')
exports.newFileStorage = function newFileStorage(logger) {
const MODULE_NAME = 'FileStorage'
const MAX_RETRY = 30
let currentRetryGetTextFile = 0
let currentRetryWriteTextFile = 0
if (logger === undefined) { // Dummy logger
logger = {}
function write() {
}
logger.write = write
}
let thisObject = {
getTextFile: getTextFile,
createTextFile: createTextFile
}
return thisObject
function getTextFile(container, filePath, callBackFunction) {
logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> Entering Function.')
container = container.toLowerCase()
/* Choose path for either bots or data */
let fileLocation
if (filePath.indexOf("/bots/") > 0) {
fileLocation = process.env.BOTS_PATH + '/' + container + '/' + filePath
} else {
fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath
}
try {
logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> fileLocation: ' + fileLocation)
/* Here we actually write the file. */
fs.readFile(fileLocation, onFileRead)
function onFileRead(err, text) {
if (err) {
if (err.code === 'ENOENT') {
let customResponse = {
result: global.CUSTOM_FAIL_RESPONSE.result,
message: 'File does not exist.'
}
callBackFunction(customResponse)
return
}
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> err = ' + err.stack)
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> container = ' + container)
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> filePath = ' + filePath)
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> err = ' + err.stack)
setTimeout(retry, 100)
return
}
if (text.toString() === "") {
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Read and Empty File -> filePath = ' + filePath)
setTimeout(retry, 100)
return
}
callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString())
}
} catch (err) {
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> Error writing file -> file = ' + fileLocation)
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> Error writing file -> err = ' + err.stack)
retry()
}
function retry() {
if (currentRetryGetTextFile < MAX_RETRY) {
currentRetryGetTextFile++
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> retry -> Will try to read the file again -> Retry #: ' + currentRetryGetTextFile)
getTextFile(container, filePath, callBackFunction)
} else {
currentRetryGetTextFile = 0
logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> Max retries reached reading a file. Giving up.')
logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> file = ' + fileLocation)
callBackFunction(global.DEFAULT_FAIL_RESPONSE)
}
}
}
function createTextFile(container, filePath, fileContent, callBackFunction) {
logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> Entering Function.')
container = container.toLowerCase()
/* Choose path for either logs or data */
let fileLocation
if (filePath.indexOf("/Logs/") > 0) {
fileLocation = process.env.LOG_PATH + '/' + container + '/' + filePath
} else {
fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath
}
try {
logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> fileLocation: ' + fileLocation)
/* If necesary a folder or folders are created before writing the file to disk. */
let directoryPath = fileLocation.substring(0, fileLocation.lastIndexOf('/') + 1);
mkDirByPathSync(directoryPath)
/* Here we actually write the file. */
fs.writeFile(fileLocation, fileContent, onFileWriten)
function onFileWriten(err) {
if (err) {
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> file = ' + fileLocation)
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> err = ' + err.stack)
setTimeout(retry, 100)
} else {
callBackFunction(global.DEFAULT_OK_RESPONSE)
}
}
} catch (err) {
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> Error writing file -> file = ' + fileLocation)
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> Error writing file -> err = ' + err.stack)
retry()
}
function retry() {
if (currentRetryWriteTextFile < MAX_RETRY) {
currentRetryWriteTextFile++
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> retry -> Will try to write the file again -> Retry #: ' + currentRetryWriteTextFile)
createTextFile(container, filePath, fileContent, callBackFunction)
} else {
currentRetryWriteTextFile = 0
logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> retry -> Max retries reached writting a file. Giving up.')
logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> retry -> file = ' + fileLocation)
callBackFunction(global.DEFAULT_FAIL_RESPONSE)
}
}
}
/* Function to create folders of missing folders at any path. */
function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
const sep = '/';
const initDir = path.isAbsolute(targetDir) ? sep : '';
const baseDir = isRelativeToScript ? __dirname : '.';
return targetDir.split(sep).reduce((parentDir, childDir) => {
const curDir = path.resolve(baseDir, parentDir, childDir);
try {
fs.mkdirSync(curDir);
} catch (err) {
if (err.code === 'EEXIST') { // curDir already exists!
return curDir;
}
// To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows.
if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure.
throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`);
}
const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1;
if (!caughtErr || caughtErr && curDir === path.resolve(targetDir)) {
throw err; // Throw if it's just the last created dir.
}
}
return curDir;
}, initDir);
}
}
+1 -1
View File
@@ -10,7 +10,7 @@
const MULTI_PERIOD_MARKET = require(ROOT_DIR + 'MultiPeriodMarket');
const MULTI_PERIOD_DAILY = require(ROOT_DIR + 'MultiPeriodDaily');
const FILE_STORAGE = require('./Integrations/FileStorage.js');
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(parentLogger);
const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog');
-224
View File
@@ -1,224 +0,0 @@
const axios = require('axios')
const Ecosystem = require('./Ecosystem')
let fs = require('fs')
const { promisify } = require('util')
const readFileAsync = promisify(fs.readFile)
const writeFileAsync = promisify(fs.writeFile)
const path = require('path')
exports.newFileStorage = function newFileStorage(logger) {
const MODULE_NAME = 'FileStorage'
const MAX_RETRY = 30
let currentRetryGetTextFile = 0
let currentRetryWriteTextFile = 0
const recoverableErrors = [
'SOCKETTIMEDOUT',
'TIMEDOUT',
'CONNRESET',
'CONNREFUSED',
'NOTFOUND',
'ENOTFOUND',
'ECONNREFUSED',
'CONNREFUSED',
'NOTFOUND',
'ESOCKETTIMEDOUT',
'ECONNRESET',
'ETIMEDOUT',
'EAI_AGAIN'
]
if (logger === undefined) {
logger = {}
function write() {
}
logger.write = write
}
return {
getTextFile,
createTextFile
}
function logInfo(message) {
log('[INFO] ' + message)
}
function log(message) {
if (logger) {
logger.write(MODULE_NAME, message);
} else {
//console.log("['" + new Date().toISOString() + "', 0,'" + MODULE_NAME + "','" + message + "']")
}
}
function logError(message) {
log('[ERROR] ' + message)
}
async function getTextFile(container, filePath, callBackFunction) {
try {
let host = await getDevTeamHost(container)
if (host.url.indexOf('localhost') !== -1) {
let fileLocation
if (filePath.indexOf("/bots/") > 0) {
fileLocation = process.env.BOTS_PATH + '/' + container + '/' + filePath
} else {
fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath
}
logInfo('getTextFile: ' + fileLocation)
//console.log('getTextFile: ' + fileLocation)
let fileContent = await readFileAsync(fileLocation)
callBackFunction(global.DEFAULT_OK_RESPONSE, fileContent.toString())
} else {
let response = await axios({
url: host.url + 'graphql',
method: 'post',
data: {
query: `
query web_FileContent($file: web_FileInput){
web_FileContent(file: $file)
}
`,
variables: {
file: {
container: container.toLowerCase(),
filePath,
storage: host.storage,
accessKey: host.accessKey
}
}
}
})
if (response.data.errors) {
callBackFunction({ code: response.data.errors[0] })
return
}
if (response.data.data.web_FileContent) {
callBackFunction(global.DEFAULT_OK_RESPONSE, response.data.data.web_FileContent)
} else {
callBackFunction({ code: 'The specified key does not exist.' })
}
}
} catch (err) {
if (verifyRetry(err.code) && currentRetryGetTextFile < MAX_RETRY) {
currentRetryGetTextFile++
logInfo('getTextFile -> Retrying connection to the server because received error: ' + err.code + '. Retry #: ' + currentRetryGetTextFile)
getTextFile(container, filePath, callBackFunction)
} else if (err.code === 'ENOENT') {
callBackFunction({ code: 'The specified key does not exist.' })
} else {
currentRetryGetTextFile = 0
logError('getTextFile -> error = ' + err.message)
callBackFunction(global.DEFAULT_FAIL_RESPONSE)
}
}
}
function createTextFile(container, filePath, fileContent, callBackFunction) {
/* Choose path for either logs or data */
let fileLocation
if (filePath.indexOf("/Logs/") > 0) {
fileLocation = process.env.LOG_PATH + '/' + container + '/' + filePath
} else {
fileLocation = process.env.STORAGE_PATH + '/' + container + '/' + filePath
}
try {
logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> fileLocation: ' + fileLocation)
/* If necesary a folder or folders are created before writing the file to disk. */
let directoryPath = fileLocation.substring(0, fileLocation.lastIndexOf('/') + 1);
mkDirByPathSync(directoryPath)
/* Here we actually write the file. */
fs.writeFile(fileLocation, fileContent, onFileWriten)
function onFileWriten(err) {
if (err) {
logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> file = ' + fileLocation)
logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> err = ' + err.stack)
retry()
} else {
callBackFunction(global.DEFAULT_OK_RESPONSE)
}
}
} catch (err) {
logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> Error writing file -> file = ' + fileLocation)
logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> Error writing file -> err = ' + err.stack)
retry()
}
function retry() {
if (currentRetryWriteTextFile < MAX_RETRY) {
currentRetryWriteTextFile++
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> retry -> Will try to write the file again -> Retry #: ' + currentRetryWriteTextFile)
createTextFile(container, filePath, fileContent, callBackFunction)
} else {
currentRetryWriteTextFile = 0
logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> retry -> Max retries reached. Giving up.')
callBackFunction(global.DEFAULT_FAIL_RESPONSE)
}
}
}
async function getDevTeamHost(devTeamName) {
let ecosystem = await Ecosystem.getEcosystem()
for (var i = 0; i < ecosystem.devTeams.length; i++) {
if (ecosystem.devTeams[i].codeName.toLowerCase() === devTeamName.toLowerCase()) {
return ecosystem.devTeams[i].host
}
}
}
function verifyRetry(errorCode) {
for (let i = 0; i < recoverableErrors.length; i++) {
const error = recoverableErrors[i];
if (error === errorCode) {
return true
}
}
return false
}
function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
const sep = '/';
const initDir = path.isAbsolute(targetDir) ? sep : '';
const baseDir = isRelativeToScript ? __dirname : '.';
return targetDir.split(sep).reduce((parentDir, childDir) => {
const curDir = path.resolve(baseDir, parentDir, childDir);
try {
fs.mkdirSync(curDir);
} catch (err) {
if (err.code === 'EEXIST') { // curDir already exists!
return curDir;
}
// To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows.
if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure.
throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`);
}
const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1;
if (!caughtErr || caughtErr && curDir === path.resolve(targetDir)) {
throw err; // Throw if it's just the last created dir.
}
}
return curDir;
}, initDir);
}
}
+1 -1
View File
@@ -22,7 +22,7 @@
let usertBot;
const FILE_STORAGE = require('./Integrations/FileStorage.js');
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let processConfig;
+1 -1
View File
@@ -21,7 +21,7 @@
let usertBot;
const FILE_STORAGE = require('./Integrations/FileStorage.js');
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let processConfig;
+1 -1
View File
@@ -19,7 +19,7 @@
let usertBot;
const FILE_STORAGE = require('./Integrations/FileStorage.js');
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let processConfig;
+1 -1
View File
@@ -26,7 +26,7 @@
<SubType>Code</SubType>
</Compile>
<Compile Include="Integrations\Ecosystem.js" />
<Compile Include="Integrations\FileStorage.js" />
<Compile Include="FileStorage.js" />
<Compile Include="MultiPeriod.js">
<SubType>Code</SubType>
</Compile>
+1 -1
View File
@@ -134,7 +134,7 @@
try {
console.log(logDisplace + "Root : [INFO] start -> getBotConfig -> Entering function. ");
const FILE_STORAGE = require('./Integrations/FileStorage.js');
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage();
let filePath = global.TASK_NODE.bot.processes[processIndex].code.team + '/bots/' + global.TASK_NODE.bot.processes[processIndex].code.repo + '/this.bot.config.json';
+1 -1
View File
@@ -8,7 +8,7 @@
let USER_BOT_MODULE;
let COMMONS_MODULE;
const FILE_STORAGE = require('./Integrations/FileStorage.js');
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(parentLogger);
const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog');
+21 -1
View File
@@ -29,7 +29,7 @@
/* Storage account to be used here. */
const FILE_STORAGE = require('./Integrations/FileStorage.js');
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let month;
@@ -240,6 +240,26 @@
let key = bot.devTeam + "-" + bot.codeName + "-" + bot.process
global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Status Report Updated')
/* We will also reaise the events for the datasets impacted by the process that just finished. */
for (let j = 0; j < bot.processes.length; j++) {
let process = bot.processes[j]
if (process.name === bot.process) {
let updatesDatasets = process.updatesDatasets
if (updatesDatasets !== undefined) {
for (let i = 0; i < updatesDatasets.length; i++) {
let updatedDataSet = updatesDatasets[i]
key = bot.devTeam + "-" + bot.codeName + "-" + updatedDataSet.product + "-" + updatedDataSet.dataSet
let event = {
lastFile: thisObject.file.lastFile
}
global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Dataset Updated', event)
}
}
}
}
callBackFunction(global.DEFAULT_OK_RESPONSE);
return;
}
+3 -3
View File
@@ -11,7 +11,7 @@
const MULTI_PERIOD_MARKET = require(ROOT_DIR + 'MultiPeriodMarket');
const MULTI_PERIOD_DAILY = require(ROOT_DIR + 'MultiPeriodDaily');
const MULTI_PERIOD = require(ROOT_DIR + 'MultiPeriod');
const FILE_STORAGE = require('./Integrations/FileStorage.js');
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(parentLogger);
@@ -277,10 +277,10 @@
/* Checking if we should process this loop or not.*/
if (global.STOP_PROCESSING === true) {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> We are going to skip this Loop bacause we were requested to stop or never asked to start."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for Backtesting or Live Trade to be started."); }
console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30)
+ " We are going to skip this Loop bacause we were requested to stop or never asked to start. ");
+ " Waiting for Backtesting or Live Trade to be started. ");
global.SYSTEM_EVENT_HANDLER.raiseEvent('Jason-Multi-Period', 'Process Stopped')
+5 -3
View File
@@ -71,7 +71,9 @@ if (global.TASK_NODE !== undefined) {
}
else { // I use this section to debug in standalone mode.
let argument = '{"type":"Task","name":"Runs Backtests, Fordwardtests & Live Trades ","bot":{"type":"Trading Engine","processes":[{"type":"Process","name":"Multi Period","code":{"team":"AAMasters","bot":"AAJason","process":"Multi-Period","repo":"AAJason-Trading-Engine-Bot"},"id":"4748c8c4-4d19-4076-96b2-e9c06524fbb3"}]},"id":"561bac18-fc78-464a-90c5-79fd821fc633"}'
let argument = ' {"type":"Task","name":"Runs Backtests, Fordwardtests & Live Trades ","bot":{"type":"Trading Engine","processes":[{"type":"Process","name":"Multi Period","code":{"team":"AAMasters","bot":"AAJason","process":"Multi-Period","repo":"AAJason-Trading-Engine-Bot"},"id":"4748c8c4-4d19-4076-96b2-e9c06524fbb3"}]},"id":"561bac18-fc78-464a-90c5-79fd821fc633"}'
// charly argument = '{"type":"Task","name":"Brings Trades Records from the Exchange","bot":{"type":"Sensor","name":"Charly","processes":[{"type":"Process","name":"Live Trades","code":{"team":"AAMasters","bot":"AACharly","process":"Live-Trades"},"id":"5846ebc7-1979-4a80-9cc7-94bb4b8659dc"},{"type":"Process","name":"Hole Fixing","code":{"team":"AAMasters","bot":"AACharly","process":"Hole-Fixing"},"id":"31fc4f05-75d4-419a-9fb0-73e25e856f15"}]},"id":"5bfef4dc-54c1-44db-ace6-1ab27a86746e"}'
// olivia argument = '{"type":"Task","name":"Generates 1 min to 24 hs Candles & Volumes","bot":{"type":"Indicator","name":"Olivia","processes":[{"type":"Process","name":"Daily","code":{"team":"AAMasters","bot":"AAOlivia","process":"Multi-Period-Daily"},"id":"68cc8e1b-e94a-477e-82d0-15ecc9f1b9e2"},{"type":"Process","name":"Market","code":{"team":"AAMasters","bot":"AAOlivia","process":"Multi-Period-Market"},"id":"e1ac5e3d-d491-4e90-baf4-d4e5b71a8b1a"}]},"id":"efe36e05-75ea-41b5-8d92-c6b27635c834"}'
try {
global.TASK_NODE = JSON.parse(argument)
} catch (err) {
@@ -133,9 +135,9 @@ global.EXIT_NODE_PROCESS = function exitProcess() {
global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Stopped') // Meaning Process Stopped
}
console.log("[INFO] Task Server -> " + global.TASK_NODE.name + " -> EXIT_NODE_PROCESS -> Task Server Stopped.");
console.log("[INFO] Task Server -> " + global.TASK_NODE.name + " -> EXIT_NODE_PROCESS -> Task Server will stop in 10 seconds.");
process.exit()
setTimeout(process.exit, 10000) // We will give 10 seconds to logs be written on file
}
let notFirstSequence = false;