Merge pull request #148 from AAMasters/develop

Beta 4
This commit is contained in:
Luis Fernando Molina authored and GitHub committed 2020-05-03 15:47:54 +02:00
commit f8c9a3c9df
21 files changed
+1500 -557

No files matched your search

+29
View File
@@ -289,6 +289,35 @@
let variable = {} // This is the structure where the user will define its own variables that will be shared across different code blocks and formulas.
let results = []
/*
We are going to build a map structure so that users can easily get the current object of
each data dependency based on the current element of the main data dependency.
We will use as key for these maps the begin and end property, meaning only elements with exactly
the same begin and end property will be matched.
*/
const PROPERTIES_COUNT = Object.keys(products).length
let dataDependencies = {}
if (PROPERTIES_COUNT > 1) { // only if we have secondary data dependencies.
for (let i = 1; i < PROPERTIES_COUNT; i++) {
let dataDependecyArray = Object.values(products)[i]
let dataDependencyName = Object.keys(products)[i]
let dataDependencyMap = new Map()
for (let j = 0; j < dataDependecyArray.length; j++) {
let record = dataDependecyArray[j]
let key = record.begin.toString() + '-' + record.end.toString()
dataDependencyMap.set(key, record)
}
dataDependencies[dataDependencyName] = dataDependencyMap
}
}
function getElement(dependencyName, currentRecordPrimaryDataDependency) {
let key = currentRecordPrimaryDataDependency.begin.toString() + '-' + currentRecordPrimaryDataDependency.end.toString()
return dataDependencies[dependencyName].get(key)
}
/* This is Initialization Code */
if (dataBuildingProcedure.initialization !== undefined) {
if (dataBuildingProcedure.initialization.javascriptCode !== undefined) {
+31 -13
View File
@@ -8,11 +8,13 @@
let thisObject = {
nodeArray: undefined,
dataSets: new Map(),
dataSetsModulesArray: [],
isItADepenency: isItADepenency,
initialize: initialize,
keys: []
finalize: finalize
};
let filter = new Map()
return thisObject;
@@ -44,10 +46,17 @@
callBackFunction(global.DEFAULT_OK_RESPONSE);
return;
}
/* Session based dependency filters */
if (bot.DEPENDENCY_FILTER !== undefined) {
for (let i = 0; i < bot.DEPENDENCY_FILTER.length; i++) {
let key = bot.DEPENDENCY_FILTER[i]
filter.set(key, true)
}
}
/*
For each dependency declared at the nodeArray, we will initialize a DataSet as part of this initialization process.
*/
let alreadyCalledBack = false;
let addCount = 0;
@@ -61,7 +70,7 @@
function onInitilized(err) {
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] initialize -> onInitilized -> err = "+ err.stack);
logger.write(MODULE_NAME, "[ERROR] initialize -> onInitilized -> err = "+ JSON.stringify(err));
alreadyCalledBack = true;
callBackFunction(err);
@@ -77,14 +86,7 @@
addCount++;
let key;
key = thisObject.nodeArray[i].dataMine + "-" + thisObject.nodeArray[i].bot + "-" + thisObject.nodeArray[i].product + "-" + thisObject.nodeArray[i].dataSet + "-" + thisObject.nodeArray[i].dataSetVersion;
thisObject.keys.push(key);
thisObject.dataSets.set(key, dataSetModule);
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> addDataSet -> DataSet added to Map. -> key = " + key); }
thisObject.dataSetsModulesArray.push(dataSetModule);
if (addCount === thisObject.nodeArray.length) {
if (alreadyCalledBack === false) {
@@ -101,4 +103,20 @@
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function finalize() {
for (let i = 0; i < thisObject.dataSetsModulesArray.length; i++) {
let dataSetModule = thisObject.dataSetsModulesArray[i]
dataSetModule.finalize()
}
thisObject.dataSetsModulesArray = undefined
filter = undefined
bot = undefined
}
function isItADepenency(timeFrame, product) {
let key = timeFrame + '-' + product
return filter.get(key)
}
};
+108 -36
View File
@@ -5,92 +5,154 @@ exports.newDataSet = function newDataSet(BOT, logger) {
let bot = BOT;
let thisObject = {
node: undefined,
networkNode: undefined,
initialize: initialize,
finalize: finalize,
getTextFile: getTextFile,
createTextFile: createTextFile
};
let dataDependencyNode;
/* Storage account to be used here. */
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let fileStorage
return thisObject;
function initialize(pDataDependencyNode, callBackFunction) {
function initialize(dataDependency, callBackFunction) {
try {
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
dataDependencyNode = pDataDependencyNode;
logger.fileName = MODULE_NAME + "." + dataDependencyNode.type + "." + dataDependencyNode.name + "." + dataDependencyNode.id;
thisObject.node = dataDependency.referenceParent;
logger.fileName = MODULE_NAME + "." + thisObject.node.type + "." + thisObject.node.name + "." + thisObject.node.id;
/* Some very basic validations that we have all the information needed. */
if (dataDependencyNode.referenceParent === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Data Dependency without Reference Parent. Data Dependency = " + JSON.stringify(dataDependencyNode));
if (thisObject.node === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Data Dependency without Reference Parent -> dataDependency = " + JSON.stringify(dataDependency));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (dataDependencyNode.referenceParent.code.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Dataset witn no codeName defined. Product Dataset = " + JSON.stringify(dataDependencyNode.referenceParent));
if (thisObject.node.code.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Dataset witn no codeName defined -> Product Dataset = " + JSON.stringify(thisObject.node));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (dataDependencyNode.referenceParent.parentNode === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Dataset not attached to a Product Definition. Dataset = " + JSON.stringify(dataDependencyNode.referenceParent));
if (thisObject.node.parentNode === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Dataset not attached to a Product Definition -> Dataset = " + JSON.stringify(thisObject.node));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (dataDependencyNode.referenceParent.parentNode.code.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Product Definition witn no codeName defined. Product Definition = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode));
if (thisObject.node.parentNode.code.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Product Definition witn no codeName defined -> Product Definition = " + JSON.stringify(thisObject.node.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (dataDependencyNode.referenceParent.parentNode.parentNode === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Product Definition not attached to a Bot. Product Definition = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode));
if (thisObject.node.parentNode.parentNode === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Product Definition not attached to a Bot -> Product Definition = " + JSON.stringify(thisObject.node.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (dataDependencyNode.referenceParent.parentNode.parentNode.code.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Bot witn no codeName defined. Bot = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode.parentNode));
if (thisObject.node.parentNode.parentNode.code.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Bot witn no codeName defined. Bot = " + JSON.stringify(thisObject.node.parentNode.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (dataDependencyNode.referenceParent.parentNode.parentNode.parentNode === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Bot not attached to a Data Mine. Bot = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode.parentNode));
if (thisObject.node.parentNode.parentNode.parentNode === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Bot not attached to a Data Mine. Bot = " + JSON.stringify(thisObject.node.parentNode.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (dataDependencyNode.referenceParent.parentNode.parentNode.parentNode.code.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Data Mine witn no codeName defined. Data Mine = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode.parentNode.parentNode));
if (thisObject.node.parentNode.parentNode.parentNode.code.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] initialize -> Data Mine witn no codeName defined. Data Mine = " + JSON.stringify(thisObject.node.parentNode.parentNode.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
/* Simplifying the access to basic info */
dataDependencyNode.dataSet = dataDependencyNode.referenceParent.code.codeName
dataDependencyNode.product = dataDependencyNode.referenceParent.parentNode.code.codeName
dataDependencyNode.bot = dataDependencyNode.referenceParent.parentNode.parentNode.code.codeName
dataDependencyNode.dataMine = dataDependencyNode.referenceParent.parentNode.parentNode.parentNode.code.codeName
/* Now we will see where do we need to fetch this data from. */
let network = global.TASK_NETWORK
let datasetProductDefinition = thisObject.node.parentNode
/* This stuff is still hardcoded and unresolved. */
dataDependencyNode.botVersion = {
"major": 1,
"minor": 0
for (let i = 0; i < network.networkNodes.length; i++) {
let networkNode = network.networkNodes[i]
if (networkNode.dataStorage !== undefined) {
if (networkNode.dataStorage.sessionIndependentData !== undefined) {
for (let j = 0; j < networkNode.dataStorage.sessionIndependentData.exchangeDataProducts.length; j++) {
let exchangeDataProduct = networkNode.dataStorage.sessionIndependentData.exchangeDataProducts[j]
for (let k = 0; k < exchangeDataProduct.singleMarketData.length; k++) {
let singleMarketData = exchangeDataProduct.singleMarketData[k]
if (singleMarketData.referenceParent !== undefined) {
let market = singleMarketData.referenceParent
let currentProcessMarket = bot.processNode.marketReference.referenceParent
if (currentProcessMarket.id === market.id) {
for (let m = 0; m < singleMarketData.dataProducts.length; m++) {
let dataProduct = singleMarketData.dataProducts[m]
if (dataProduct.referenceParent !== undefined) {
let productDefinition = dataProduct.referenceParent
if (datasetProductDefinition.id === productDefinition.id) {
/* We found where the data is located on the network. */
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving data from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); }
fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort);
callBackFunction(global.DEFAULT_OK_RESPONSE);
return
}
}
}
}
}
}
}
}
if (networkNode.dataStorage.sessionBasedData !== undefined) {
for (let j = 0; j < networkNode.dataStorage.sessionBasedData.exchangeDataProducts.length; j++) {
let exchangeDataProduct = networkNode.dataStorage.sessionBasedData.exchangeDataProducts[j]
for (let s = 0; s < exchangeDataProduct.sessionReferences.length; s++) {
let sessionReference = exchangeDataProduct.sessionReferences[s]
let singleMarketData = sessionReference.singleMarketData
if (singleMarketData.referenceParent !== undefined) {
let market = singleMarketData.referenceParent
let currentProcessMarket = bot.processNode.marketReference.referenceParent
if (currentProcessMarket.id === market.id) {
for (let m = 0; m < singleMarketData.dataProducts.length; m++) {
let dataProduct = singleMarketData.dataProducts[m]
if (dataProduct.referenceParent !== undefined) {
let productDefinition = dataProduct.referenceParent
if (datasetProductDefinition.id === productDefinition.id) {
/* We found where the data is located on the network. */
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving data from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); }
fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort);
callBackFunction(global.DEFAULT_OK_RESPONSE);
return
}
}
}
}
}
}
}
}
}
}
dataDependencyNode.dataSetVersion = "dataSet.V1"
callBackFunction(global.DEFAULT_OK_RESPONSE);
logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this dataset is located within the network. Check the logs for more info.");
logger.write(MODULE_NAME, "[ERROR] initialize -> Could not find where " + datasetProductDefinition.name + " for " + bot.exchange + " " + bot.market.baseAsset + "/" + bot.market.quotedAsset + " is stored within the network.");
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack);
@@ -98,13 +160,23 @@ exports.newDataSet = function newDataSet(BOT, logger) {
}
}
function finalize() {
fileStorage = undefined
thisObject.networkNode = undefined
bot = undefined
thisObject = undefined
}
function getTextFile(pFolderPath, pFileName, callBackFunction) {
try {
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getTextFile -> Entering function."); }
let filePathRoot = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + dataDependencyNode.dataMine + "/" + dataDependencyNode.bot;
let dataMineCodeName = thisObject.node.parentNode.parentNode.parentNode.code.codeName
let botCodeName = thisObject.node.parentNode.parentNode.code.codeName
let filePathRoot = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + dataMineCodeName + "/" + botCodeName;
let filePath = filePathRoot + "/Output/" + pFolderPath;
filePath += '/' + pFileName
@@ -132,7 +204,7 @@ exports.newDataSet = function newDataSet(BOT, logger) {
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] createTextFile -> pFolderPath = " + pFolderPath); }
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] createTextFile -> pFileName = " + pFileName); }
let ownerId = dataDependencyNode.dataMine + "-" + dataDependencyNode.bot + "-" + dataDependencyNode.botVersion.major + "-" + dataDependencyNode.botVersion.minor + "-" + dataDependencyNode.dataSetVersion;
let ownerId = thisObject.node.dataMine + "-" + thisObject.node.bot + "-" + thisObject.node.botVersion.major + "-" + thisObject.node.botVersion.minor + "-" + thisObject.node.dataSetVersion;
let botId = bot.dataMine + "-" + bot.codeName + "-" + bot.version.major + "-" + bot.version.minor + "-" + bot.dataSetVersion;
if (ownerId !== botId) {
@@ -146,7 +218,7 @@ exports.newDataSet = function newDataSet(BOT, logger) {
return;
}
let filePathRoot = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + dataDependencyNode.dataMine + "/" + dataDependencyNode.bot;
let filePathRoot = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + thisObject.node.dataMine + "/" + thisObject.node.bot;
let filePath = filePathRoot + "/Output/" + pFolderPath + '/' + pFileName;
fileStorage.createTextFile(filePath, pFileContent, onFileCreated);
+12 -4
View File
@@ -17,7 +17,8 @@ exports.newDebugLog = function newDebugLog() {
write: write,
newInternalLoop: newInternalLoop,
persist: persist, // This method is executed at the end of each Main Loop.
initialize: initialize
initialize: initialize,
finalize: finalize
};
let accumulatedLog = "[";
@@ -37,6 +38,10 @@ exports.newDebugLog = function newDebugLog() {
}
}
function finalize() {
persist()
}
function newInternalLoop(pBot, pProcess, date, percentage) {
if (percentage === undefined) {
percentage = ""
@@ -47,7 +52,7 @@ exports.newDebugLog = function newDebugLog() {
if (date === undefined) { date = thisObject.bot.processDatetime }
date = date.getUTCFullYear() + '-' + strPad(date.getUTCMonth() + 1, 2, "0") + '-' + strPad(date.getUTCDate(), 2, "0");
console.log(new Date().toISOString() + " " + strPad(thisObject.bot.exchange, 20) + " " + strPad(thisObject.bot.market.baseAsset + '/' + thisObject.bot.market.quotedAsset, 10) + " " + strPad(pBot, 20) + " " + strPad(pProcess, 30)
console.log(new Date().toISOString() + " " + strPad(thisObject.bot.exchange, 20) + " " + strPad(thisObject.bot.market.baseAsset + '/' + thisObject.bot.market.quotedAsset, 10) + " " + strPad(pBot, 30) + " " + strPad(pProcess, 30)
+ " Internal Loop # " + strPad(internalLoopCounter + 1, 8) + " " + strPad(date, 30) + " " + strPad(percentage, 10))
persist();
@@ -61,9 +66,10 @@ exports.newDebugLog = function newDebugLog() {
function persist() {
/* Here we actually write the content of the in-memory log to a blob */
/* Here we actually write the content of the in-memory log to a file */
try {
if (accumulatedLog === "[") {return} // nothing to persist at the moment.
internalLoopCounter++;
@@ -96,7 +102,9 @@ exports.newDebugLog = function newDebugLog() {
writeLog();
/* This is the implementation of the mechanism to auto-mantain logs. */
thisObject.bot.LOGS_TO_DELETE_QUEUE.push(filePath + '/' + fileName)
if (contentToPersist.indexOf('[ERROR]') < 0) {
thisObject.bot.LOGS_TO_DELETE_QUEUE.push(filePath + '/' + fileName)
}
if (thisObject.bot.LOGS_TO_DELETE_QUEUE.length > thisObject.bot.DELETE_QUEUE_SIZE) {
let fileToDelete = thisObject.bot.LOGS_TO_DELETE_QUEUE[0]
thisObject.bot.LOGS_TO_DELETE_QUEUE.splice(0, 1)
+10 -3
View File
@@ -1,4 +1,4 @@
exports.newEventsServerClient = function newEventsServerClient() {
exports.newEventsServerClient = function newEventsServerClient(host, port) {
const MODULE = "Task Server "
const ERROR_LOG = true
@@ -19,7 +19,14 @@
let WEB_SOCKETS_CLIENT
const WEB_SOCKET = require('ws')
let port = process.env.WEB_SOCKETS_SERVER_PORT
if (host === undefined) {
host = 'localhost'
}
if (port === undefined) {
port = process.env.WEB_SOCKETS_SERVER_PORT
}
return thisObject
@@ -32,7 +39,7 @@
function setuptWebSockets(callBackFunction) {
try {
WEB_SOCKETS_CLIENT = new WEB_SOCKET('ws://localhost:' + port )
WEB_SOCKETS_CLIENT = new WEB_SOCKET('ws://' + host + ':' + port )
WEB_SOCKETS_CLIENT.onerror = error => {
console.log('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> On connection error -> error = ' + JSON.stringify(error))
+1 -1
View File
@@ -38,7 +38,7 @@
'secret': secret,
'timeout': 30000,
'enableRateLimit': true,
verbose: true,
verbose: false,
options: options
}
+217 -42
View File
@@ -1,7 +1,7 @@
const path = require('path')
exports.newFileStorage = function newFileStorage(logger) {
exports.newFileStorage = function newFileStorage(logger, host, port) {
const MODULE_NAME = 'FileStorage'
const MAX_RETRY = 30
@@ -24,13 +24,13 @@ exports.newFileStorage = function newFileStorage(logger) {
return thisObject
function getTextFile(filePath, callBackFunction, noRetry) {
function getTextFile(filePath, callBackFunction, noRetry, canUsePrevious) {
let currentRetryGetTextFile = 0
recursiveGetTextFile(filePath, callBackFunction, noRetry)
recursiveGetTextFile(filePath, callBackFunction, noRetry, canUsePrevious)
function recursiveGetTextFile(filePath, callBackFunction, noRetry) {
function recursiveGetTextFile(filePath, callBackFunction, noRetry, canUsePrevious) {
logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> Entering Function.')
@@ -38,19 +38,28 @@ exports.newFileStorage = function newFileStorage(logger) {
/* Choose path for either bots or data */
let fileLocation
let mustBeJason
if (filePath.indexOf("/bots/") > 0) {
fileLocation = process.env.BOTS_PATH + '/' + filePath
mustBeJason = false
} else {
fileLocation = process.env.STORAGE_PATH + '/' + filePath
mustBeJason = true
}
try {
logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> fileLocation: ' + fileLocation)
/* Here we actually write the file. */
const fs = require('fs')
fs.readFile(fileLocation, onFileRead)
/* Here we actually reaad the file. */
if (host === undefined || host === 'localhost') {
/* We read the file from the local file system. */
const fs = require('fs')
fs.readFile(fileLocation, onFileRead)
} else {
/* We read the file via a web server over http */
getFileViaHTTP(fileLocation, onFileRead, callBackFunction)
}
function onFileRead(err, text) {
let retryTimeToUse = FAST_RETRY_TIME_IN_MILISECONDS
@@ -68,12 +77,12 @@ exports.newFileStorage = function newFileStorage(logger) {
}
callBackFunction(customResponse)
return
} else {
fileDoesNotExist = true
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> File does not exist. Retrying. ')
setTimeout(retry, retryTimeToUse)
return
}
}
fileDoesNotExist = true
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> File does not exist. Retrying. ')
setTimeout(retry, retryTimeToUse)
return
}
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> err = ' + err.stack)
@@ -83,13 +92,68 @@ exports.newFileStorage = function newFileStorage(logger) {
return
}
if (text.toString() === "") {
/*
It might happen that we try to read a file just at the moment it is being written by another process. It that case we might get
an empty file. If we are not allowed to use the Previous version of the file in this case, then we will retry to read it again
a little bit later.
*/
if (text.toString() === "" && canUsePrevious !== true) {
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Read and Empty File -> filePath = ' + filePath)
setTimeout(retry, retryTimeToUse)
return
}
callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString())
if (mustBeJason === false) {
/* In this case there is nothing else to check. */
callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString())
return
}
/*
We are going to check if the file is a valir JSON object
*/
try {
let jsonCheck = JSON.parse(text.toString())
/* The file was correctly read and could be parsed. */
callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString())
return
} catch (err) {
if (canUsePrevious === true) {
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Could read the file, but could not parse it as it is not a valid JSON. Will try to read the PREVIOUS version instead. -> file = ' + fileLocation)
if (host === undefined || host === 'localhost') {
/* We read the file from the local file system. */
const fs = require('fs')
fs.readFile(fileLocation + '.Previous.json', onPreviousFileRead)
} else {
/* We read the file via a web server over http */
getFileViaHTTP(fileLocation + '.Previous.json', onPreviousFileRead, callBackFunction)
}
function onPreviousFileRead(err, text) {
if (err) {
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> canUsePrevious -> Could not read the Previous file either. Giving up.')
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> canUsePrevious -> err = ' + err.stack)
callBackFunction(global.DEFAULT_FAIL_RESPONSE)
return
}
/* Returning the previous file */
callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString())
return
}
} else {
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Could read the file, but could not parse it as it is not a valid JSON.')
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> err = ' + err.stack)
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> text.toString() = ' + text.toString())
setTimeout(retry, retryTimeToUse)
return
}
}
}
} catch (err) {
@@ -103,7 +167,7 @@ exports.newFileStorage = function newFileStorage(logger) {
if (currentRetryGetTextFile < MAX_RETRY) {
currentRetryGetTextFile++
logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> retry -> Will try to read the file again -> Retry #: ' + currentRetryGetTextFile)
recursiveGetTextFile(filePath, callBackFunction)
recursiveGetTextFile(filePath, callBackFunction, noRetry, canUsePrevious)
} else {
currentRetryGetTextFile = 0
@@ -116,6 +180,7 @@ exports.newFileStorage = function newFileStorage(logger) {
callBackFunction(customResponse)
return
} else {
logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> Max retries reached reading a file. Giving up.')
logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> file = ' + fileLocation)
callBackFunction(global.DEFAULT_FAIL_RESPONSE)
@@ -126,13 +191,13 @@ exports.newFileStorage = function newFileStorage(logger) {
}
}
function createTextFile(filePath, fileContent, callBackFunction) {
function createTextFile(filePath, fileContent, callBackFunction, keepPrevious) {
let currentRetryWriteTextFile = 0
recursiveCreateTextFile(filePath, fileContent, callBackFunction)
recursiveCreateTextFile(filePath, fileContent, callBackFunction, keepPrevious)
function recursiveCreateTextFile(filePath, fileContent, callBackFunction) {
function recursiveCreateTextFile(filePath, fileContent, callBackFunction, keepPrevious) {
logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> Entering Function.')
@@ -165,32 +230,85 @@ exports.newFileStorage = function newFileStorage(logger) {
retryTimeToUse = SLOW_RETRY_TIME_IN_MILISECONDS
}
if (err) {
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> Error writing file -> file = ' + fileLocation)
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> Error writing file -> file = ' + fileLocation + '.tmp')
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> Error writing file -> err = ' + err.stack)
setTimeout(retry, retryTimeToUse)
} else {
const fs = require('fs')
fs.unlink(fileLocation, onUnlinked)
if (keepPrevious === true) {
/*
In some cases, we are going to keep a copy of the previous version of the file being written. This will be usefull to recover from crashes
when the file written gets corrupted for any reason.
*/
fs.unlink(fileLocation + '.Previous.json', onUnlinked)
function onUnlinked(err) {
let code = ''
if (err) {
code = err.code
}
if (code !== '' && code !== 'ENOENT') {
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error renaming file -> file = ' + fileLocation)
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error renaming file -> err = ' + err.stack)
setTimeout(retry, retryTimeToUse)
} else {
function onUnlinked(err) {
let code = ''
if (err) {
code = err.code
}
if (code !== '' && code !== 'ENOENT') {
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> file = ' + fileLocation + '.Previous.json')
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> err = ' + err.stack)
setTimeout(retry, retryTimeToUse)
return
}
const fs = require('fs')
fs.rename(fileLocation + '.tmp', fileLocation, onRenamed)
/* Rename de Original into Previous */
fs.rename(fileLocation, fileLocation + '.Previous.json', onOriginalRenamed)
function onRenamed(err) {
function onOriginalRenamed(err) {
let code = ''
if (err) {
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onRenamed -> Error renaming file -> file = ' + fileLocation + '.tmp')
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onRenamed -> Error renaming file -> err = ' + err.stack)
code = err.code
}
if (code !== '' && code !== 'ENOENT') { // Unless the file does not exists we do this...
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> Error renaming original file -> file = ' + fileLocation)
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> Error renaming original file -> err = ' + err.stack)
setTimeout(retry, retryTimeToUse)
return
}
/* Rename de Temp into Original */
fs.rename(fileLocation + '.tmp', fileLocation, onTempRenamed)
function onTempRenamed(err) {
if (err) {
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> file = ' + fileLocation + '.tmp')
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> err = ' + err.stack)
setTimeout(retry, retryTimeToUse)
} else {
callBackFunction(global.DEFAULT_OK_RESPONSE)
}
}
}
}
} else {
/* In this case, there is no need to keep a copy of the file being replaced, so we just delete and that's it. */
fs.unlink(fileLocation , onUnlinked)
function onUnlinked(err) {
let code = ''
if (err) {
code = err.code
}
if (code !== '' && code !== 'ENOENT') {
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> file = ' + fileLocation )
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> err = ' + err.stack)
setTimeout(retry, retryTimeToUse)
return
}
/* Rename de Temp into Original */
fs.rename(fileLocation + '.tmp', fileLocation, onTempRenamed)
function onTempRenamed(err) {
if (err) {
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> file = ' + fileLocation + '.tmp')
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> err = ' + err.stack)
setTimeout(retry, retryTimeToUse)
} else {
@@ -213,7 +331,7 @@ exports.newFileStorage = function newFileStorage(logger) {
if (currentRetryWriteTextFile < MAX_RETRY) {
currentRetryWriteTextFile++
logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> retry -> Will try to write the file again -> Retry #: ' + currentRetryWriteTextFile)
recursiveCreateTextFile(filePath, fileContent, callBackFunction)
recursiveCreateTextFile(filePath, fileContent, callBackFunction, keepPrevious)
} else {
currentRetryWriteTextFile = 0
logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> retry -> Max retries reached writting a file. Giving up.')
@@ -286,9 +404,8 @@ exports.newFileStorage = function newFileStorage(logger) {
}
}
/* Function to create folders of missing folders at any path. */
function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
/* Function to create folders of missing folders at any path. */
function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
const sep = '/';
const initDir = path.isAbsolute(targetDir) ? sep : '';
const baseDir = isRelativeToScript ? __dirname : '.';
@@ -316,5 +433,63 @@ exports.newFileStorage = function newFileStorage(logger) {
return curDir;
}, initDir);
}
}
function getFileViaHTTP(filePath, callback, callBackFunction) {
try {
/* The filePath received is the one that is needed to fetch data from with fs. To do it via http we need to remove the prefix that includes this: ./Data-Storage/ */
filePath = filePath.substring(15, filePath.length)
let http = require('http');
let url = 'http://' + host +
':' + port +
'/Storage/' +
filePath
logger.write(MODULE_NAME, '[INFO] FileStorage -> getFileViaHTTP -> url = ' + url)
let request = http.get(url, onResponse);
request.on('error', function (err) {
logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onError -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onError -> Failed to fetch file via HTTP. Will retry later. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
})
function onResponse(response) {
try {
const chunks = []
response.on('data', onMessegesArrived)
response.on('end', onEnd)
function onMessegesArrived(chunk) {
chunks.push(chunk)
}
function onEnd() {
let fileContent = Buffer.concat(chunks).toString('utf8')
let err = null
if (fileContent.indexOf('does not exist') >= 0) {
err = {
code: "ENOENT" // This is how fs would have returned upon this situation.
}
fileContent = undefined
}
callback(err, fileContent)
}
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onResponse -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onResponse -> Failed to fetch file via HTTP. Will retry later. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
}
}
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> Failed to fetch file via HTTP. Will retry later. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
}
}
}
+18 -4
View File
@@ -116,7 +116,11 @@
/* For each loop we want to create a new log file. */
if (logger !== undefined) {
logger.finalize()
}
logger = DEBUG_MODULE.newDebugLog();
global.LOGGER_MAP.set(MODULE_NAME, logger)
logger.bot = bot;
logger.initialize();
@@ -145,7 +149,7 @@
/* High level log entry */
console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30)
console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 30) + " " + pad(bot.process, 30)
+ " Main Loop # " + pad(Number(bot.loopCounter), 8))
/* We will prepare first the infraestructure needed for the bot to run. There are 3 modules we need to sucessfullly initialize first. */
@@ -252,6 +256,12 @@
switch (err.result) {
case global.DEFAULT_OK_RESPONSE.result: {
logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well.");
if (global.STOP_TASK_GRACEFULLY === true) {
loopControl()
return
}
initializeStatusDependencies();
return;
}
@@ -776,10 +786,13 @@
function onFinished(err) {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> onFinished -> Entering function."); }
processFramework.finalize()
processFramework = undefined
dataDependencies.finalize()
dataDependencies = undefined
statusDependencies.finalize()
statusDependencies = undefined
switch (err.result) {
case global.DEFAULT_OK_RESPONSE.result: {
@@ -876,11 +889,12 @@
processExecutionEvents.finish(onFinished);
function onFinished(err) {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Entering function."); }
processExecutionEvents.finalize()
processExecutionEvents = undefined
switch (err.result) {
case global.DEFAULT_OK_RESPONSE.result: {
logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Execution finished well.");
+261 -158
View File
@@ -15,8 +15,7 @@
let utilities = UTILITIES.newCloudUtilities(logger);
let statusDependencies;
let dataDependencies;
let datasets = [];
let dataDependenciesModule;
let dataFiles = new Map();
let multiPeriodDataFiles = new Map();
@@ -39,27 +38,9 @@
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
statusDependencies = pStatusDependencies;
dataDependencies = pDataDependencies;
dataDependenciesModule = pDataDependencies;
processConfig = pProcessConfig;
for (let i = 0; i < dataDependencies.nodeArray.length; i++) {
let key;
let dataset;
let dependency = dataDependencies.nodeArray[i];
key = dependency.dataMine + "-" +
dependency.bot + "-" +
dependency.product + "-" +
dependency.dataSet + "-" +
dependency.dataSetVersion
dataset = dataDependencies.dataSets.get(key);
datasets.push(dataset);
}
let USER_BOT_MODULE = require("./TradingBot")
botInstance = USER_BOT_MODULE.newTradingBot(bot, logger, UTILITIES, FILE_STORAGE);
@@ -72,11 +53,10 @@
}
function finalize() {
datasets = undefined
dataFiles = undefined
multiPeriodDataFiles = undefined
statusDependencies = undefined
dataDependencies = undefined
dataDependenciesModule = undefined
botInstance.finalize()
botInstance = undefined
fileStorage = undefined
@@ -91,7 +71,7 @@
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); }
let currentTimeFrame
let currentOutputPeriodName
let currentTimeFrameLabel
let market = bot.market;
let botNeverRan = true;
@@ -144,7 +124,7 @@
thisReport = statusReport.file;
if (thisReport.lastFile === undefined) {
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> thisReport = " + JSON.stringify(thisReport));
let customOK = {
result: global.CUSTOM_OK_RESPONSE.result,
@@ -199,7 +179,7 @@
contextVariables.lastFile = new Date(thisReport.lastFile);
interExecutionMemoryArray = thisReport.interExecutionMemoryArray;
processTimeFramesMarketFiles();
processSingleFiles();
return;
} else {
@@ -230,7 +210,7 @@
interExecutionMemoryArray.push(emptyObject);
}
processTimeFramesMarketFiles();
processSingleFiles();
}
@@ -244,44 +224,157 @@
}
}
function processTimeFramesMarketFiles() {
function processSingleFiles() {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> Entering function."); }
try {
let dependencyIndex = 0;
dataFiles = new Map();
dependencyLoopBody();
function dependencyLoopBody() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyLoopBody -> Entering function."); }
let dependency = dataDependenciesModule.nodeArray[dependencyIndex];
let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex];
getFile();
function getFile() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyLoopBody -> getFile -> Entering function."); }
if (datasetModule.node.code.codeName !== "Single-File") {
dependencyControlLoop();
return
}
if (dataDependenciesModule.isItADepenency('atAnyTimeFrame', datasetModule.node.parentNode.code.singularVariableName) !== true) {
dependencyControlLoop();
return
}
let fileName = "Data.json";
let filePath = datasetModule.node.parentNode.code.codeName + '/' + datasetModule.node.code.codeName;
datasetModule.getTextFile(filePath, fileName, onFileReceived);
function onFileReceived(err, text) {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); }
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); }
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
if (err.message === 'File does not exist.') {
logger.write(MODULE_NAME, "[ERROR] The file " + filePath + '/' + fileName + ' does not exist and it is required to continue. This process will retry to read it in a while. In the meantime make yourself sure that the process that generates it has ran properly.');
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
} else {
logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message);
callBackFunction(err);
}
return;
}
let dataFile = JSON.parse(text);
dataFiles.set(dependency.id, dataFile);
dependencyControlLoop();
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> dependencyLoopBody -> getFile -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> timeFramesLoop -> dependencyLoopBody -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function dependencyControlLoop() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyControlLoop -> Entering function."); }
dependencyIndex++;
if (dependencyIndex < dataDependenciesModule.nodeArray.length) {
dependencyLoopBody();
} else {
let mapKey = "Single Files"
multiPeriodDataFiles.set(mapKey, dataFiles)
processMarketFiles();
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> dependencyControlLoop -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function processMarketFiles() {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> Entering function."); }
try {
let n;
periodsLoop();
timeFramesLoop();
function periodsLoop() {
function timeFramesLoop() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoop -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoop -> Entering function."); }
/*
We will iterate through all posible periods.
We will iterate through all posible timeFrames.
*/
n = 0 // loop Variable representing each possible period as defined at the periods array.
n = 0 // loop Variable representing each possible period as defined at the timeFrames array.
periodsLoopBody();
timeFramesLoopBody();
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoop -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoop -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function periodsLoopBody() {
function timeFramesLoopBody() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoopBody -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> Entering function."); }
const timeFrame = global.marketFilesPeriods[n][0];
const timeFrameLabel = global.marketFilesPeriods[n][1];
@@ -291,7 +384,7 @@
if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) {
currentTimeFrame = global.marketFilesPeriods[n][0];
currentOutputPeriodName = global.marketFilesPeriods[n][1];
currentTimeFrameLabel = global.marketFilesPeriods[n][1];
}
dependencyLoopBody();
@@ -299,44 +392,46 @@
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> Entering function."); }
let dependency = dataDependencies.nodeArray[dependencyIndex];
let dataset = datasets[dependencyIndex];
let dependency = dataDependenciesModule.nodeArray[dependencyIndex];
let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex];
getFile();
function getFile() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> Entering function."); }
let fileName = "Data.json";
if (dependency.dataSet !== "Multi-Period-Market") {
if (dependency.referenceParent.code.codeName !== "Multi-Period-Market") {
dependencyControlLoop();
return
}
let filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel;
if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) {
if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) {
dependencyControlLoop();
return
}
}
dataset.getTextFile(filePath, fileName, onFileReceived);
let fileName = "Data.json";
let filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel;
datasetModule.getTextFile(filePath, fileName, onFileReceived);
function onFileReceived(err, text) {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); }
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); }
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); }
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
if (err.message === 'File does not exist.') {
logger.write(MODULE_NAME, "[ERROR] The file " + filePath + '/' + fileName + ' does not exist and it is required to continue. This process will retry to read it in a while. In the meantime make yourself sure that the process that generates it has ran properly.');
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
} else {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message);
logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message);
callBackFunction(err);
}
return;
@@ -349,20 +444,20 @@
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoop -> dependencyLoopBody -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoop -> dependencyLoopBody -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
@@ -371,11 +466,11 @@
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> dependencyControlLoop -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> dependencyControlLoop -> Entering function."); }
dependencyIndex++;
if (dependencyIndex < dataDependencies.nodeArray.length) {
if (dependencyIndex < dataDependenciesModule.nodeArray.length) {
dependencyLoopBody();
@@ -384,56 +479,56 @@
let mapKey = global.marketFilesPeriods[n][1];
multiPeriodDataFiles.set(mapKey, dataFiles)
periodsControlLoop();
timeFramesControlLoop();
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> dependencyControlLoop -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> dependencyControlLoop -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoopBody -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoopBody -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function periodsControlLoop() {
function timeFramesControlLoop() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsControlLoop -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesControlLoop -> Entering function."); }
n++;
if (n < global.marketFilesPeriods.length) {
periodsLoopBody();
timeFramesLoopBody();
} else {
processTimeFramesDailyFiles();
processDailyFiles();
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsControlLoop -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesControlLoop -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function processTimeFramesDailyFiles() {
function processDailyFiles() {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> Entering function."); }
try {
@@ -452,20 +547,20 @@
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> advanceTime -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> Entering function."); }
bot.multiPeriodProcessDatetime = new Date(bot.multiPeriodProcessDatetime.valueOf() + ONE_DAY_IN_MILISECONDS);
previousDay = new Date(bot.multiPeriodProcessDatetime.valueOf() - ONE_DAY_IN_MILISECONDS);
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> advanceTime -> bot.multiPeriodProcessDatetime = " + bot.multiPeriodProcessDatetime); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> advanceTime -> previousDay = " + previousDay); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> bot.multiPeriodProcessDatetime = " + bot.multiPeriodProcessDatetime); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> previousDay = " + previousDay); }
/* Validation that we are not going past the user defined end date. */
if (bot.multiPeriodProcessDatetime.valueOf() >= bot.VALUES_TO_USE.timeRange.finalDatetime.valueOf()) {
const logText = "User Defined End Datetime reached @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + ".";
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> advanceTime -> " + logText); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> " + logText); }
bot.STOP_SESSION = true
callBackFunction(global.DEFAULT_OK_RESPONSE);
@@ -478,7 +573,7 @@
if (bot.multiPeriodProcessDatetime.valueOf() > contextVariables.dateEndOfMarket.valueOf() + ONE_DAY_IN_MILISECONDS - 1) {
const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + ".";
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> advanceTime -> " + logText); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> " + logText); }
callBackFunction(global.DEFAULT_OK_RESPONSE);
return;
@@ -488,7 +583,7 @@
checkStopTaskGracefully();
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> advanceTime -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> advanceTime -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
@@ -497,7 +592,7 @@
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> checkStopTaskGracefully -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> checkStopTaskGracefully -> Entering function."); }
/* Validation that we dont need to stop. */
@@ -511,7 +606,7 @@
checkStopProcessing();
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> checkStopTaskGracefully -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> checkStopTaskGracefully -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
@@ -520,7 +615,7 @@
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> checkStopProcessing -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> checkStopProcessing -> Entering function."); }
/* Validation that we dont need to stop. */
@@ -531,19 +626,19 @@
}
periodsLoop();
timeFramesLoop();
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> checkStopProcessing -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> checkStopProcessing -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function periodsLoop() {
function timeFramesLoop() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoop -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoop -> Entering function."); }
/* Telling the world we are alive and doing well */
let processingDate = bot.multiPeriodProcessDatetime.getUTCFullYear() + '-' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCMonth() + 1, 2) + '-' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCDate(), 2);
@@ -551,46 +646,46 @@
/*
We will iterate through all posible periods.
We will iterate through all posible timeFrames.
*/
n = 0 // loop Variable representing each possible period as defined at the periods array.
n = 0 // loop Variable representing each possible period as defined at the timeFrames array.
periodsLoopBody();
timeFramesLoopBody();
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoop -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoop -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function periodsLoopBody() {
function timeFramesLoopBody() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> Entering function."); }
const timeFrame = global.dailyFilePeriods[n][0];
const timeFrameLabel = global.dailyFilePeriods[n][1];
if (processConfig.framework.validPeriods !== undefined) {
if (processConfig.framework.validtimeFrames !== undefined) {
let validPeriod = false;
for (let i = 0; i < processConfig.framework.validPeriods.length; i++) {
let period = processConfig.framework.validPeriods[i];
for (let i = 0; i < processConfig.framework.validtimeFrames.length; i++) {
let period = processConfig.framework.validtimeFrames[i];
if (period === timeFrameLabel) { validPeriod = true }
}
if (validPeriod === false) {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> Discarding period for not being listed as a valid period. -> timeFrameLabel = " + timeFrameLabel); }
periodsControlLoop();
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> Discarding period for not being listed as a valid period. -> timeFrameLabel = " + timeFrameLabel); }
timeFramesControlLoop();
return;
}
}
if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) {
currentTimeFrame = global.dailyFilePeriods[n][0];
currentOutputPeriodName = global.dailyFilePeriods[n][1];
currentTimeFrameLabel = global.dailyFilePeriods[n][1];
}
let dependencyIndex = 0;
@@ -601,10 +696,10 @@
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> Entering function."); }
let dependency = dataDependencies.nodeArray[dependencyIndex];
let dataset = datasets[dependencyIndex];
let dependency = dataDependenciesModule.nodeArray[dependencyIndex];
let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex];
let previousFile;
let currentFile;
@@ -617,35 +712,39 @@
}
function getPreviousFile() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); }
if (dependency.dataSet === "Multi-Period-Market") {
if (dependency.referenceParent.code.codeName !== "Multi-Period-Daily") {
dependencyControlLoop();
return
}
if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) {
if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) {
dependencyControlLoop();
return
}
}
let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2);
let filePath
if (dependency.dataSet === "Multi-Period-Daily") {
filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel + "/" + dateForPath;
if (dependency.referenceParent.code.codeName === "Multi-Period-Daily") {
filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel + "/" + dateForPath;
} else {
filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath;
filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + dateForPath;
}
let fileName = "Data.json";
dataset.getTextFile(filePath, fileName, onFileReceived);
datasetModule.getTextFile(filePath, fileName, onFileReceived);
function onFileReceived(err, text) {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); }
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); }
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); }
if ((err.message === "File does not exist." && botNeverRan === true) || err.code === 'The specified key does not exist.') {
@@ -656,7 +755,7 @@
After that, we will not accept more missing files on any of the dependencies, and if any is missing we will abort the processing.
*/
logger.write(MODULE_NAME, "[WARN] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found.");
logger.write(MODULE_NAME, "[WARN] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found.");
advanceTime();
return;
@@ -671,7 +770,7 @@
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack);
callBackFunction(err);
return;
}
@@ -682,46 +781,50 @@
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function getCurrentFile() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); }
if (dependency.dataSet === "Multi-Period-Market") {
if (dependency.referenceParent.code.codeName !== "Multi-Period-Daily") {
dependencyControlLoop();
return
}
if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) {
if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) {
dependencyControlLoop();
return
}
}
let dateForPath = bot.multiPeriodProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCDate(), 2);
let filePath
if (dependency.dataSet === "Multi-Period-Daily") {
filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel + "/" + dateForPath;
if (dependency.referenceParent.code.codeName === "Multi-Period-Daily") {
filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel + "/" + dateForPath;
} else {
filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath;
filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + dateForPath;
}
let fileName = "Data.json";
dataset.getTextFile(filePath, fileName, onFileReceived);
datasetModule.getTextFile(filePath, fileName, onFileReceived);
function onFileReceived(err, text) {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); }
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); }
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); }
if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') {
@@ -732,7 +835,7 @@
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Not Ok -> err = " + err.code);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Not Ok -> err = " + err.code);
callBackFunction(err);
return;
}
@@ -747,19 +850,19 @@
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoop -> dependencyLoopBody -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoop -> dependencyLoopBody -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
@@ -768,11 +871,11 @@
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> dependencyControlLoop -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> dependencyControlLoop -> Entering function."); }
dependencyIndex++;
if (dependencyIndex < dataDependencies.nodeArray.length) {
if (dependencyIndex < dataDependenciesModule.nodeArray.length) {
dependencyLoopBody();
@@ -781,34 +884,34 @@
let mapKey = global.dailyFilePeriods[n][1];
multiPeriodDataFiles.set(mapKey, dataFiles)
periodsControlLoop();
timeFramesControlLoop();
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> dependencyControlLoop -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> dependencyControlLoop -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function periodsControlLoop() {
function timeFramesControlLoop() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsControlLoop -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesControlLoop -> Entering function."); }
n++;
if (n < global.dailyFilePeriods.length) {
periodsLoopBody();
timeFramesLoopBody();
} else {
@@ -817,13 +920,13 @@
if (currentTimeFrame !== undefined) {
callTheBot();
} else {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsControlLoop -> Time Frame not Recognized. Can not continue." );
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesControlLoop -> Time Frame not Recognized. Can not continue." );
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsControlLoop -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesControlLoop -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
@@ -832,12 +935,12 @@
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> Entering function."); }
botInstance.start(
multiPeriodDataFiles,
currentTimeFrame,
currentOutputPeriodName,
currentTimeFrameLabel,
bot.multiPeriodProcessDatetime,
interExecutionMemoryArray[n],
onBotFinished);
@@ -846,7 +949,7 @@
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> onBotFinished -> Entering function."); }
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
callBackFunction(err);
@@ -859,17 +962,17 @@
writeMarketStatusReport(onMarketStatusReport)
} else {
writeDataRanges(currentOutputPeriodName, onWritten);
writeDataRanges(currentTimeFrameLabel, onWritten);
}
function onWritten(err) {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onWritten -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> onBotFinished -> onWritten -> Entering function."); }
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack);
callBackFunction(err);
return;
}
@@ -877,7 +980,7 @@
writeDailyStatusReport(bot.multiPeriodProcessDatetime, onDailyStatusReport);
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
@@ -885,21 +988,21 @@
function onDailyStatusReport() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onDailyStatusReport -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> onBotFinished -> onDailyStatusReport -> Entering function."); }
/* The next run we need the process to continue at the date it finished. */
processConfig.framework.startDate.resumeExecution = true;
advanceTime();
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onDailyStatusReport -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> onBotFinished -> onDailyStatusReport -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function onMarketStatusReport() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> Entering function."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> Entering function."); }
/* This is where we check if we need reached the user defined end datetime. */
@@ -907,7 +1010,7 @@
if (now.valueOf() >= bot.VALUES_TO_USE.timeRange.finalDatetime.valueOf()) {
const logText = "User Defined End Datetime reached @ " + now.getUTCFullYear() + "/" + (now.getUTCMonth() + 1) + "/" + now.getUTCDate() + ".";
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> " + logText); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> " + logText); }
bot.STOP_SESSION = true
callBackFunction(global.DEFAULT_OK_RESPONSE);
@@ -917,30 +1020,30 @@
callBackFunction(global.DEFAULT_OK_RESPONSE);
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> onBotFinished -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function writeDataRanges(currentOutputPeriodName, callBack) {
function writeDataRanges(currentTimeFrameLabel, callBack) {
try {
@@ -951,7 +1054,7 @@
function productLoopBody() {
let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.code.codeName;
writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodProcessDatetime, productCodeName, currentOutputPeriodName, controlLoop);
writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodProcessDatetime, productCodeName, currentTimeFrameLabel, controlLoop);
}
function controlLoop() {
@@ -972,7 +1075,7 @@
}
function writeDataRange(pBegin, pEnd, productCodeName, currentOutputPeriodName, callBack) {
function writeDataRange(pBegin, pEnd, productCodeName, currentTimeFrameLabel, callBack) {
try {
+86 -118
View File
@@ -15,8 +15,7 @@
let utilities = UTILITIES.newCloudUtilities(logger);
let statusDependencies;
let dataDependencies;
let datasets = [];
let dataDependenciesModule;
let dataFiles = new Map;
let botInstance;
@@ -40,27 +39,9 @@
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
statusDependencies = pStatusDependencies;
dataDependencies = pDataDependencies;
dataDependenciesModule = pDataDependencies;
processConfig = pProcessConfig;
for (let i = 0; i < dataDependencies.nodeArray.length; i++) {
let key;
let dataset;
let dependency = dataDependencies.nodeArray[i];
key = dependency.dataMine + "-" +
dependency.bot + "-" +
dependency.product + "-" +
dependency.dataSet + "-" +
dependency.dataSetVersion
dataset = dataDependencies.dataSets.get(key);
datasets.push(dataset);
}
let USER_BOT_MODULE = require("./IndicatorBot")
botInstance = USER_BOT_MODULE.newIndicatorBot(bot, logger, UTILITIES, FILE_STORAGE);
@@ -73,10 +54,9 @@
}
function finalize() {
datasets = undefined
dataFiles = undefined
statusDependencies = undefined
dataDependencies = undefined
dataDependenciesModule = undefined
botInstance = undefined
fileStorage = undefined
processConfig = undefined
@@ -115,95 +95,75 @@
let reportKey;
let statusReport;
if (processConfig.framework.startDate.fixedDate !== undefined) {
/*
We look first for the bot who knows the begining of the market in order to get when the market starts.
*/
/* The starting date is fixed, we will start from there. */
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> We have got a user defined startDate. -> startDate = " + processConfig.framework.startDate.fixedDate); }
contextVariables.dateBeginOfMarket = new Date(processConfig.framework.startDate.fixedDate);
} else {
/*
We look first for the bot who knows the begining of the market in order to get when the market starts.
*/
statusReport = statusDependencies.reportsByMainUtility.get("Market Starting Point")
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Market Starting Point -> Status Report does not exist or Market Starting Point not defined. Retrying Later. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
if (statusReport.status === "Status Report is corrupt.") {
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
thisReport = statusReport.file;
if (thisReport.beginingOfMarket === undefined) {
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet.");
let customOK = {
result: global.CUSTOM_OK_RESPONSE.result,
message: "Dependency does not exist."
}
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
callBackFunction(customOK);
return;
}
contextVariables.dateBeginOfMarket = new Date(thisReport.beginingOfMarket.year + "-" + thisReport.beginingOfMarket.month + "-" + thisReport.beginingOfMarket.days + " " + thisReport.beginingOfMarket.hours + ":" + thisReport.beginingOfMarket.minutes + GMT_SECONDS);
statusReport = statusDependencies.reportsByMainUtility.get("Market Starting Point")
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Market Starting Point -> Status Report does not exist or Market Starting Point not defined. Retrying Later. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
if (processConfig.framework.endDate.fixedDate !== undefined) {
/* The ending date is fixed, we will end there. */
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> We have got a user defined endDate. -> endDate = " + processConfig.framework.endDate.fixedDate); }
contextVariables.dateEndOfMarket = new Date(processConfig.framework.endDate.fixedDate);
} else {
/*
Here we get the status report from the bot who knows which is the end of the market.
*/
statusReport = statusDependencies.reportsByMainUtility.get("Market Ending Point")
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Market Ending Point -> Status Report does not exist or Market Ending Point not defined. Retrying Later. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
if (statusReport.status === "Status Report is corrupt.") {
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
thisReport = statusReport.file;
if (thisReport.lastFile === undefined) {
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
let customOK = {
result: global.CUSTOM_OK_RESPONSE.result,
message: "Dependency not ready."
}
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
callBackFunction(customOK);
return;
}
contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf());
if (statusReport.status === "Status Report is corrupt.") {
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
thisReport = statusReport.file;
if (thisReport.beginingOfMarket === undefined) {
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet.");
let customOK = {
result: global.CUSTOM_OK_RESPONSE.result,
message: "Dependency does not exist."
}
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
callBackFunction(customOK);
return;
}
contextVariables.dateBeginOfMarket = new Date(thisReport.beginingOfMarket.year + "-" + thisReport.beginingOfMarket.month + "-" + thisReport.beginingOfMarket.days + " " + thisReport.beginingOfMarket.hours + ":" + thisReport.beginingOfMarket.minutes + GMT_SECONDS);
/*
Here we get the status report from the bot who knows which is the end of the market.
*/
statusReport = statusDependencies.reportsByMainUtility.get("Market Ending Point")
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Market Ending Point -> Status Report does not exist or Market Ending Point not defined. Retrying Later. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
if (statusReport.status === "Status Report is corrupt.") {
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
thisReport = statusReport.file;
if (thisReport.lastFile === undefined) {
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
let customOK = {
result: global.CUSTOM_OK_RESPONSE.result,
message: "Dependency not ready."
}
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
callBackFunction(customOK);
return;
}
contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf());
/* Finally we get our own Status Report. */
statusReport = statusDependencies.reportsByMainUtility.get("Self Reference")
@@ -234,7 +194,7 @@
return;
}
if (bot.hasTheBotJustStarted === true && processConfig.framework.startDate.resumeExecution === false) {
if (bot.hasTheBotJustStarted === true && processConfig.framework.startDate !== undefined && processConfig.framework.startDate.resumeExecution === false) {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Starting from the begining because bot has just started and resume execution was true."); }
startFromBegining();
@@ -433,8 +393,16 @@
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> Entering function."); }
let dependency = dataDependencies.nodeArray[dependencyIndex];
let dataset = datasets[dependencyIndex];
let dependency = dataDependenciesModule.nodeArray[dependencyIndex];
if (dependency === undefined ) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> You need to add at least one Data Dependency to the process Multi Period Daily. Aborting process." );
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex];
let previousFile;
let currentFile;
@@ -454,14 +422,14 @@
let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2);
let filePath
if (dependency.dataSet === "Multi-Period-Daily") {
filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel + "/" + dateForPath;
if (dependency.referenceParent.code.codeName === "Multi-Period-Daily") {
filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel + "/" + dateForPath;
} else {
filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath;
filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + dateForPath;
}
let fileName = "Data.json";
dataset.getTextFile(filePath, fileName, onFileReceived);
datasetModule.getTextFile(filePath, fileName, onFileReceived);
function onFileReceived(err, text) {
@@ -524,14 +492,14 @@
let dateForPath = bot.multiPeriodDailyProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCDate(), 2);
let filePath
if (dependency.dataSet === "Multi-Period-Daily") {
filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel + "/" + dateForPath;
if (dependency.referenceParent.code.codeName === "Multi-Period-Daily") {
filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel + "/" + dateForPath;
} else {
filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath;
filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + dateForPath;
}
let fileName = "Data.json";
dataset.getTextFile(filePath, fileName, onFileReceived);
datasetModule.getTextFile(filePath, fileName, onFileReceived);
function onFileReceived(err, text) {
@@ -600,7 +568,7 @@
dependencyIndex++;
if (dependencyIndex < dataDependencies.nodeArray.length) {
if (dependencyIndex < dataDependenciesModule.nodeArray.length) {
dependencyLoopBody();
+10 -46
View File
@@ -13,8 +13,7 @@
let utilities = UTILITIES.newCloudUtilities(logger);
let statusDependencies;
let dataDependencies;
let datasets = [];
let dataDependenciesModule;
let dataFiles = new Map();
let botInstance;
@@ -33,27 +32,9 @@
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
statusDependencies = pStatusDependencies;
dataDependencies = pDataDependencies;
dataDependenciesModule = pDataDependencies;
processConfig = pProcessConfig;
for (let i = 0; i < dataDependencies.nodeArray.length; i++) {
let key;
let dataset;
let dependency = dataDependencies.nodeArray[i];
key = dependency.dataMine + "-" +
dependency.bot + "-" +
dependency.product + "-" +
dependency.dataSet + "-" +
dependency.dataSetVersion
dataset = dataDependencies.dataSets.get(key);
datasets.push(dataset);
}
let USER_BOT_MODULE = require("./IndicatorBot")
botInstance = USER_BOT_MODULE.newIndicatorBot(bot, logger, UTILITIES, FILE_STORAGE);
@@ -66,10 +47,9 @@
}
function finalize() {
datasets = undefined
dataFiles = undefined
statusDependencies = undefined
dataDependencies = undefined
dataDependenciesModule = undefined
botInstance = undefined
processConfig = undefined
thisObject = undefined
@@ -90,22 +70,6 @@
dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process.
};
if (processConfig.framework.startDate.fixedDate !== undefined) {
/* The starting date is fixed, we will start from there. */
contextVariables.dateBeginOfMarket = new Date(processConfig.framework.startDate.fixedDate);
}
if (processConfig.framework.endDate.fixedDate !== undefined) {
/* The ending date is fixed, we will end there. */
contextVariables.dateEndOfMarket = new Date(processConfig.framework.endDate.fixedDate);
}
processTimeFrames();
function processTimeFrames() {
@@ -161,8 +125,8 @@
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> Entering function."); }
let dependency = dataDependencies.nodeArray[dependencyIndex];
let dataset = datasets[dependencyIndex];
let dependency = dataDependenciesModule.nodeArray[dependencyIndex];
let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex];
getFile();
@@ -176,12 +140,12 @@
let fileName = "Data.json";
let filePath
if (dependency.dataSet === "Multi-Period-Market") {
filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel;
if (dependency.referenceParent.code.codeName === "Multi-Period-Market") {
filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel;
} else {
filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath;
filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + dateForPath;
}
dataset.getTextFile(filePath, fileName, onFileReceived);
datasetModule.getTextFile(filePath, fileName, onFileReceived);
function onFileReceived(err, text) {
@@ -230,7 +194,7 @@
dependencyIndex++;
if (dependencyIndex < dataDependencies.nodeArray.length) {
if (dependencyIndex < dataDependenciesModule.nodeArray.length) {
dependencyLoopBody();
+99 -19
View File
@@ -8,57 +8,137 @@
*/
const MODULE_NAME = "Process Execution Events";
logger.fileName = MODULE_NAME
let bot = BOT;
let thisObject = {
networkNode: undefined,
initialize, initialize,
finalize: finalize,
start: start,
finish: finish
};
let processThisDependsOn; // This is the process this process needs to wait for it to finishes in order to start.
let currentProcessKey
let EVENT_SERVER_CLIENT = require('./EventServerClient.js');
let eventServerClient
return thisObject;
function initialize(callBackFunction) {
try {
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
let name = 'Not Depends on any Process'
logger.fileName = MODULE_NAME + "." + name;
if (bot.processNode.referenceParent !== undefined) {
if (bot.processNode.referenceParent === undefined) {
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because the Process Instance needs to have a Reference Parent.");
return
}
let market = bot.market.baseAsset + '/' + bot.market.quotedAsset
currentProcessKey = bot.processNode.referenceParent.name + "-" + bot.processNode.referenceParent.type + "-" + bot.processNode.referenceParent.id + "-" + bot.exchange + "-" + market
let market = bot.market.baseAsset + '/' + bot.market.quotedAsset
currentProcessKey = bot.processNode.referenceParent.name + "-" + bot.processNode.referenceParent.type + "-" + bot.processNode.referenceParent.id + "-" + bot.exchange + "-" + market
if (bot.processNode.referenceParent.executionStartedEvent !== undefined) {
if (bot.processNode.referenceParent.executionStartedEvent.referenceParent !== undefined) {
if (bot.processNode.referenceParent.executionStartedEvent.referenceParent.parentNode !== undefined) {
processThisDependsOn = bot.processNode.referenceParent.executionStartedEvent.referenceParent.parentNode
if (bot.processNode.referenceParent.executionStartedEvent !== undefined) {
if (bot.processNode.referenceParent.executionStartedEvent.referenceParent !== undefined) {
if (bot.processNode.referenceParent.executionStartedEvent.referenceParent.parentNode !== undefined) {
processThisDependsOn = bot.processNode.referenceParent.executionStartedEvent.referenceParent.parentNode
name = processThisDependsOn.name
if (processThisDependsOn.code.monthlyInstances === true) {
logger.fileName = MODULE_NAME + "." + name;
}
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> " + currentProcessKey + " depends on " + name); }
name = processThisDependsOn.name
if (processThisDependsOn.code.monthlyInstances === true) {
logger.fileName = MODULE_NAME + "." + name;
}
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> " + currentProcessKey + " depends on " + name); }
}
}
}
logger.fileName = MODULE_NAME + "." + name;
callBackFunction(global.DEFAULT_OK_RESPONSE);
if (processThisDependsOn !== undefined) {
/* We need to find at which network node is running the process that we need to hear from when it finished. */
let network = global.TASK_NETWORK
for (let i = 0; i < network.networkNodes.length; i++) {
let networkNode = network.networkNodes[i]
if (networkNode.dataMining !== undefined) {
for (let j = 0; j < networkNode.dataMining.exchangeTasks.length; j++) {
let exchangeTasks = networkNode.dataMining.exchangeTasks[j]
for (let k = 0; k < exchangeTasks.taskManagers.length; k++) {
let taskManager = exchangeTasks.taskManagers[k]
for (let m = 0; m < taskManager.tasks.length; m++) {
let task = taskManager.tasks[m]
if (task.bot !== undefined) {
for (let n = 0; n < task.bot.processes.length; n++) {
let process = task.bot.processes[n]
if (process.marketReference !== undefined) {
if (process.marketReference.referenceParent !== undefined) {
let market = process.marketReference.referenceParent
let currentProcessMarket = bot.processNode.marketReference.referenceParent
if (currentProcessMarket.id === market.id) {
if (process.referenceParent !== undefined) {
let processDefinition = process.referenceParent
if (processThisDependsOn.id === processDefinition.id) {
/* We found where the task that runs the process definition we are waiting for is located on the network. */
thisObject.networkNode = networkNode
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Connecting to Websockets Server " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webSocketsPort + '.'); }
eventServerClient = EVENT_SERVER_CLIENT.newEventsServerClient(networkNode.code.host, networkNode.code.webSocketsPort)
eventServerClient.initialize(onConnected)
function onConnected() {
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Connected to Websockets Server " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webSocketsPort + '.'); }
callBackFunction(global.DEFAULT_OK_RESPONSE);
return
}
return
}
}
}
}
}
}
}
}
}
}
}
}
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the task that runs the Process Definition we are waiting for is located within the network. Check the logs for more info.");
return
}
callBackFunction(global.DEFAULT_OK_RESPONSE);
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function finalize() {
if (eventServerClient !== undefined) {
eventServerClient.finalize()
eventServerClient = undefined
}
processThisDependsOn = undefined
currentProcessKey = undefined
thisObject.networkNode = undefined
bot = undefined
thisObject = undefined
}
function start(callBackFunction) {
try {
@@ -81,9 +161,9 @@
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> waitForDependantProcess -> key = " + key); }
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> waitForDependantProcess -> callerId = " + callerId); }
global.EVENT_SERVER_CLIENT.listenToEvent(key, 'Process Execution Finished', undefined, callerId, responseCallBack, eventsCallBack)
eventServerClient.listenToEvent(key, 'Process Execution Finished', undefined, callerId, responseCallBack, eventsCallBack)
bot.processHeartBeat(undefined, undefined, "Waiting for " + processThisDependsOn.parentNode.parentNode.name + "->" + processThisDependsOn.parentNode.code.codeName + "->" + processThisDependsOn.code.codeName)
bot.processHeartBeat(undefined, undefined, "Waiting for " + thisObject.networkNode.name + "->" + processThisDependsOn.parentNode.parentNode.name + "->" + processThisDependsOn.parentNode.code.codeName + "->" + processThisDependsOn.code.codeName)
function responseCallBack(message) {
if (message.result !== global.DEFAULT_OK_RESPONSE.result) {
@@ -102,7 +182,7 @@
/* We continue the normal flow after we learn the dependent process has finished its execution. */
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> eventsCallBack -> stopListening to Process Execution Finished. "); }
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> eventsCallBack -> subscriptionId = " + subscriptionId); }
global.EVENT_SERVER_CLIENT.stopListening(key, 'Process Execution Finished', subscriptionId)
eventServerClient.stopListening(key, 'Process Execution Finished', subscriptionId)
if (message.event.err.result === global.DEFAULT_OK_RESPONSE.result) {
/* We emit our own event that the process started. */
+42 -11
View File
@@ -254,20 +254,24 @@
if (processConfig.framework !== undefined) {
if (processConfig.framework.name === "Multi-Period-Daily" || processConfig.framework.name === "Multi-Period-Market" || processConfig.framework.name === "Multi-Period") {
processConfig.framework.startDate.resumeExecution = true;
if (processConfig.startMode.noTime !== undefined) {
if (processConfig.startMode.noTime.run === "true") {
if (processConfig.startMode.noTime.beginDatetime !== undefined) {
processConfig.framework.startDate.fixedDate = processConfig.startMode.noTime.beginDatetime;
processConfig.framework.startDate.resumeExecution = true;
if (processConfig.framework.startDate !== undefined) {
processConfig.framework.startDate.resumeExecution = true;
if (processConfig.startMode.noTime !== undefined) {
if (processConfig.startMode.noTime.run === "true") {
if (processConfig.startMode.noTime.beginDatetime !== undefined) {
processConfig.framework.startDate.fixedDate = processConfig.startMode.noTime.beginDatetime;
processConfig.framework.startDate.resumeExecution = true;
}
}
}
}
if (processConfig.startMode.userDefined !== undefined) {
if (processConfig.startMode.userDefined.run === "true") {
if (processConfig.startMode.userDefined.beginDatetime !== undefined) {
processConfig.framework.startDate.fixedDate = processConfig.startMode.userDefined.beginDatetime;
processConfig.framework.startDate.resumeExecution = processConfig.startMode.userDefined.resumeExecution;
if (processConfig.startMode !== undefined) {
if (processConfig.startMode.userDefined !== undefined) {
if (processConfig.startMode.userDefined.run === "true") {
if (processConfig.startMode.userDefined.beginDatetime !== undefined) {
processConfig.framework.startDate.fixedDate = processConfig.startMode.userDefined.beginDatetime;
processConfig.framework.startDate.resumeExecution = processConfig.startMode.userDefined.resumeExecution;
}
}
}
}
@@ -276,6 +280,30 @@
try {
if (processConfig.startMode === undefined) { // Default
botConfig.hasTheBotJustStarted = true;
switch (botConfig.type) {
case 'Sensor Bot': {
runSensorBot(botConfig, processConfig);
break;
}
case 'Indicator Bot': {
runIndicatorBot(botConfig, processConfig);
break;
}
case 'Trading Bot': {
runTradingBot(botConfig, processConfig);
break;
}
default: {
console.log(logDisplace + "Root : [ERROR] start -> bootingBot -> Unexpected bot type. -> botConfig.type = " + botConfig.type);
}
}
return
}
if (processConfig.startMode.noTime !== undefined) {
if (processConfig.startMode.noTime.run === "true") {
@@ -363,6 +391,7 @@
let logger;
logger = DEBUG_MODULE.newDebugLog();
global.LOGGER_MAP.set('runSensorBot', logger)
logger.bot = pBotConfig;
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runSensorBot -> Entering function."); }
@@ -427,6 +456,7 @@
let logger;
logger = DEBUG_MODULE.newDebugLog();
global.LOGGER_MAP.set('runIndicatorBot', logger)
logger.bot = pBotConfig;
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runIndicatorBot -> Entering function."); }
@@ -489,6 +519,7 @@
let logger;
logger = DEBUG_MODULE.newDebugLog();
global.LOGGER_MAP.set('runTradingBot', logger)
logger.bot = pBotConfig;
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runTradingBot -> Entering function."); }
+14 -3
View File
@@ -121,7 +121,11 @@
/* For each loop we want to create a new log file. */
if (logger !== undefined) {
logger.finalize()
}
logger = DEBUG_MODULE.newDebugLog();
global.LOGGER_MAP.set(MODULE_NAME, logger)
logger.bot = bot;
logger.initialize();
@@ -148,7 +152,7 @@
/* High level log entry */
console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30)
console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 30) + " " + pad(bot.process, 30)
+ " Main Loop # " + pad(Number(bot.loopCounter), 8) )
/* We will prepare first the infraestructure needed for the bot to run. There are 4 modules we need to sucessfullly initialize first. */
@@ -254,6 +258,12 @@
switch (err.result) {
case global.DEFAULT_OK_RESPONSE.result: {
logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well.");
if (global.STOP_TASK_GRACEFULLY === true) {
loopControl()
return
}
initializeStatusDependencies();
return;
}
@@ -584,11 +594,12 @@
processExecutionEvents.finish(onFinished);
function onFinished(err) {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Entering function."); }
processExecutionEvents.finalize()
processExecutionEvents = undefined
switch (err.result) {
case global.DEFAULT_OK_RESPONSE.result: {
logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Execution finished well.");
+123 -61
View File
@@ -10,10 +10,12 @@
return thisObject;
function initialize(callBackFunction) {
function initialize(pProcessConfig, callBackFunction) {
try {
if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
let telegramAPI
/* Initialize this info so that everything is logged propeerly */
bot.SESSION = {
@@ -37,6 +39,7 @@
/* Listen to event to start or stop the session. */
bot.sessionKey = bot.processNode.session.name + '-' + bot.processNode.session.type + '-' + bot.processNode.session.id
global.SESSION_MAP.set(bot.sessionKey, bot.sessionKey)
global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Run Session', undefined, bot.sessionKey, undefined, runSession)
global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Stop Session', undefined, bot.sessionKey, undefined, stopSession)
@@ -45,66 +48,80 @@
return
function runSession(message) {
if (bot.SESSION_STATUS === 'Idle' || bot.SESSION_STATUS === 'Running') { return } // This happens when the UI is reloaded, the session was running and tries to run it again.
/* We are going to run the Definition comming at the event. */
bot.TRADING_SYSTEM = JSON.parse(message.event.tradingSystem)
bot.SESSION = JSON.parse(message.event.session)
/* Set the folderName for logging, reports, context and data output */
let code
if (bot.SESSION.code !== undefined) {
code = bot.SESSION.code
if (code.folderName === undefined) {
bot.SESSION.folderName = bot.SESSION.id
try {
if (bot.SESSION_STATUS === 'Idle' || bot.SESSION_STATUS === 'Running') { return } // This happens when the UI is reloaded, the session was running and tries to run it again.
/* We are going to run the Definition comming at the event. */
bot.TRADING_SYSTEM = JSON.parse(message.event.tradingSystem)
bot.SESSION = JSON.parse(message.event.session)
bot.DEPENDENCY_FILTER = JSON.parse(message.event.dependencyFilter)
/* Set the folderName for logging, reports, context and data output */
let code
if (bot.SESSION.code !== undefined) {
code = bot.SESSION.code
if (code.folderName === undefined) {
bot.SESSION.folderName = bot.SESSION.id
} else {
bot.SESSION.folderName = code.folderName + "-" + bot.SESSION.id
}
}
/* Extract values from different sources and consolidate them under one structure that is going to be used later on. */
setValuesToUse(message)
/* Set up Social Bots */
initializeSocialBots()
let allGood
switch (bot.SESSION.type) {
case 'Backtesting Session': {
allGood = startBackTesting(message)
break
}
case 'Live Trading Session': {
allGood = startLiveTrading(message)
break
}
case 'Fordward Testing Session': {
allGood = startFordwardTesting(message)
break
}
case 'Paper Trading Session': {
allGood = startPaperTrading(message)
break
}
}
if (allGood === true) {
bot.SESSION_STATUS = 'Idle'
bot.STOP_SESSION = false
} else {
bot.SESSION.folderName = code.folderName + "-" + bot.SESSION.id
bot.STOP_SESSION = true
}
}
/* Set up Social Bots */
setUpSocialBots()
/* Extract values from different sources and consolidate them under one structure that is going to be used later on. */
setValuesToUse(message)
let allGood
switch (bot.SESSION.type) {
case 'Backtesting Session': {
allGood = startBackTesting(message)
break
}
case 'Live Trading Session': {
allGood = startLiveTrading(message)
break
}
case 'Fordward Testing Session': {
allGood = startFordwardTesting(message)
break
}
case 'Paper Trading Session': {
allGood = startPaperTrading(message)
break
}
}
if (allGood === true) {
bot.SESSION_STATUS = 'Idle'
bot.STOP_SESSION = false
} else {
bot.STOP_SESSION = true
} catch (err) {
parentLogger.write(MODULE_NAME, "[ERROR] initialize -> runSession -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function stopSession(message) {
bot.STOP_SESSION = true
try {
bot.STOP_SESSION = true
finalizeSocialBots()
} catch (err) {
parentLogger.write(MODULE_NAME, "[ERROR] initialize -> stopSession -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function checkDatetimes() {
if (bot.VALUES_TO_USE.timeRange.initialDatetime.valueOf() < (new Date()).valueOf()) {
if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> startLiveTrading -> Overriding initialDatetime with present datetime because " + bot.VALUES_TO_USE.timeRange.initialDatetime + " is in the past."); }
if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> checkDatetimes -> Overriding initialDatetime with present datetime because " + bot.VALUES_TO_USE.timeRange.initialDatetime + " is in the past."); }
bot.VALUES_TO_USE.timeRange.initialDatetime = new Date()
}
if (bot.VALUES_TO_USE.timeRange.finalDatetime.valueOf() < (new Date()).valueOf()) {
if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> startLiveTrading -> Overriding finalDatetime with present datetime plus one year because " + bot.VALUES_TO_USE.timeRange.finalDatetime + " is in the past."); }
if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> checkDatetimes -> Overriding finalDatetime with present datetime plus one year because " + bot.VALUES_TO_USE.timeRange.finalDatetime + " is in the past."); }
bot.VALUES_TO_USE.timeRange.finalDatetime = new Date() + ONE_YEAR_IN_MILISECONDS
}
}
@@ -133,13 +150,14 @@
bot.resumeExecution = false;
bot.multiPeriodProcessDatetime = new Date(bot.VALUES_TO_USE.timeRange.initialDatetime.valueOf())
bot.hasTheBotJustStarted = true
pProcessConfig.liveWaitTime = getTimeFrameFromLabel(bot.VALUES_TO_USE.timeFrame)
return true
}
function startFordwardTesting(message) {
if (process.env.KEY === undefined || process.env.SECRET === undefined) {
if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> startLiveTrading -> Key name or Secret not provided, not possible to run the process in Forward Testing mode."); }
if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> startFordwardTesting -> Key name or Secret not provided, not possible to run the process in Forward Testing mode."); }
console.log("Key 'codeName' or 'secret' not provided. Plese check that and try again.")
return
}
@@ -166,6 +184,9 @@
bot.VALUES_TO_USE.maximumBalanceA = bot.VALUES_TO_USE.maximumBalanceA * balancePercentage / 100
bot.VALUES_TO_USE.maximumBalanceB = bot.VALUES_TO_USE.maximumBalanceB * balancePercentage / 100
pProcessConfig.normalWaitTime = getTimeFrameFromLabel(bot.VALUES_TO_USE.timeFrame)
return true
}
@@ -176,9 +197,31 @@
bot.resumeExecution = false;
bot.hasTheBotJustStarted = true
bot.multiPeriodProcessDatetime = new Date(bot.VALUES_TO_USE.timeRange.initialDatetime.valueOf())
pProcessConfig.normalWaitTime = getTimeFrameFromLabel(bot.VALUES_TO_USE.timeFrame)
return true
}
function getTimeFrameFromLabel(timeFrameLabel) {
for (let i = 0; i < global.marketFilesPeriods.length; i++) {
let value = global.marketFilesPeriods[i][0]
let label = global.marketFilesPeriods[i][1]
if (timeFrameLabel === label) {
return value
}
}
for (let i = 0; i < global.dailyFilePeriods.length; i++) {
let value = global.dailyFilePeriods[i][0]
let label = global.dailyFilePeriods[i][1]
if (timeFrameLabel === label) {
return value
}
}
}
function setValuesToUse(message) {
/*
Base on the information received we will determined which values ultimatelly are going to be used,
@@ -378,7 +421,7 @@
}
}
function setUpSocialBots() {
function initializeSocialBots() {
if (bot.SESSION.socialBots !== undefined) {
if (bot.SESSION.socialBots.bots !== undefined) {
@@ -386,7 +429,7 @@
let socialBot = bot.SESSION.socialBots.bots[i]
if (socialBot.type === "Telegram Bot") {
let code = socialBot.code
socialBot.botInstance = setUpTelegramBot(code.botToken, code.chatId)
socialBot.botInstance = initializeTelegramBot(code.botToken, code.chatId)
}
}
}
@@ -400,14 +443,14 @@
if (socialBot.type === "Telegram Bot") {
if (announcement.formulaValue !== undefined) {
socialBot.botInstance.telegramAPI.sendMessage(socialBot.botInstance.chatId, announcement.formulaValue).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> setUpSocialBots -> announce -> Telegram API error -> err = " + err))
socialBot.botInstance.telegramAPI.sendMessage(socialBot.botInstance.chatId, announcement.formulaValue).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeSocialBots -> announce -> Telegram API error -> err = " + err))
} else {
socialBot.botInstance.telegramAPI.sendMessage(socialBot.botInstance.chatId, code.text).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> setUpSocialBots -> announce -> Telegram API error -> err = " + err))
socialBot.botInstance.telegramAPI.sendMessage(socialBot.botInstance.chatId, code.text).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeSocialBots -> announce -> Telegram API error -> err = " + err))
}
}
} catch (err) {
parentLogger.write(MODULE_NAME, "[WARN] initialize -> setUpSocialBots -> announce -> err = " + err.stack);
parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeSocialBots -> announce -> err = " + err.stack);
}
}
}
@@ -417,22 +460,36 @@
}
}
function setUpTelegramBot(botToken, chatId) {
function finalizeSocialBots() {
if (bot.SESSION.socialBots === undefined) {return}
if (bot.SESSION.socialBots.bots !== undefined) {
for (let i = 0; i < bot.SESSION.socialBots.bots.length; i++) {
let socialBot = bot.SESSION.socialBots.bots[i]
if (socialBot.type === "Telegram Bot") {
let code = socialBot.code
socialBot.botInstance = finalizeTelegramBot(code.chatId)
}
}
}
}
function initializeTelegramBot(botToken, chatId) {
/* Telegram Bot Initialization */
try {
const Telegraf = require('telegraf')
let telegramBot
telegramBot = new Telegraf(botToken)
telegramBot.start((ctx) => ctx.reply('Hi! I am a Telegram Bot powered by Superalgos'))
telegramBot.help((ctx) => ctx.reply('Send me a sticker'))
telegramBot.on('sticker', (ctx) => ctx.reply('??'))
telegramBot.hears('hi', (ctx) => ctx.reply('Hey there'))
telegramBot.start((ctx) => ctx.reply('Hi! I am the Telegram Bot that will tell you what happens with your trading session.'))
telegramBot.help((ctx) => ctx.reply('I can stop the session if you say STOP.'))
telegramBot.hears('STOP', (ctx) => stopSession())
telegramBot.launch()
const Telegram = require('telegraf/telegram')
let telegramAPI
telegramAPI = new Telegram(botToken)
telegramAPI.sendMessage(chatId, bot.SESSION.type + " '" + bot.SESSION.name + "' was Started.").catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> setUpTelegramBot -> Telegram API error -> err = " + err))
const messge = bot.SESSION.type + " '" + bot.SESSION.name + "' was started with an initial balance of " + " " + bot.VALUES_TO_USE.initialBalanceA + " " + bot.VALUES_TO_USE.baseAsset + "."
telegramAPI.sendMessage(chatId, messge ).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeTelegramBot -> Telegram API error -> err = " + err))
let botInstance = {
telegramBot: telegramBot,
@@ -441,10 +498,15 @@
}
return botInstance
} catch (err) {
parentLogger.write(MODULE_NAME, "[WARN] initialize -> setUpTelegramBot -> err = " + err.stack);
parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeTelegramBot -> err = " + err.stack);
}
}
function finalizeTelegramBot(chatId) {
const messge = bot.SESSION.type + " '" + bot.SESSION.name + "' was stopped."
telegramAPI.sendMessage(chatId, messge).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeTelegramBot -> Telegram API error -> err = " + err))
}
} catch (err) {
parentLogger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack);
+10
View File
@@ -11,6 +11,7 @@
statusReports: new Map(),
reportsByMainUtility: new Map(),
initialize: initialize,
finalize: finalize,
keys: []
};
@@ -165,4 +166,13 @@
}
}
function finalize() {
thisObject.statusReports.forEach(forEachStatusDependency)
function forEachStatusDependency(statusDependency) {
statusDependency.finalize()
}
thisObject.statusReports = undefined
bot = undefined
thisObject = undefined
}
};
+95 -15
View File
@@ -13,9 +13,11 @@
let bot = BOT;
let thisObject = {
networkNode: undefined,
mainUtility: undefined,
file: undefined, // Here we have the JSON object representing the file content.
initialize: initialize,
finalize: finalize,
load: load,
save: save,
status: undefined
@@ -30,7 +32,7 @@
/* Storage account to be used here. */
const FILE_STORAGE = require('./FileStorage.js');
let fileStorage = FILE_STORAGE.newFileStorage(logger);
let fileStorage
let sessionPath = ''
@@ -93,17 +95,6 @@
statusDependencyNode.bottype = statusDependencyNode.referenceParent.parentNode.parentNode.type
statusDependencyNode.dataMine = statusDependencyNode.referenceParent.parentNode.parentNode.parentNode.code.codeName
/* Let's see if the process is run monthly or not. */
if (statusDependencyNode.referenceParent.parentNode.code.startMode !== undefined) {
if (statusDependencyNode.referenceParent.parentNode.code.startMode.allMonths !== undefined) {
if (statusDependencyNode.referenceParent.parentNode.code.startMode.oneMonth !== undefined) {
if (statusDependencyNode.referenceParent.parentNode.code.startMode.allMonths.run === "true" || statusDependencyNode.referenceParent.parentNode.code.startMode.oneMonth.run === "true") {
statusDependencyNode.processRunMonthly = true
}
}
}
}
/* We retrieve the report main utility */
if (statusDependencyNode.code !== undefined) {
if (statusDependencyNode.code.mainUtility !== undefined) {
@@ -122,7 +113,74 @@
sessionPath = bot.SESSION.folderName + "/"
}
callBackFunction(global.DEFAULT_OK_RESPONSE);
/* Now we will see where do we need to fetch this status report from. */
let network = global.TASK_NETWORK
let processThisDependsOn = statusDependencyNode.referenceParent.parentNode
for (let i = 0; i < network.networkNodes.length; i++) {
let networkNode = network.networkNodes[i]
if (checkThisBranch(networkNode.dataMining) === true) { return }
if (checkThisBranch(networkNode.testingEnvironment) === true) {return}
if (checkThisBranch(networkNode.productionEnvironment) === true) { return }
function checkThisBranch(branch) {
if (branch === undefined) {return}
for (let j = 0; j < branch.exchangeTasks.length; j++) {
let exchangeTasks = branch.exchangeTasks[j]
for (let k = 0; k < exchangeTasks.taskManagers.length; k++) {
let taskManager = exchangeTasks.taskManagers[k]
for (let m = 0; m < taskManager.tasks.length; m++) {
let task = taskManager.tasks[m]
if (task.bot !== undefined) {
for (let n = 0; n < task.bot.processes.length; n++) {
let process = task.bot.processes[n]
if (process.marketReference !== undefined) {
if (process.marketReference.referenceParent !== undefined) {
let market = process.marketReference.referenceParent
let currentProcessMarket = bot.processNode.marketReference.referenceParent
if (currentProcessMarket.id === market.id) {
if (process.referenceParent !== undefined) {
let processDefinition = process.referenceParent
if (processThisDependsOn.id === processDefinition.id) {
if (process.type === 'Trading Process Instance') {
if (process.session !== undefined) {
if (bot.processNode.session.id !== process.session.id) {
continue
}
} else {
continue
}
}
/* We found where the task that runs the process definition this status report depends on and where it is located on the network. */
thisObject.networkNode = networkNode
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving status report from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); }
fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort);
callBackFunction(global.DEFAULT_OK_RESPONSE);
return true
}
}
}
}
}
}
}
}
}
}
}
}
logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this status report is located within the network. Check the logs for more info.");
logger.write(MODULE_NAME, "[ERROR] initialize -> bot = " + statusDependencyNode.bot);
logger.write(MODULE_NAME, "[ERROR] initialize -> process = " + statusDependencyNode.process);
logger.write(MODULE_NAME, "[ERROR] initialize -> bottype = " + statusDependencyNode.bottype);
logger.write(MODULE_NAME, "[ERROR] initialize -> dataMine = " + statusDependencyNode.dataMine);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack);
@@ -130,6 +188,13 @@
}
}
function finalize() {
fileStorage = undefined
thisObject.networkNode = undefined
bot = undefined
thisObject = undefined
}
function load(callBackFunction) {
try {
@@ -153,7 +218,21 @@
filePath += '/' + fileName
fileStorage.getTextFile(filePath, onFileReceived);
let canUserPrevious
/*
If we are funning Trading Engines, we can not allow ourselves to use a Status Report that is not the latest one, because it might contain
transactioinal information related to the context of the operations the trading engine is doing.
On the contraty, if we are running a Sensor bot or an Indicator bot, we might, if necesary, use a previous version of a Status Report since
there will be no big impact, just some reprocessing.
*/
if (bot.SESSION !== undefined) {
canUserPrevious = false
} else {
canUserPrevious = true
}
fileStorage.getTextFile(filePath, onFileReceived, undefined, canUserPrevious);
function onFileReceived(err, text) {
@@ -198,6 +277,7 @@
*/
logger.write(MODULE_NAME, "[ERROR] load -> onFileReceived -> Error Parsing the Status report. -> Err = " + err.message);
logger.write(MODULE_NAME, "[WARN] load -> onFileReceived -> Error Parsing the Status report. -> text = " + text);
let customFail = {
result: global.CUSTOM_FAIL_RESPONSE.result,
@@ -241,7 +321,7 @@
filePath += '/' + fileName
let fileContent = JSON.stringify(thisObject.file);
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated);
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated, true);
function onFileCreated(err) {
+31 -9
View File
@@ -55,13 +55,6 @@
let chart = {}
let mainDependency = {}
let processingDailyFiles
if (currentDay !== undefined) {
processingDailyFiles = true
} else {
processingDailyFiles = false
}
/* The first phase here is about checking that we have everything we need at the definition level. */
let dataDependencies = bot.processNode.referenceParent.processDependencies.dataDependencies
@@ -103,6 +96,21 @@
}
}
/* Single Files */
let dataFiles = multiPeriodDataFiles.get('Single Files')
let products = {}
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Inflating Data Files from Single Files."); }
if (dataFiles !== undefined) {
commons.inflateDatafiles(dataFiles, dataDependencies, products, mainDependency, timeFrame)
let propertyName = 'atAnyTimeFrame';
chart[propertyName] = products
}
/* Simulation */
const TRADING_SIMULATION = require('./TradingSimulation.js');
let tradingSimulation = TRADING_SIMULATION.newTradingSimulation(bot, logger, UTILITIES);
@@ -205,6 +213,8 @@
record.maximumBalanceB + "," +
record.baseAsset + "," +
record.quotedAsset + "," +
record.marketBaseAsset + "," +
record.marketQuotedAsset + "," +
record.positionPeriods + "," +
record.positionDays + "," +
record.distanceToLastTriggerOn + "," +
@@ -551,8 +561,14 @@
record.maximumBalanceB + "," +
record.baseAsset + "," +
record.quotedAsset + "," +
record.marketBaseAsset + "," +
record.marketQuotedAsset + "," +
record.positionPeriods + "," +
record.positionDays + "]";
record.positionDays + "," +
record.distanceToLastTriggerOn + "," +
record.distanceToLastTriggerOff + "," +
record.distanceToLastTakePosition + "," +
record.distanceToLastClosePosition + "]";
if (separator === "") { separator = ","; }
@@ -656,8 +672,14 @@
record.maximumBalanceB + "," +
record.baseAsset + "," +
record.quotedAsset + "," +
record.marketBaseAsset + "," +
record.marketQuotedAsset + "," +
record.positionPeriods + "," +
record.positionDays + "]";
record.positionDays + "," +
record.distanceToLastTriggerOn + "," +
record.distanceToLastTriggerOff + "," +
record.distanceToLastTakePosition + "," +
record.distanceToLastClosePosition + "]";
if (separator === "") { separator = ","; }
+27 -7
View File
@@ -43,7 +43,7 @@
if (bot.definedByUI === true) {
/* The code of the bot is defined at the UI. No need to load a file with the code. */
session.initialize(onSessionInitialized)
session.initialize(processConfig, onSessionInitialized)
function onSessionInitialized(err) {
callBackFunction(err);
@@ -124,8 +124,11 @@
}
/* For each loop we want to create a new log file. */
if (logger !== undefined) {
logger.finalize()
}
logger = DEBUG_MODULE.newDebugLog();
global.LOGGER_MAP.set(MODULE_NAME, logger)
logger.bot = bot;
logger.initialize();
@@ -158,8 +161,8 @@
/* High level log entry */
console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30)
+ " Main Loop # " + pad(Number(bot.loopCounter), 8) + " " + bot.sessionKey)
console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 30) + " " + pad(bot.process, 30)
+ " Main Loop # " + pad(Number(bot.loopCounter), 8) + " " + bot.processNode.session.type + " " + bot.processNode.session.name)
/* Checking if we need to need to emit any event */
@@ -288,6 +291,12 @@
switch (err.result) {
case global.DEFAULT_OK_RESPONSE.result: {
logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well.");
if (global.STOP_TASK_GRACEFULLY === true) {
loopControl()
return
}
initializeStatusDependencies();
return;
}
@@ -694,6 +703,10 @@
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> onFinished -> Entering function."); }
processFramework.finalize()
processFramework = undefined
dataDependencies.finalize()
dataDependencies = undefined
statusDependencies.finalize()
statusDependencies = undefined
switch (err.result) {
case global.DEFAULT_OK_RESPONSE.result: {
@@ -792,8 +805,9 @@
processExecutionEvents.finish(onFinished);
function onFinished(err) {
try {
processExecutionEvents.finalize()
processExecutionEvents = undefined
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Entering function."); }
@@ -902,12 +916,18 @@
waitTime = processConfig.normalWaitTime
}
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (waitTime / 1000) + " seconds."); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (waitTime / 1000 / 60) + " minute/s."); }
if (processConfig.deadWaitTime > 0) {
checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.deadWaitTime, bot.loopCounter);
}
nextLoopTimeoutHandle = setTimeout(loop, waitTime);
processHeartBeat(undefined, undefined, "Waiting " + waitTime / 1000 + " seconds for next execution.")
let waitingTime = waitTime / 1000 / 60
let label = 'minute/s'
if (waitingTime < 1) {
waitingTime = waitTime / 1000
label = 'seconds'
}
processHeartBeat(undefined, undefined, "Waiting " + waitingTime + " " + label + " for next execution.")
if (global.WRITE_LOGS_TO_FILES === 'true') {
logger.persist();
}
+241 -1
View File
@@ -526,6 +526,7 @@
for (let k = 0; k < dataDependencies.length; k++) {
let dataDependencyNode = dataDependencies[k]
if (dataDependencyNode.referenceParent.code.codeName !== 'Multi-Period-Daily') { continue }
let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName
let pluralVariableName = dataDependencyNode.referenceParent.parentNode.code.pluralVariableName
let currentElement = getElement(thisChart[pluralVariableName], candle, 'Daily' + '-' + mapKey + '-' + pluralVariableName)
@@ -534,6 +535,7 @@
}
}
/* Finding the Current Element on Market Files */
for (let j = 0; j < global.marketFilesPeriods.length; j++) {
let mapKey = marketFilesPeriods[j][1]
@@ -542,6 +544,7 @@
for (let k = 0; k < dataDependencies.length; k++) {
let dataDependencyNode = dataDependencies[k]
if (dataDependencyNode.referenceParent.code.codeName !== 'Multi-Period-Market') {continue}
let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName
let pluralVariableName = dataDependencyNode.referenceParent.parentNode.code.pluralVariableName
let currentElement = getElement(thisChart[pluralVariableName], candle, 'Market' + '-' + mapKey + '-' + pluralVariableName)
@@ -549,6 +552,32 @@
}
}
/* Finding the Current Element on Single Files */
function isItInside(elementWithTimestamp, elementWithBeginEnd) {
if (elementWithTimestamp.timestamp >= elementWithBeginEnd.begin && elementWithTimestamp.timestamp <= elementWithBeginEnd.end) {
return true
} else {
return false
}
}
let propertyName = 'atAnyTimeFrame'
let thisChart = chart[propertyName]
for (let k = 0; k < dataDependencies.length; k++) {
let dataDependencyNode = dataDependencies[k]
if (dataDependencyNode.referenceParent.code.codeName !== 'Single-File') { continue }
let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName
let pluralVariableName = dataDependencyNode.referenceParent.parentNode.code.pluralVariableName
let elementArray = thisChart[pluralVariableName]
let currentElement
if (elementArray !== undefined) {
currentElement = elementArray[elementArray.length - 1]
}
thisChart[singularVariableName] = currentElement
}
/* While we are processing the previous day. */
let positionedAtYesterday = false
if (currentDay) {
@@ -589,6 +618,7 @@
previousLoopingDay = loopingDay.valueOf()
/* If any of the needed data dependencies is missing for this particular candle, then we jump the candle*/
/*
for (let k = 0; k < dataDependencies.length; k++) {
let dataDependencyNode = dataDependencies[k]
let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName
@@ -603,6 +633,7 @@
return
}
}
*/
periods++;
days = periods * timeFrame / ONE_DAY_IN_MILISECONDS;
@@ -842,6 +873,28 @@
}
}
}
/* move to phase events */
for (let n = 0; n < phase.moveToPhaseEvents.length; n++) {
let moveToPhaseEvent = phase.moveToPhaseEvents[n];
if (moveToPhaseEvent !== undefined) {
for (let k = 0; k < moveToPhaseEvent.situations.length; k++) {
let situation = moveToPhaseEvent.situations[k];
for (let m = 0; m < situation.conditions.length; m++) {
let condition = situation.conditions[m];
let key = j + '-' + 'openStage' + '-' + 'initialDefinition' + '-' + 'stopLoss' + '-' + p + '-' + n + '-' + k + '-' + m;
if (condition.javascriptCode !== undefined) {
newCondition(key, condition.javascriptCode, chart);
}
}
}
}
}
}
}
@@ -908,6 +961,28 @@
}
}
}
/* move to phase events */
for (let n = 0; n < phase.moveToPhaseEvents.length; n++) {
let moveToPhaseEvent = phase.moveToPhaseEvents[n];
if (moveToPhaseEvent !== undefined) {
for (let k = 0; k < moveToPhaseEvent.situations.length; k++) {
let situation = moveToPhaseEvent.situations[k];
for (let m = 0; m < situation.conditions.length; m++) {
let condition = situation.conditions[m];
let key = j + '-' + 'openStage' + '-' + 'initialDefinition' + '-' + 'takeProfit' + '-' + p + '-' + n + '-' + k + '-' + m;
if (condition.javascriptCode !== undefined) {
newCondition(key, condition.javascriptCode, chart);
}
}
}
}
}
}
}
}
@@ -984,6 +1059,28 @@
}
}
}
/* move to phase events */
for (let n = 0; n < phase.moveToPhaseEvents.length; n++) {
let moveToPhaseEvent = phase.moveToPhaseEvents[n];
if (moveToPhaseEvent !== undefined) {
for (let k = 0; k < moveToPhaseEvent.situations.length; k++) {
let situation = moveToPhaseEvent.situations[k];
for (let m = 0; m < situation.conditions.length; m++) {
let condition = situation.conditions[m];
let key = j + '-' + 'manageStage' + '-' + 'stopLoss' + '-' + p + '-' + n + '-' + k + '-' + m;
if (condition.javascriptCode !== undefined) {
newCondition(key, condition.javascriptCode, chart);
}
}
}
}
}
}
}
@@ -1051,6 +1148,28 @@
}
}
}
/* move to phase events */
for (let n = 0; n < phase.moveToPhaseEvents.length; n++) {
let moveToPhaseEvent = phase.moveToPhaseEvents[n];
if (moveToPhaseEvent !== undefined) {
for (let k = 0; k < moveToPhaseEvent.situations.length; k++) {
let situation = moveToPhaseEvent.situations[k];
for (let m = 0; m < situation.conditions.length; m++) {
let condition = situation.conditions[m];
let key = j + '-' + 'manageStage' + '-' + 'takeProfit' + '-' + p + '-' + n + '-' + k + '-' + m;
if (condition.javascriptCode !== undefined) {
newCondition(key, condition.javascriptCode, chart);
}
}
}
}
}
}
}
}
@@ -1387,6 +1506,7 @@
if (parentNode !== undefined) {
let phase = parentNode.stopLoss.phases[p];
/* Check the next Phase Event. */
let nextPhaseEvent = phase.nextPhaseEvent;
if (nextPhaseEvent !== undefined) {
@@ -1430,6 +1550,64 @@
}
}
}
/* Check the Move to Phase Events. */
for (let n = 0; n < phase.moveToPhaseEvents.length; n++) {
let moveToPhaseEvent = phase.moveToPhaseEvents[n];
if (moveToPhaseEvent !== undefined) {
for (let k = 0; k < moveToPhaseEvent.situations.length; k++) {
let situation = moveToPhaseEvent.situations[k];
let passed = true;
for (let m = 0; m < situation.conditions.length; m++) {
let condition = situation.conditions[m];
let key = j + '-' + stageKey + initialDefinitionKey + '-' + 'stopLoss' + '-' + p + '-' + n + '-' + k + '-' + m;
let value = false
if (conditions.get(key) !== undefined) {
value = conditions.get(key).value;
}
if (value === false) { passed = false; }
}
if (passed) {
let moveToPhase = moveToPhaseEvent.referenceParent
if (moveToPhase !== undefined) {
for (let q = 0; q < parentNode.stopLoss.phases.length; q++) {
if (parentNode.stopLoss.phases[q].id === moveToPhase.id) {
stopLossPhase = q + 1
}
}
} else {
moveToPhaseEvent.error = "This Node needs to reference a Phase."
continue
}
stopLossStage = 'Manage Stage'
if (takeProfitPhase > 0) {
strategyStage = 'Manage Stage'
checkAnnouncements(manageStage, 'Take Profit')
}
if (processingDailyFiles) {
if (positionedAtYesterday) {
yesterday.stopLossPhase = stopLossPhase;
yesterday.stopLossStage = stopLossStage;
yesterday.strategyStage = strategyStage;
}
}
checkAnnouncements(moveToPhaseEvent)
return;
}
}
}
}
}
}
@@ -1522,6 +1700,7 @@
let phase = parentNode.takeProfit.phases[p];
if (phase === undefined) {return} // trying to jump to a phase that does not exists.
/* Check the next Phase Event. */
let nextPhaseEvent = phase.nextPhaseEvent;
if (nextPhaseEvent !== undefined) {
@@ -1565,6 +1744,64 @@
}
}
}
/* Check the Move to Phase Events. */
for (let n = 0; n < phase.moveToPhaseEvents.length; n++) {
let moveToPhaseEvent = phase.moveToPhaseEvents[n];
if (moveToPhaseEvent !== undefined) {
for (let k = 0; k < moveToPhaseEvent.situations.length; k++) {
let situation = moveToPhaseEvent.situations[k];
let passed = true;
for (let m = 0; m < situation.conditions.length; m++) {
let condition = situation.conditions[m];
let key = j + '-' + stageKey + initialDefinitionKey + '-' + 'takeProfit' + '-' + p + '-' + n + '-' + k + '-' + m;
let value = false
if (conditions.get(key) !== undefined) {
value = conditions.get(key).value;
}
if (value === false) { passed = false; }
}
if (passed) {
let moveToPhase = moveToPhaseEvent.referenceParent
if (moveToPhase !== undefined) {
for (let q = 0; q < parentNode.takeProfit.phases.length; q++) {
if (parentNode.takeProfit.phases[q].id === moveToPhase.id) {
takeProfitPhase = q + 1
}
}
} else {
moveToPhaseEvent.error = "This Node needs to reference a Phase."
continue
}
takeProfitStage = 'Manage Stage'
if (stopLossPhase > 0) {
strategyStage = 'Manage Stage'
checkAnnouncements(manageStage, 'Stop')
}
if (processingDailyFiles) {
if (positionedAtYesterday) {
yesterday.takeProfitPhase = takeProfitPhase;
yesterday.takeProfitStage = takeProfitStage;
yesterday.strategyStage = strategyStage;
}
}
checkAnnouncements(moveToPhaseEvent)
return;
}
}
}
}
}
}
@@ -2561,6 +2798,8 @@
maximumBalanceB: bot.VALUES_TO_USE.maximumBalanceB,
baseAsset: quotedBaseAsset,
quotedAsset: quotedQuotedAsset,
marketBaseAsset: '"' + bot.market.baseAsset + '"',
marketQuotedAsset: '"' + bot.market.quotedAsset + '"' ,
positionPeriods: positionPeriods,
positionDays: positionDays,
distanceToLastTriggerOn: distanceToLast.triggerOn,
@@ -2879,7 +3118,7 @@
}
function getElement(pArray, currentCandle, datasetName) {
if (pArray === undefined) {return}
try {
let element;
for (let i = 0; i < pArray.length; i++) {
@@ -2912,6 +3151,7 @@
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] runSimulation -> getElement -> datasetName = " + datasetName);
logger.write(MODULE_NAME, "[ERROR] runSimulation -> getElement -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
+35 -6
View File
@@ -26,19 +26,37 @@ global.CUSTOM_FAIL_RESPONSE = {
};
const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000;
global.LOGGER_MAP = new Map()
global.SESSION_MAP = new Map()
process.on('uncaughtException', function (err) {
console.log('[ERROR] Task Server -> server -> uncaughtException -> err.message = ' + err.message)
console.log('[ERROR] Task Server -> server -> uncaughtException -> err.stack = ' + err.stack)
process.exit(1)
global.EXIT_NODE_PROCESS()
})
process.on('unhandledRejection', (reason, p) => {
console.log('[ERROR] Task Server -> server -> unhandledRejection -> reason = ' + JSON.stringify(reason))
console.log('[ERROR] Task Server -> server -> unhandledRejection -> p = ' + JSON.stringify(p))
process.exit(1)
global.EXIT_NODE_PROCESS()
})
function finalizeLoggers() {
global.LOGGER_MAP.forEach(forEachLogger)
function forEachLogger(logger) {
logger.finalize()
}
}
function finalizeSessions() {
global.SESSION_MAP.forEach(forEachSession)
function forEachSession(session) {
global.EVENT_SERVER_CLIENT.raiseEvent(session, 'Stopped')
}
}
process.on('exit', function (code) {
if (global.TASK_NODE !== undefined) {
@@ -69,8 +87,16 @@ process.on('message', message => {
}
});
let shuttingDownProcess = false
global.EXIT_NODE_PROCESS = function exitProcess() {
if (shuttingDownProcess === true) { return }
shuttingDownProcess = true
/* Signal that all sessions are stopping. */
finalizeSessions()
/* Cleaning Before Exiting. */
clearInterval(global.HEARTBEAT_INTERVAL_HANDLER)
@@ -82,6 +108,7 @@ global.EXIT_NODE_PROCESS = function exitProcess() {
global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Stopped') // Meaning Process Stopped
}
finalizeLoggers()
//console.log("[INFO] Task Server -> " + global.TASK_NODE.name + " -> EXIT_NODE_PROCESS -> Task Server will stop in 10 seconds.");
setTimeout(process.exit, 10000) // We will give 10 seconds to logs be written on file
@@ -125,12 +152,13 @@ function preLoader() {
global.EVENT_SERVER_CLIENT.raiseEvent('Task Manager - ' + taskId, 'Nodejs Process Ready for Task')
function eventReceived(message) {
global.TASK_NODE = message
global.TASK_NODE = JSON.parse(message.event.definition)
global.TASK_NODE = JSON.parse(message.event.taskDefinition)
global.TASK_NETWORK = JSON.parse(message.event.networkDefinition)
bootLoader()
}
} catch (err) {
console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE -> ' + err.stack)
console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE))
console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE).substring(0, 1000))
}
}
else { /* This process was started not by the Task Manager, but independently (most likely for debugging purposes). In this case we listen to an event with the Task Info that should be emitted at the UI */
@@ -139,12 +167,13 @@ function preLoader() {
global.EVENT_SERVER_CLIENT.listenToEvent('Task Server', 'Debug Task Started', undefined, 'Task Server', undefined, startDebugging)
function startDebugging(message) {
global.TASK_NODE = message
global.TASK_NODE = JSON.parse(message.event.definition)
global.TASK_NODE = JSON.parse(message.event.taskDefinition)
global.TASK_NETWORK = JSON.parse(message.event.networkDefinition)
bootLoader()
}
} catch (err) {
console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE -> ' + err.stack)
console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE))
console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE).substring(0, 1000))
}
}
}