From f0b22da4cbb6bcc0425a670346eb0a2f3f0a0711 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 10 Apr 2020 10:57:47 +0200 Subject: [PATCH 01/35] Update DebugLog.js --- DebugLog.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DebugLog.js b/DebugLog.js index 123a45006..5a33a9bf6 100644 --- a/DebugLog.js +++ b/DebugLog.js @@ -61,7 +61,7 @@ exports.newDebugLog = function newDebugLog() { function persist() { - /* Here we actually write the content of the in-memory log to a blob */ + /* Here we actually write the content of the in-memory log to a file */ try { @@ -96,7 +96,9 @@ exports.newDebugLog = function newDebugLog() { writeLog(); /* This is the implementation of the mechanism to auto-mantain logs. */ - thisObject.bot.LOGS_TO_DELETE_QUEUE.push(filePath + '/' + fileName) + if (contentToPersist.indexOf('[ERROR]') < 0) { + thisObject.bot.LOGS_TO_DELETE_QUEUE.push(filePath + '/' + fileName) + } if (thisObject.bot.LOGS_TO_DELETE_QUEUE.length > thisObject.bot.DELETE_QUEUE_SIZE) { let fileToDelete = thisObject.bot.LOGS_TO_DELETE_QUEUE[0] thisObject.bot.LOGS_TO_DELETE_QUEUE.splice(0, 1) From b9f74aea1931616ec2467d1aed8855575a53d29c Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 11 Apr 2020 13:00:39 +0200 Subject: [PATCH 02/35] Function getElement and Previous Status Report --- Commons.js | 29 +++++++++ FileStorage.js | 159 ++++++++++++++++++++++++++++++++++++++---------- StatusReport.js | 18 +++++- 3 files changed, 173 insertions(+), 33 deletions(-) diff --git a/Commons.js b/Commons.js index 8bf6a01da..70200a529 100644 --- a/Commons.js +++ b/Commons.js @@ -289,6 +289,35 @@ let variable = {} // This is the structure where the user will define its own variables that will be shared across different code blocks and formulas. let results = [] + /* + We are going to build a map structure so that users can easily get the current object of + each data dependency based on the current element of the main data dependency. + We will use as key for these maps the begin and end property, meaning only elements with exactly + the same begin and end property will be matched. + */ + + const PROPERTIES_COUNT = Object.keys(products).length + + let dataDependencies = {} + if (PROPERTIES_COUNT > 1) { // only if we have secondary data dependencies. + for (let i = 1; i < PROPERTIES_COUNT; i++) { + let dataDependecyArray = Object.values(products)[i] + let dataDependencyName = Object.keys(products)[i] + let dataDependencyMap = new Map() + for (let j = 0; j < dataDependecyArray.length; j++) { + let record = dataDependecyArray[j] + let key = record.begin.toString() + '-' + record.end.toString() + dataDependencyMap.set(key, record) + } + dataDependencies[dataDependencyName] = dataDependencyMap + } + } + + function getElement(dependencyName, currentRecordPrimaryDataDependency) { + let key = currentRecordPrimaryDataDependency.begin.toString() + '-' + currentRecordPrimaryDataDependency.end.toString() + return dataDependencies[dependencyName].get(key) + } + /* This is Initialization Code */ if (dataBuildingProcedure.initialization !== undefined) { if (dataBuildingProcedure.initialization.javascriptCode !== undefined) { diff --git a/FileStorage.js b/FileStorage.js index be6f08ccd..6e95fe028 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -24,13 +24,13 @@ exports.newFileStorage = function newFileStorage(logger) { return thisObject - function getTextFile(filePath, callBackFunction, noRetry) { + function getTextFile(filePath, callBackFunction, noRetry, canUsePrevious) { let currentRetryGetTextFile = 0 - recursiveGetTextFile(filePath, callBackFunction, noRetry) + recursiveGetTextFile(filePath, callBackFunction, noRetry, canUsePrevious) - function recursiveGetTextFile(filePath, callBackFunction, noRetry) { + function recursiveGetTextFile(filePath, callBackFunction, noRetry, canUsePrevious) { logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> Entering Function.') @@ -38,10 +38,13 @@ exports.newFileStorage = function newFileStorage(logger) { /* Choose path for either bots or data */ let fileLocation + let mustBeJason if (filePath.indexOf("/bots/") > 0) { fileLocation = process.env.BOTS_PATH + '/' + filePath + mustBeJason = false } else { fileLocation = process.env.STORAGE_PATH + '/' + filePath + mustBeJason = true } try { @@ -83,13 +86,59 @@ exports.newFileStorage = function newFileStorage(logger) { return } - if (text.toString() === "") { + /* + It might happen that we try to read a file just at the moment it is being written by another process. It that case we might get + an empty file. If we are not allowed to use the Previous version of the file in this case, then we will retry to read it again + a little bit later. + */ + if (text.toString() === "" && canUsePrevious !== true) { logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Read and Empty File -> filePath = ' + filePath) setTimeout(retry, retryTimeToUse) return } - callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString()) + if (mustBeJason === false) { + /* In this case there is nothing else to check. */ + callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString()) + return + } + + /* + We are going to check if the file is a valir JSON object + */ + try { + let jsonCheck = JSON.parse(text.toString()) + + /* The file was correctly read and could be parsed. */ + callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString()) + return + + } catch (err) { + + if (canUsePrevious === true) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Could read the file, but could not parse it as it is not a valid JSON. Will try to read the PREVIOUS version instead. -> file = ' + fileLocation) + + fs.readFile(fileLocation + '.Previous.json', onPreviousFileRead) + + function onPreviousFileRead(err, text) { + + if (err) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> canUsePrevious -> Could not read the Previous file either. Giving up.') + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> canUsePrevious -> err = ' + err.stack) + callBackFunction(global.DEFAULT_FAIL_RESPONSE) + } else { + /* Returning the previous file */ + callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString()) + return + } + } + } else { + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Could read the file, but could not parse it as it is not a valid JSON.') + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> err = ' + err.stack) + setTimeout(retry, retryTimeToUse) + return + } + } } } catch (err) { @@ -103,7 +152,7 @@ exports.newFileStorage = function newFileStorage(logger) { if (currentRetryGetTextFile < MAX_RETRY) { currentRetryGetTextFile++ logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> retry -> Will try to read the file again -> Retry #: ' + currentRetryGetTextFile) - recursiveGetTextFile(filePath, callBackFunction) + recursiveGetTextFile(filePath, callBackFunction, noRetry, canUsePrevious) } else { currentRetryGetTextFile = 0 @@ -116,6 +165,7 @@ exports.newFileStorage = function newFileStorage(logger) { callBackFunction(customResponse) return } else { + logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> Max retries reached reading a file. Giving up.') logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> file = ' + fileLocation) callBackFunction(global.DEFAULT_FAIL_RESPONSE) @@ -126,13 +176,13 @@ exports.newFileStorage = function newFileStorage(logger) { } } - function createTextFile(filePath, fileContent, callBackFunction) { + function createTextFile(filePath, fileContent, callBackFunction, keepPrevious) { let currentRetryWriteTextFile = 0 - recursiveCreateTextFile(filePath, fileContent, callBackFunction) + recursiveCreateTextFile(filePath, fileContent, callBackFunction, keepPrevious) - function recursiveCreateTextFile(filePath, fileContent, callBackFunction) { + function recursiveCreateTextFile(filePath, fileContent, callBackFunction, keepPrevious) { logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> Entering Function.') @@ -165,37 +215,84 @@ exports.newFileStorage = function newFileStorage(logger) { retryTimeToUse = SLOW_RETRY_TIME_IN_MILISECONDS } if (err) { - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> Error writing file -> file = ' + fileLocation) + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> Error writing file -> file = ' + fileLocation + '.tmp') logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> Error writing file -> err = ' + err.stack) setTimeout(retry, retryTimeToUse) } else { - const fs = require('fs') - fs.unlink(fileLocation, onUnlinked) + if (keepPrevious === true) { + /* + In some cases, we are going to keep a copy of the previous version of the file being written. This will be usefull to recover from crashes + when the file written gets corrupted for any reason. + */ + fs.unlink(fileLocation + '.Previous.json', onUnlinked) - function onUnlinked(err) { - let code = '' - if (err) { - code = err.code + function onUnlinked(err) { + let code = '' + if (err) { + code = err.code + } + if (code !== '' && code !== 'ENOENT') { + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> file = ' + fileLocation + '.Previous.json') + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> err = ' + err.stack) + setTimeout(retry, retryTimeToUse) + } else { + + fs.rename(fileLocation, fileLocation + '.Previous.json', onOriginalRenamed) + + function onOriginalRenamed(err) { + if (err) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> Error renaming original file -> file = ' + fileLocation) + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> Error renaming original file -> err = ' + err.stack) + setTimeout(retry, retryTimeToUse) + } else { + + fs.rename(fileLocation + '.tmp', fileLocation, onTempRenamed) + + function onTempRenamed(err) { + if (err) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> file = ' + fileLocation + '.tmp') + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> err = ' + err.stack) + setTimeout(retry, retryTimeToUse) + } else { + + callBackFunction(global.DEFAULT_OK_RESPONSE) + + } + } + } + } + } } - if (code !== '' && code !== 'ENOENT') { - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error renaming file -> file = ' + fileLocation) - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error renaming file -> err = ' + err.stack) - setTimeout(retry, retryTimeToUse) - } else { + } else { - const fs = require('fs') - fs.rename(fileLocation + '.tmp', fileLocation, onRenamed) + /* In this case, there is no need to keep a copy of the file being replaced, so we just delete and that's it. */ - function onRenamed(err) { - if (err) { - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onRenamed -> Error renaming file -> file = ' + fileLocation + '.tmp') - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onRenamed -> Error renaming file -> err = ' + err.stack) - setTimeout(retry, retryTimeToUse) - } else { + fs.unlink(fileLocation , onUnlinked) - callBackFunction(global.DEFAULT_OK_RESPONSE) + function onUnlinked(err) { + let code = '' + if (err) { + code = err.code + } + if (code !== '' && code !== 'ENOENT') { + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> file = ' + fileLocation ) + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> err = ' + err.stack) + setTimeout(retry, retryTimeToUse) + } else { + fs.rename(fileLocation + '.tmp', fileLocation, onTempRenamed) + + function onTempRenamed(err) { + if (err) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> file = ' + fileLocation + '.tmp') + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> err = ' + err.stack) + setTimeout(retry, retryTimeToUse) + } else { + + callBackFunction(global.DEFAULT_OK_RESPONSE) + + } } } } @@ -213,7 +310,7 @@ exports.newFileStorage = function newFileStorage(logger) { if (currentRetryWriteTextFile < MAX_RETRY) { currentRetryWriteTextFile++ logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> retry -> Will try to write the file again -> Retry #: ' + currentRetryWriteTextFile) - recursiveCreateTextFile(filePath, fileContent, callBackFunction) + recursiveCreateTextFile(filePath, fileContent, callBackFunction, keepPrevious) } else { currentRetryWriteTextFile = 0 logger.write(MODULE_NAME, '[ERROR] FileStorage -> createTextFile -> retry -> Max retries reached writting a file. Giving up.') diff --git a/StatusReport.js b/StatusReport.js index 4e1eec387..66839dac9 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -153,7 +153,21 @@ filePath += '/' + fileName - fileStorage.getTextFile(filePath, onFileReceived); + let canUserPrevious + /* + If we are funning Trading Engines, we can not allow ourselves to use a Status Report that is not the latest one, because it might contain + transactioinal information related to the context of the operations the trading engine is doing. + + On the contraty, if we are running a Sensor bot or an Indicator bot, we might, if necesary, use a previous version of a Status Report since + there will be no big impact, just some reprocessing. + */ + if (bot.SESSION !== undefined) { + canUserPrevious = false + } else { + canUserPrevious = true + } + + fileStorage.getTextFile(filePath, onFileReceived, undefined, canUserPrevious); function onFileReceived(err, text) { @@ -241,7 +255,7 @@ filePath += '/' + fileName let fileContent = JSON.stringify(thisObject.file); - fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated); + fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated, true); function onFileCreated(err) { From 1da366c734b351c1c463753878c91cb64b8ae20f Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 11 Apr 2020 15:14:49 +0200 Subject: [PATCH 03/35] Update Session.js --- Session.js | 147 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 90 insertions(+), 57 deletions(-) diff --git a/Session.js b/Session.js index 09e4aa90e..ff5464808 100644 --- a/Session.js +++ b/Session.js @@ -14,6 +14,8 @@ try { if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } + let telegramAPI + /* Initialize this info so that everything is logged propeerly */ bot.SESSION = { @@ -45,57 +47,70 @@ return function runSession(message) { - if (bot.SESSION_STATUS === 'Idle' || bot.SESSION_STATUS === 'Running') { return } // This happens when the UI is reloaded, the session was running and tries to run it again. - /* We are going to run the Definition comming at the event. */ - bot.TRADING_SYSTEM = JSON.parse(message.event.tradingSystem) - bot.SESSION = JSON.parse(message.event.session) - - /* Set the folderName for logging, reports, context and data output */ - let code - if (bot.SESSION.code !== undefined) { - code = bot.SESSION.code - if (code.folderName === undefined) { - bot.SESSION.folderName = bot.SESSION.id + try { + if (bot.SESSION_STATUS === 'Idle' || bot.SESSION_STATUS === 'Running') { return } // This happens when the UI is reloaded, the session was running and tries to run it again. + + /* We are going to run the Definition comming at the event. */ + bot.TRADING_SYSTEM = JSON.parse(message.event.tradingSystem) + bot.SESSION = JSON.parse(message.event.session) + + /* Set the folderName for logging, reports, context and data output */ + let code + if (bot.SESSION.code !== undefined) { + code = bot.SESSION.code + if (code.folderName === undefined) { + bot.SESSION.folderName = bot.SESSION.id + } else { + bot.SESSION.folderName = code.folderName + "-" + bot.SESSION.id + } + } + + /* Extract values from different sources and consolidate them under one structure that is going to be used later on. */ + setValuesToUse(message) + + /* Set up Social Bots */ + initializeSocialBots() + + let allGood + switch (bot.SESSION.type) { + case 'Backtesting Session': { + allGood = startBackTesting(message) + break + } + case 'Live Trading Session': { + allGood = startLiveTrading(message) + break + } + case 'Fordward Testing Session': { + allGood = startFordwardTesting(message) + break + } + case 'Paper Trading Session': { + allGood = startPaperTrading(message) + break + } + } + if (allGood === true) { + bot.SESSION_STATUS = 'Idle' + bot.STOP_SESSION = false } else { - bot.SESSION.folderName = code.folderName + "-" + bot.SESSION.id + bot.STOP_SESSION = true } - } - - /* Set up Social Bots */ - setUpSocialBots() - - /* Extract values from different sources and consolidate them under one structure that is going to be used later on. */ - setValuesToUse(message) - let allGood - switch (bot.SESSION.type) { - case 'Backtesting Session': { - allGood = startBackTesting(message) - break - } - case 'Live Trading Session': { - allGood = startLiveTrading(message) - break - } - case 'Fordward Testing Session': { - allGood = startFordwardTesting(message) - break - } - case 'Paper Trading Session': { - allGood = startPaperTrading(message) - break - } - } - if (allGood === true) { - bot.SESSION_STATUS = 'Idle' - bot.STOP_SESSION = false - } else { - bot.STOP_SESSION = true + } catch (err) { + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> runSession -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } function stopSession(message) { - bot.STOP_SESSION = true + try { + bot.STOP_SESSION = true + finalizeSocialBots() + } catch (err) { + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> stopSession -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } } function checkDatetimes() { @@ -378,7 +393,7 @@ } } - function setUpSocialBots() { + function initializeSocialBots() { if (bot.SESSION.socialBots !== undefined) { if (bot.SESSION.socialBots.bots !== undefined) { @@ -386,7 +401,7 @@ let socialBot = bot.SESSION.socialBots.bots[i] if (socialBot.type === "Telegram Bot") { let code = socialBot.code - socialBot.botInstance = setUpTelegramBot(code.botToken, code.chatId) + socialBot.botInstance = initializeTelegramBot(code.botToken, code.chatId) } } } @@ -400,14 +415,14 @@ if (socialBot.type === "Telegram Bot") { if (announcement.formulaValue !== undefined) { - socialBot.botInstance.telegramAPI.sendMessage(socialBot.botInstance.chatId, announcement.formulaValue).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> setUpSocialBots -> announce -> Telegram API error -> err = " + err)) + socialBot.botInstance.telegramAPI.sendMessage(socialBot.botInstance.chatId, announcement.formulaValue).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeSocialBots -> announce -> Telegram API error -> err = " + err)) } else { - socialBot.botInstance.telegramAPI.sendMessage(socialBot.botInstance.chatId, code.text).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> setUpSocialBots -> announce -> Telegram API error -> err = " + err)) + socialBot.botInstance.telegramAPI.sendMessage(socialBot.botInstance.chatId, code.text).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeSocialBots -> announce -> Telegram API error -> err = " + err)) } } } catch (err) { - parentLogger.write(MODULE_NAME, "[WARN] initialize -> setUpSocialBots -> announce -> err = " + err.stack); + parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeSocialBots -> announce -> err = " + err.stack); } } } @@ -417,22 +432,35 @@ } } - function setUpTelegramBot(botToken, chatId) { + function finalizeSocialBots() { + if (bot.SESSION.socialBots.bots !== undefined) { + for (let i = 0; i < bot.SESSION.socialBots.bots.length; i++) { + let socialBot = bot.SESSION.socialBots.bots[i] + if (socialBot.type === "Telegram Bot") { + let code = socialBot.code + socialBot.botInstance = finalizeTelegramBot(code.chatId) + } + } + } + } + + function initializeTelegramBot(botToken, chatId) { /* Telegram Bot Initialization */ try { const Telegraf = require('telegraf') let telegramBot telegramBot = new Telegraf(botToken) - telegramBot.start((ctx) => ctx.reply('Hi! I am a Telegram Bot powered by Superalgos')) - telegramBot.help((ctx) => ctx.reply('Send me a sticker')) - telegramBot.on('sticker', (ctx) => ctx.reply('??')) - telegramBot.hears('hi', (ctx) => ctx.reply('Hey there')) + telegramBot.start((ctx) => ctx.reply('Hi! I am the Telegram Bot that will tell you what happens with your trading session.')) + telegramBot.help((ctx) => ctx.reply('I can stop the session if you say STOP.')) + telegramBot.hears('STOP', (ctx) => stopSession()) telegramBot.launch() const Telegram = require('telegraf/telegram') - let telegramAPI + telegramAPI = new Telegram(botToken) - telegramAPI.sendMessage(chatId, bot.SESSION.type + " '" + bot.SESSION.name + "' was Started.").catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> setUpTelegramBot -> Telegram API error -> err = " + err)) + + const messge = bot.SESSION.type + " '" + bot.SESSION.name + "' was started with an initial balance of " + " " + bot.VALUES_TO_USE.initialBalanceA + " " + bot.VALUES_TO_USE.baseAsset + "." + telegramAPI.sendMessage(chatId, messge ).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeTelegramBot -> Telegram API error -> err = " + err)) let botInstance = { telegramBot: telegramBot, @@ -441,10 +469,15 @@ } return botInstance } catch (err) { - parentLogger.write(MODULE_NAME, "[WARN] initialize -> setUpTelegramBot -> err = " + err.stack); + parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeTelegramBot -> err = " + err.stack); } } + + function finalizeTelegramBot(chatId) { + const messge = bot.SESSION.type + " '" + bot.SESSION.name + "' was stopped." + telegramAPI.sendMessage(chatId, messge).catch(err => parentLogger.write(MODULE_NAME, "[WARN] initialize -> initializeTelegramBot -> Telegram API error -> err = " + err)) + } } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); From f786fa014a8c687a7c555705291222512ce63ede Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 12 Apr 2020 11:06:46 +0200 Subject: [PATCH 04/35] Update FileStorage.js --- FileStorage.js | 111 ++++++++++++++++++++++++++----------------------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/FileStorage.js b/FileStorage.js index 6e95fe028..625c78fd5 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -71,12 +71,12 @@ exports.newFileStorage = function newFileStorage(logger) { } callBackFunction(customResponse) return - } else { - fileDoesNotExist = true - logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> File does not exist. Retrying. ') - setTimeout(retry, retryTimeToUse) - return - } + } + + fileDoesNotExist = true + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> File does not exist. Retrying. ') + setTimeout(retry, retryTimeToUse) + return } logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> err = ' + err.stack) @@ -126,11 +126,12 @@ exports.newFileStorage = function newFileStorage(logger) { logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> canUsePrevious -> Could not read the Previous file either. Giving up.') logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> canUsePrevious -> err = ' + err.stack) callBackFunction(global.DEFAULT_FAIL_RESPONSE) - } else { - /* Returning the previous file */ - callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString()) - return - } + return + } + + /* Returning the previous file */ + callBackFunction(global.DEFAULT_OK_RESPONSE, text.toString()) + return } } else { logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Could read the file, but could not parse it as it is not a valid JSON.') @@ -236,51 +237,25 @@ exports.newFileStorage = function newFileStorage(logger) { logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> file = ' + fileLocation + '.Previous.json') logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> err = ' + err.stack) setTimeout(retry, retryTimeToUse) - } else { + return + } - fs.rename(fileLocation, fileLocation + '.Previous.json', onOriginalRenamed) + /* Rename de Original into Previous */ + fs.rename(fileLocation, fileLocation + '.Previous.json', onOriginalRenamed) - function onOriginalRenamed(err) { - if (err) { - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> Error renaming original file -> file = ' + fileLocation) - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> Error renaming original file -> err = ' + err.stack) - setTimeout(retry, retryTimeToUse) - } else { - - fs.rename(fileLocation + '.tmp', fileLocation, onTempRenamed) - - function onTempRenamed(err) { - if (err) { - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> file = ' + fileLocation + '.tmp') - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> err = ' + err.stack) - setTimeout(retry, retryTimeToUse) - } else { - - callBackFunction(global.DEFAULT_OK_RESPONSE) - - } - } - } + function onOriginalRenamed(err) { + let code = '' + if (err) { + code = err.code + } + if (code !== '' && code !== 'ENOENT') { // Unless the file does not exists we do this... + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> Error renaming original file -> file = ' + fileLocation) + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> Error renaming original file -> err = ' + err.stack) + setTimeout(retry, retryTimeToUse) + return } - } - } - } else { - - /* In this case, there is no need to keep a copy of the file being replaced, so we just delete and that's it. */ - - fs.unlink(fileLocation , onUnlinked) - - function onUnlinked(err) { - let code = '' - if (err) { - code = err.code - } - if (code !== '' && code !== 'ENOENT') { - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> file = ' + fileLocation ) - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> err = ' + err.stack) - setTimeout(retry, retryTimeToUse) - } else { + /* Rename de Temp into Original */ fs.rename(fileLocation + '.tmp', fileLocation, onTempRenamed) function onTempRenamed(err) { @@ -293,6 +268,38 @@ exports.newFileStorage = function newFileStorage(logger) { callBackFunction(global.DEFAULT_OK_RESPONSE) } + } + } + } + } else { + + /* In this case, there is no need to keep a copy of the file being replaced, so we just delete and that's it. */ + fs.unlink(fileLocation , onUnlinked) + + function onUnlinked(err) { + let code = '' + if (err) { + code = err.code + } + if (code !== '' && code !== 'ENOENT') { + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> file = ' + fileLocation ) + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error deleting file -> err = ' + err.stack) + setTimeout(retry, retryTimeToUse) + return + } + + /* Rename de Temp into Original */ + fs.rename(fileLocation + '.tmp', fileLocation, onTempRenamed) + + function onTempRenamed(err) { + if (err) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> file = ' + fileLocation + '.tmp') + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onOriginalRenamed -> onTempRenamed -> Error renaming temp file -> err = ' + err.stack) + setTimeout(retry, retryTimeToUse) + } else { + + callBackFunction(global.DEFAULT_OK_RESPONSE) + } } } From e14023b6ef9e66b052736ed2ff1f1aa418d4af13 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 12 Apr 2020 11:54:35 +0200 Subject: [PATCH 05/35] bug fixed --- Session.js | 28 +++++++++++++++++++++++++++- TradingBotProcessMainLoop.js | 12 +++++++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/Session.js b/Session.js index ff5464808..6f6861af5 100644 --- a/Session.js +++ b/Session.js @@ -10,7 +10,7 @@ return thisObject; - function initialize(callBackFunction) { + function initialize(pProcessConfig, callBackFunction) { try { if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } @@ -148,6 +148,7 @@ bot.resumeExecution = false; bot.multiPeriodProcessDatetime = new Date(bot.VALUES_TO_USE.timeRange.initialDatetime.valueOf()) bot.hasTheBotJustStarted = true + pProcessConfig.normalWaitTime = getTimeFrameFromLabel(bot.VALUES_TO_USE.timeFrame) return true } @@ -181,6 +182,9 @@ bot.VALUES_TO_USE.maximumBalanceA = bot.VALUES_TO_USE.maximumBalanceA * balancePercentage / 100 bot.VALUES_TO_USE.maximumBalanceB = bot.VALUES_TO_USE.maximumBalanceB * balancePercentage / 100 + + pProcessConfig.normalWaitTime = getTimeFrameFromLabel(bot.VALUES_TO_USE.timeFrame) + return true } @@ -191,9 +195,31 @@ bot.resumeExecution = false; bot.hasTheBotJustStarted = true bot.multiPeriodProcessDatetime = new Date(bot.VALUES_TO_USE.timeRange.initialDatetime.valueOf()) + pProcessConfig.normalWaitTime = getTimeFrameFromLabel(bot.VALUES_TO_USE.timeFrame) return true } + function getTimeFrameFromLabel(timeFrameLabel) { + + for (let i = 0; i < global.marketFilesPeriods.length; i++) { + let value = global.marketFilesPeriods[i][0] + let label = global.marketFilesPeriods[i][1] + + if (timeFrameLabel === label) { + return value + } + } + + for (let i = 0; i < global.dailyFilePeriods.length; i++) { + let value = global.dailyFilePeriods[i][0] + let label = global.dailyFilePeriods[i][1] + + if (timeFrameLabel === label) { + return value + } + } + } + function setValuesToUse(message) { /* Base on the information received we will determined which values ultimatelly are going to be used, diff --git a/TradingBotProcessMainLoop.js b/TradingBotProcessMainLoop.js index 81fb60613..2fbb349f8 100644 --- a/TradingBotProcessMainLoop.js +++ b/TradingBotProcessMainLoop.js @@ -43,7 +43,7 @@ if (bot.definedByUI === true) { /* The code of the bot is defined at the UI. No need to load a file with the code. */ - session.initialize(onSessionInitialized) + session.initialize(processConfig, onSessionInitialized) function onSessionInitialized(err) { callBackFunction(err); @@ -902,12 +902,18 @@ waitTime = processConfig.normalWaitTime } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (waitTime / 1000) + " seconds."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (waitTime / 1000 / 60) + " minute/s."); } if (processConfig.deadWaitTime > 0) { checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.deadWaitTime, bot.loopCounter); } nextLoopTimeoutHandle = setTimeout(loop, waitTime); - processHeartBeat(undefined, undefined, "Waiting " + waitTime / 1000 + " seconds for next execution.") + let waitingTime = waitTime / 1000 / 60 + let label = 'minute/s' + if (waitingTime < 1) { + waitingTime = waitTime / 1000 + label = 'seconds' + } + processHeartBeat(undefined, undefined, "Waiting " + waitingTime + " " + label + " for next execution.") if (global.WRITE_LOGS_TO_FILES === 'true') { logger.persist(); } From 42347d50f956e1d2dafd6bef58f5c4b52163afe4 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 12 Apr 2020 12:30:21 +0200 Subject: [PATCH 06/35] Update Session.js --- Session.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Session.js b/Session.js index 6f6861af5..a2fc4d83a 100644 --- a/Session.js +++ b/Session.js @@ -115,11 +115,11 @@ function checkDatetimes() { if (bot.VALUES_TO_USE.timeRange.initialDatetime.valueOf() < (new Date()).valueOf()) { - if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> startLiveTrading -> Overriding initialDatetime with present datetime because " + bot.VALUES_TO_USE.timeRange.initialDatetime + " is in the past."); } + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> checkDatetimes -> Overriding initialDatetime with present datetime because " + bot.VALUES_TO_USE.timeRange.initialDatetime + " is in the past."); } bot.VALUES_TO_USE.timeRange.initialDatetime = new Date() } if (bot.VALUES_TO_USE.timeRange.finalDatetime.valueOf() < (new Date()).valueOf()) { - if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> startLiveTrading -> Overriding finalDatetime with present datetime plus one year because " + bot.VALUES_TO_USE.timeRange.finalDatetime + " is in the past."); } + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> checkDatetimes -> Overriding finalDatetime with present datetime plus one year because " + bot.VALUES_TO_USE.timeRange.finalDatetime + " is in the past."); } bot.VALUES_TO_USE.timeRange.finalDatetime = new Date() + ONE_YEAR_IN_MILISECONDS } } @@ -148,14 +148,14 @@ bot.resumeExecution = false; bot.multiPeriodProcessDatetime = new Date(bot.VALUES_TO_USE.timeRange.initialDatetime.valueOf()) bot.hasTheBotJustStarted = true - pProcessConfig.normalWaitTime = getTimeFrameFromLabel(bot.VALUES_TO_USE.timeFrame) + pProcessConfig.liveWaitTime = getTimeFrameFromLabel(bot.VALUES_TO_USE.timeFrame) return true } function startFordwardTesting(message) { if (process.env.KEY === undefined || process.env.SECRET === undefined) { - if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> startLiveTrading -> Key name or Secret not provided, not possible to run the process in Forward Testing mode."); } + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> startFordwardTesting -> Key name or Secret not provided, not possible to run the process in Forward Testing mode."); } console.log("Key 'codeName' or 'secret' not provided. Plese check that and try again.") return } From ac4f0309f87ce1dc3c6923ed0031be820c47546a Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 13 Apr 2020 17:44:31 +0200 Subject: [PATCH 07/35] Configuration Simplification --- MultiPeriodDaily.js | 154 ++++++++++++++++++++----------------------- MultiPeriodMarket.js | 16 ----- Root.js | 50 ++++++++++---- 3 files changed, 110 insertions(+), 110 deletions(-) diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 01850e771..75eb7ab7f 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -115,95 +115,75 @@ let reportKey; let statusReport; - if (processConfig.framework.startDate.fixedDate !== undefined) { + /* + We look first for the bot who knows the begining of the market in order to get when the market starts. + */ - /* The starting date is fixed, we will start from there. */ - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> We have got a user defined startDate. -> startDate = " + processConfig.framework.startDate.fixedDate); } - contextVariables.dateBeginOfMarket = new Date(processConfig.framework.startDate.fixedDate); - - } else { - - /* - We look first for the bot who knows the begining of the market in order to get when the market starts. - */ - - statusReport = statusDependencies.reportsByMainUtility.get("Market Starting Point") - - if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Market Starting Point -> Status Report does not exist or Market Starting Point not defined. Retrying Later. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - if (statusReport.status === "Status Report is corrupt.") { - logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - thisReport = statusReport.file; - - if (thisReport.beginingOfMarket === undefined) { - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); - logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet."); - - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Dependency does not exist." - } - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); - callBackFunction(customOK); - return; - } - - contextVariables.dateBeginOfMarket = new Date(thisReport.beginingOfMarket.year + "-" + thisReport.beginingOfMarket.month + "-" + thisReport.beginingOfMarket.days + " " + thisReport.beginingOfMarket.hours + ":" + thisReport.beginingOfMarket.minutes + GMT_SECONDS); + statusReport = statusDependencies.reportsByMainUtility.get("Market Starting Point") + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Market Starting Point -> Status Report does not exist or Market Starting Point not defined. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; } - if (processConfig.framework.endDate.fixedDate !== undefined) { - - /* The ending date is fixed, we will end there. */ - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> We have got a user defined endDate. -> endDate = " + processConfig.framework.endDate.fixedDate); } - contextVariables.dateEndOfMarket = new Date(processConfig.framework.endDate.fixedDate); - - } else { - - /* - Here we get the status report from the bot who knows which is the end of the market. - */ - - statusReport = statusDependencies.reportsByMainUtility.get("Market Ending Point") - - if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Market Ending Point -> Status Report does not exist or Market Ending Point not defined. Retrying Later. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - if (statusReport.status === "Status Report is corrupt.") { - logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - thisReport = statusReport.file; - - if (thisReport.lastFile === undefined) { - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); - - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Dependency not ready." - } - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); - callBackFunction(customOK); - return; - } - - contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf()); + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; } + thisReport = statusReport.file; + + if (thisReport.beginingOfMarket === undefined) { + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); + logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet."); + + let customOK = { + result: global.CUSTOM_OK_RESPONSE.result, + message: "Dependency does not exist." + } + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); + callBackFunction(customOK); + return; + } + + contextVariables.dateBeginOfMarket = new Date(thisReport.beginingOfMarket.year + "-" + thisReport.beginingOfMarket.month + "-" + thisReport.beginingOfMarket.days + " " + thisReport.beginingOfMarket.hours + ":" + thisReport.beginingOfMarket.minutes + GMT_SECONDS); + + /* + Here we get the status report from the bot who knows which is the end of the market. + */ + + statusReport = statusDependencies.reportsByMainUtility.get("Market Ending Point") + + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Market Ending Point -> Status Report does not exist or Market Ending Point not defined. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + thisReport = statusReport.file; + + if (thisReport.lastFile === undefined) { + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); + + let customOK = { + result: global.CUSTOM_OK_RESPONSE.result, + message: "Dependency not ready." + } + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); + callBackFunction(customOK); + return; + } + + contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf()); + /* Finally we get our own Status Report. */ statusReport = statusDependencies.reportsByMainUtility.get("Self Reference") @@ -434,6 +414,14 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } let dependency = dataDependencies.nodeArray[dependencyIndex]; + + if (dependency === undefined ) { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> You need to add at least one Data Dependency to the process Multi Period Daily. Aborting process." ); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + let dataset = datasets[dependencyIndex]; let previousFile; diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index 4df897da6..a94bff1e7 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -90,22 +90,6 @@ dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process. }; - if (processConfig.framework.startDate.fixedDate !== undefined) { - - /* The starting date is fixed, we will start from there. */ - - contextVariables.dateBeginOfMarket = new Date(processConfig.framework.startDate.fixedDate); - - } - - if (processConfig.framework.endDate.fixedDate !== undefined) { - - /* The ending date is fixed, we will end there. */ - - contextVariables.dateEndOfMarket = new Date(processConfig.framework.endDate.fixedDate); - - } - processTimeFrames(); function processTimeFrames() { diff --git a/Root.js b/Root.js index 7ed96f1c7..bd1a498aa 100644 --- a/Root.js +++ b/Root.js @@ -254,20 +254,24 @@ if (processConfig.framework !== undefined) { if (processConfig.framework.name === "Multi-Period-Daily" || processConfig.framework.name === "Multi-Period-Market" || processConfig.framework.name === "Multi-Period") { - processConfig.framework.startDate.resumeExecution = true; - if (processConfig.startMode.noTime !== undefined) { - if (processConfig.startMode.noTime.run === "true") { - if (processConfig.startMode.noTime.beginDatetime !== undefined) { - processConfig.framework.startDate.fixedDate = processConfig.startMode.noTime.beginDatetime; - processConfig.framework.startDate.resumeExecution = true; + if (processConfig.framework.startDate !== undefined) { + processConfig.framework.startDate.resumeExecution = true; + if (processConfig.startMode.noTime !== undefined) { + if (processConfig.startMode.noTime.run === "true") { + if (processConfig.startMode.noTime.beginDatetime !== undefined) { + processConfig.framework.startDate.fixedDate = processConfig.startMode.noTime.beginDatetime; + processConfig.framework.startDate.resumeExecution = true; + } } } } - if (processConfig.startMode.userDefined !== undefined) { - if (processConfig.startMode.userDefined.run === "true") { - if (processConfig.startMode.userDefined.beginDatetime !== undefined) { - processConfig.framework.startDate.fixedDate = processConfig.startMode.userDefined.beginDatetime; - processConfig.framework.startDate.resumeExecution = processConfig.startMode.userDefined.resumeExecution; + if (processConfig.startMode !== undefined) { + if (processConfig.startMode.userDefined !== undefined) { + if (processConfig.startMode.userDefined.run === "true") { + if (processConfig.startMode.userDefined.beginDatetime !== undefined) { + processConfig.framework.startDate.fixedDate = processConfig.startMode.userDefined.beginDatetime; + processConfig.framework.startDate.resumeExecution = processConfig.startMode.userDefined.resumeExecution; + } } } } @@ -276,6 +280,30 @@ try { + if (processConfig.startMode === undefined) { // Default + + botConfig.hasTheBotJustStarted = true; + + switch (botConfig.type) { + case 'Sensor Bot': { + runSensorBot(botConfig, processConfig); + break; + } + case 'Indicator Bot': { + runIndicatorBot(botConfig, processConfig); + break; + } + case 'Trading Bot': { + runTradingBot(botConfig, processConfig); + break; + } + default: { + console.log(logDisplace + "Root : [ERROR] start -> bootingBot -> Unexpected bot type. -> botConfig.type = " + botConfig.type); + } + } + return + } + if (processConfig.startMode.noTime !== undefined) { if (processConfig.startMode.noTime.run === "true") { From 1d9c2dd3f29e999847891da2b770df01ef8f8823 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 13 Apr 2020 17:56:43 +0200 Subject: [PATCH 08/35] Update MultiPeriodDaily.js --- MultiPeriodDaily.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 75eb7ab7f..054f9a37e 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -214,7 +214,7 @@ return; } - if (bot.hasTheBotJustStarted === true && processConfig.framework.startDate.resumeExecution === false) { + if (bot.hasTheBotJustStarted === true && processConfig.framework.startDate !== undefined && processConfig.framework.startDate.resumeExecution === false) { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Starting from the begining because bot has just started and resume execution was true."); } startFromBegining(); From fb0442df30ddd82087ff1bbe6d51d830730412db Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 14 Apr 2020 14:50:13 +0200 Subject: [PATCH 09/35] Better Logging --- DebugLog.js | 2 +- IndicatorBotProcessMainLoop.js | 2 +- SensorBotProcessMainLoop.js | 2 +- TradingBotProcessMainLoop.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DebugLog.js b/DebugLog.js index 5a33a9bf6..efb51d469 100644 --- a/DebugLog.js +++ b/DebugLog.js @@ -47,7 +47,7 @@ exports.newDebugLog = function newDebugLog() { if (date === undefined) { date = thisObject.bot.processDatetime } date = date.getUTCFullYear() + '-' + strPad(date.getUTCMonth() + 1, 2, "0") + '-' + strPad(date.getUTCDate(), 2, "0"); - console.log(new Date().toISOString() + " " + strPad(thisObject.bot.exchange, 20) + " " + strPad(thisObject.bot.market.baseAsset + '/' + thisObject.bot.market.quotedAsset, 10) + " " + strPad(pBot, 20) + " " + strPad(pProcess, 30) + console.log(new Date().toISOString() + " " + strPad(thisObject.bot.exchange, 20) + " " + strPad(thisObject.bot.market.baseAsset + '/' + thisObject.bot.market.quotedAsset, 10) + " " + strPad(pBot, 30) + " " + strPad(pProcess, 30) + " Internal Loop # " + strPad(internalLoopCounter + 1, 8) + " " + strPad(date, 30) + " " + strPad(percentage, 10)) persist(); diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index d24bf9bf9..bf61953d3 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -145,7 +145,7 @@ /* High level log entry */ - console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 30) + " " + pad(bot.process, 30) + " Main Loop # " + pad(Number(bot.loopCounter), 8)) /* We will prepare first the infraestructure needed for the bot to run. There are 3 modules we need to sucessfullly initialize first. */ diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index dbcc04b3f..5a5d55444 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -148,7 +148,7 @@ /* High level log entry */ - console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 30) + " " + pad(bot.process, 30) + " Main Loop # " + pad(Number(bot.loopCounter), 8) ) /* We will prepare first the infraestructure needed for the bot to run. There are 4 modules we need to sucessfullly initialize first. */ diff --git a/TradingBotProcessMainLoop.js b/TradingBotProcessMainLoop.js index 2fbb349f8..76cbc9439 100644 --- a/TradingBotProcessMainLoop.js +++ b/TradingBotProcessMainLoop.js @@ -158,7 +158,7 @@ /* High level log entry */ - console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 30) + " " + pad(bot.process, 30) + " Main Loop # " + pad(Number(bot.loopCounter), 8) + " " + bot.sessionKey) /* Checking if we need to need to emit any event */ From 74c4e08a0f8f320eb1b136fe3e63ad7784b9850b Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 16 Apr 2020 10:00:20 +0200 Subject: [PATCH 10/35] Update TradingBotProcessMainLoop.js --- TradingBotProcessMainLoop.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TradingBotProcessMainLoop.js b/TradingBotProcessMainLoop.js index 76cbc9439..7d7ae1657 100644 --- a/TradingBotProcessMainLoop.js +++ b/TradingBotProcessMainLoop.js @@ -159,7 +159,7 @@ /* High level log entry */ console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 30) + " " + pad(bot.process, 30) - + " Main Loop # " + pad(Number(bot.loopCounter), 8) + " " + bot.sessionKey) + + " Main Loop # " + pad(Number(bot.loopCounter), 8)) /* Checking if we need to need to emit any event */ From fe008522f309ce3d12d078c4366f6ca1d562fd34 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 17 Apr 2020 11:13:12 +0200 Subject: [PATCH 11/35] refactoring --- MultiPeriod.js | 146 ++++++++++++++++++++++++------------------------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/MultiPeriod.js b/MultiPeriod.js index 67ed3d365..8b08acb0b 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -225,7 +225,7 @@ interExecutionMemoryArray = []; - for (let i = 0; i < global.dailyFilePeriods.length; i++) { + for (let i = 0; i < global.dailyFiletimeFrames.length; i++) { let emptyObject = {}; interExecutionMemoryArray.push(emptyObject); } @@ -252,46 +252,46 @@ let n; - periodsLoop(); + timeFramesLoop(); - function periodsLoop() { + function timeFramesLoop() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoop -> Entering function."); } /* - We will iterate through all posible periods. + We will iterate through all posible timeFrames. */ - n = 0 // loop Variable representing each possible period as defined at the periods array. + n = 0 // loop Variable representing each possible period as defined at the timeFrames array. - periodsLoopBody(); + timeFramesLoopBody(); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } - function periodsLoopBody() { + function timeFramesLoopBody() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> Entering function."); } - const timeFrame = global.marketFilesPeriods[n][0]; - const timeFrameLabel = global.marketFilesPeriods[n][1]; + const timeFrame = global.marketFilestimeFrames[n][0]; + const timeFrameLabel = global.marketFilestimeFrames[n][1]; let dependencyIndex = 0; dataFiles = new Map(); if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) { - currentTimeFrame = global.marketFilesPeriods[n][0]; - currentOutputPeriodName = global.marketFilesPeriods[n][1]; + currentTimeFrame = global.marketFilestimeFrames[n][0]; + currentOutputPeriodName = global.marketFilestimeFrames[n][1]; } dependencyLoopBody(); @@ -299,7 +299,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> Entering function."); } let dependency = dataDependencies.nodeArray[dependencyIndex]; let dataset = datasets[dependencyIndex]; @@ -310,7 +310,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> Entering function."); } let fileName = "Data.json"; @@ -328,15 +328,15 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { if (err.message === 'File does not exist.') { logger.write(MODULE_NAME, "[ERROR] The file " + filePath + '/' + fileName + ' does not exist and it is required to continue. This process will retry to read it in a while. In the meantime make yourself sure that the process that generates it has ran properly.'); callBackFunction(global.DEFAULT_RETRY_RESPONSE); } else { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message); callBackFunction(err); } return; @@ -349,20 +349,20 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoop -> dependencyLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoop -> dependencyLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -381,10 +381,10 @@ } else { - let mapKey = global.marketFilesPeriods[n][1]; + let mapKey = global.marketFilestimeFrames[n][1]; multiPeriodDataFiles.set(mapKey, dataFiles) - periodsControlLoop(); + timeFramesControlLoop(); } } @@ -395,22 +395,22 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } - function periodsControlLoop() { + function timeFramesControlLoop() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> periodsControlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesControlLoop -> Entering function."); } n++; - if (n < global.marketFilesPeriods.length) { + if (n < global.marketFilestimeFrames.length) { - periodsLoopBody(); + timeFramesLoopBody(); } else { @@ -419,7 +419,7 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> periodsControlLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesControlLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -531,7 +531,7 @@ } - periodsLoop(); + timeFramesLoop(); } catch (err) { logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> checkStopProcessing -> err = " + err.stack); @@ -539,11 +539,11 @@ } } - function periodsLoop() { + function timeFramesLoop() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoop -> Entering function."); } /* Telling the world we are alive and doing well */ let processingDate = bot.multiPeriodProcessDatetime.getUTCFullYear() + '-' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCMonth() + 1, 2) + '-' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCDate(), 2); @@ -551,46 +551,46 @@ /* - We will iterate through all posible periods. + We will iterate through all posible timeFrames. */ - n = 0 // loop Variable representing each possible period as defined at the periods array. + n = 0 // loop Variable representing each possible period as defined at the timeFrames array. - periodsLoopBody(); + timeFramesLoopBody(); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } - function periodsLoopBody() { + function timeFramesLoopBody() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> Entering function."); } - const timeFrame = global.dailyFilePeriods[n][0]; - const timeFrameLabel = global.dailyFilePeriods[n][1]; + const timeFrame = global.dailyFiletimeFrames[n][0]; + const timeFrameLabel = global.dailyFiletimeFrames[n][1]; - if (processConfig.framework.validPeriods !== undefined) { + if (processConfig.framework.validtimeFrames !== undefined) { let validPeriod = false; - for (let i = 0; i < processConfig.framework.validPeriods.length; i++) { - let period = processConfig.framework.validPeriods[i]; + for (let i = 0; i < processConfig.framework.validtimeFrames.length; i++) { + let period = processConfig.framework.validtimeFrames[i]; if (period === timeFrameLabel) { validPeriod = true } } if (validPeriod === false) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> Discarding period for not being listed as a valid period. -> timeFrameLabel = " + timeFrameLabel); } - periodsControlLoop(); + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> Discarding period for not being listed as a valid period. -> timeFrameLabel = " + timeFrameLabel); } + timeFramesControlLoop(); return; } } if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) { - currentTimeFrame = global.dailyFilePeriods[n][0]; - currentOutputPeriodName = global.dailyFilePeriods[n][1]; + currentTimeFrame = global.dailyFiletimeFrames[n][0]; + currentOutputPeriodName = global.dailyFiletimeFrames[n][1]; } let dependencyIndex = 0; @@ -601,7 +601,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> Entering function."); } let dependency = dataDependencies.nodeArray[dependencyIndex]; let dataset = datasets[dependencyIndex]; @@ -620,7 +620,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } if (dependency.dataSet === "Multi-Period-Market") { @@ -644,8 +644,8 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); } if ((err.message === "File does not exist." && botNeverRan === true) || err.code === 'The specified key does not exist.') { @@ -656,7 +656,7 @@ After that, we will not accept more missing files on any of the dependencies, and if any is missing we will abort the processing. */ - logger.write(MODULE_NAME, "[WARN] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found."); + logger.write(MODULE_NAME, "[WARN] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found."); advanceTime(); return; @@ -671,7 +671,7 @@ if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); callBackFunction(err); return; } @@ -682,13 +682,13 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -697,7 +697,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } if (dependency.dataSet === "Multi-Period-Market") { @@ -720,8 +720,8 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); } if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { @@ -732,7 +732,7 @@ if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Not Ok -> err = " + err.code); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Not Ok -> err = " + err.code); callBackFunction(err); return; } @@ -747,19 +747,19 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoop -> dependencyLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoop -> dependencyLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -778,10 +778,10 @@ } else { - let mapKey = global.dailyFilePeriods[n][1]; + let mapKey = global.dailyFiletimeFrames[n][1]; multiPeriodDataFiles.set(mapKey, dataFiles) - periodsControlLoop(); + timeFramesControlLoop(); } } @@ -793,22 +793,22 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } - function periodsControlLoop() { + function timeFramesControlLoop() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> periodsControlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesControlLoop -> Entering function."); } n++; - if (n < global.dailyFilePeriods.length) { + if (n < global.dailyFiletimeFrames.length) { - periodsLoopBody(); + timeFramesLoopBody(); } else { @@ -817,13 +817,13 @@ if (currentTimeFrame !== undefined) { callTheBot(); } else { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsControlLoop -> Time Frame not Recognized. Can not continue." ); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesControlLoop -> Time Frame not Recognized. Can not continue." ); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> periodsControlLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesControlLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -855,7 +855,7 @@ botNeverRan = false; - if (currentTimeFrame > global.dailyFilePeriods[0][0]) { + if (currentTimeFrame > global.dailyFiletimeFrames[0][0]) { writeMarketStatusReport(onMarketStatusReport) } else { From 9ad952a3b3326ebf76a92620a56e0d23d5a18eb1 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 17 Apr 2020 21:02:50 +0200 Subject: [PATCH 12/35] Single File Implemented at Multi Period --- DataDependencies.js | 15 +- DataSet.js | 63 ++++---- MultiPeriod.js | 351 +++++++++++++++++++++++++++---------------- MultiPeriodDaily.js | 48 ++---- MultiPeriodMarket.js | 40 ++--- Session.js | 1 + TradingBot.js | 22 ++- 7 files changed, 292 insertions(+), 248 deletions(-) diff --git a/DataDependencies.js b/DataDependencies.js index 084d4704e..f5ce0e3d2 100644 --- a/DataDependencies.js +++ b/DataDependencies.js @@ -8,12 +8,10 @@ let thisObject = { nodeArray: undefined, - dataSets: new Map(), - initialize: initialize, - keys: [] + dataSetsModulesArray: [], + initialize: initialize }; - return thisObject; function initialize(callBackFunction) { @@ -77,14 +75,7 @@ addCount++; - let key; - - key = thisObject.nodeArray[i].dataMine + "-" + thisObject.nodeArray[i].bot + "-" + thisObject.nodeArray[i].product + "-" + thisObject.nodeArray[i].dataSet + "-" + thisObject.nodeArray[i].dataSetVersion; - - thisObject.keys.push(key); - thisObject.dataSets.set(key, dataSetModule); - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> addDataSet -> DataSet added to Map. -> key = " + key); } + thisObject.dataSetsModulesArray.push(dataSetModule); if (addCount === thisObject.nodeArray.length) { if (alreadyCalledBack === false) { diff --git a/DataSet.js b/DataSet.js index ee04b4a21..434a86ce1 100644 --- a/DataSet.js +++ b/DataSet.js @@ -5,13 +5,12 @@ exports.newDataSet = function newDataSet(BOT, logger) { let bot = BOT; let thisObject = { + node: undefined, initialize: initialize, getTextFile: getTextFile, createTextFile: createTextFile }; - let dataDependencyNode; - /* Storage account to be used here. */ const FILE_STORAGE = require('./FileStorage.js'); @@ -19,77 +18,64 @@ exports.newDataSet = function newDataSet(BOT, logger) { return thisObject; - function initialize(pDataDependencyNode, callBackFunction) { + function initialize(dataDependency, callBackFunction) { try { if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } - dataDependencyNode = pDataDependencyNode; - logger.fileName = MODULE_NAME + "." + dataDependencyNode.type + "." + dataDependencyNode.name + "." + dataDependencyNode.id; + thisObject.node = dataDependency.referenceParent; + logger.fileName = MODULE_NAME + "." + thisObject.node.type + "." + thisObject.node.name + "." + thisObject.node.id; /* Some very basic validations that we have all the information needed. */ - if (dataDependencyNode.referenceParent === undefined) { - logger.write(MODULE_NAME, "[ERROR] initialize -> Data Dependency without Reference Parent. Data Dependency = " + JSON.stringify(dataDependencyNode)); + if (thisObject.node === undefined) { + logger.write(MODULE_NAME, "[ERROR] initialize -> Data Dependency without Reference Parent -> dataDependency = " + JSON.stringify(dataDependency)); callBackFunction(global.DEFAULT_FAIL_RESPONSE); return } - if (dataDependencyNode.referenceParent.code.codeName === undefined) { - logger.write(MODULE_NAME, "[ERROR] initialize -> Dataset witn no codeName defined. Product Dataset = " + JSON.stringify(dataDependencyNode.referenceParent)); + if (thisObject.node.code.codeName === undefined) { + logger.write(MODULE_NAME, "[ERROR] initialize -> Dataset witn no codeName defined -> Product Dataset = " + JSON.stringify(thisObject.node)); callBackFunction(global.DEFAULT_FAIL_RESPONSE); return } - if (dataDependencyNode.referenceParent.parentNode === undefined) { - logger.write(MODULE_NAME, "[ERROR] initialize -> Dataset not attached to a Product Definition. Dataset = " + JSON.stringify(dataDependencyNode.referenceParent)); + if (thisObject.node.parentNode === undefined) { + logger.write(MODULE_NAME, "[ERROR] initialize -> Dataset not attached to a Product Definition -> Dataset = " + JSON.stringify(thisObject.node)); callBackFunction(global.DEFAULT_FAIL_RESPONSE); return } - if (dataDependencyNode.referenceParent.parentNode.code.codeName === undefined) { - logger.write(MODULE_NAME, "[ERROR] initialize -> Product Definition witn no codeName defined. Product Definition = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode)); + if (thisObject.node.parentNode.code.codeName === undefined) { + logger.write(MODULE_NAME, "[ERROR] initialize -> Product Definition witn no codeName defined -> Product Definition = " + JSON.stringify(thisObject.node.parentNode)); callBackFunction(global.DEFAULT_FAIL_RESPONSE); return } - if (dataDependencyNode.referenceParent.parentNode.parentNode === undefined) { - logger.write(MODULE_NAME, "[ERROR] initialize -> Product Definition not attached to a Bot. Product Definition = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode)); + if (thisObject.node.parentNode.parentNode === undefined) { + logger.write(MODULE_NAME, "[ERROR] initialize -> Product Definition not attached to a Bot -> Product Definition = " + JSON.stringify(thisObject.node.parentNode)); callBackFunction(global.DEFAULT_FAIL_RESPONSE); return } - if (dataDependencyNode.referenceParent.parentNode.parentNode.code.codeName === undefined) { - logger.write(MODULE_NAME, "[ERROR] initialize -> Bot witn no codeName defined. Bot = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode.parentNode)); + if (thisObject.node.parentNode.parentNode.code.codeName === undefined) { + logger.write(MODULE_NAME, "[ERROR] initialize -> Bot witn no codeName defined. Bot = " + JSON.stringify(thisObject.node.parentNode.parentNode)); callBackFunction(global.DEFAULT_FAIL_RESPONSE); return } - if (dataDependencyNode.referenceParent.parentNode.parentNode.parentNode === undefined) { - logger.write(MODULE_NAME, "[ERROR] initialize -> Bot not attached to a Data Mine. Bot = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode.parentNode)); + if (thisObject.node.parentNode.parentNode.parentNode === undefined) { + logger.write(MODULE_NAME, "[ERROR] initialize -> Bot not attached to a Data Mine. Bot = " + JSON.stringify(thisObject.node.parentNode.parentNode)); callBackFunction(global.DEFAULT_FAIL_RESPONSE); return } - if (dataDependencyNode.referenceParent.parentNode.parentNode.parentNode.code.codeName === undefined) { - logger.write(MODULE_NAME, "[ERROR] initialize -> Data Mine witn no codeName defined. Data Mine = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode.parentNode.parentNode)); + if (thisObject.node.parentNode.parentNode.parentNode.code.codeName === undefined) { + logger.write(MODULE_NAME, "[ERROR] initialize -> Data Mine witn no codeName defined. Data Mine = " + JSON.stringify(thisObject.node.parentNode.parentNode.parentNode)); callBackFunction(global.DEFAULT_FAIL_RESPONSE); return } - /* Simplifying the access to basic info */ - dataDependencyNode.dataSet = dataDependencyNode.referenceParent.code.codeName - dataDependencyNode.product = dataDependencyNode.referenceParent.parentNode.code.codeName - dataDependencyNode.bot = dataDependencyNode.referenceParent.parentNode.parentNode.code.codeName - dataDependencyNode.dataMine = dataDependencyNode.referenceParent.parentNode.parentNode.parentNode.code.codeName - - /* This stuff is still hardcoded and unresolved. */ - dataDependencyNode.botVersion = { - "major": 1, - "minor": 0 - } - dataDependencyNode.dataSetVersion = "dataSet.V1" - callBackFunction(global.DEFAULT_OK_RESPONSE); } catch (err) { @@ -104,7 +90,10 @@ exports.newDataSet = function newDataSet(BOT, logger) { if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getTextFile -> Entering function."); } - let filePathRoot = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + dataDependencyNode.dataMine + "/" + dataDependencyNode.bot; + let dataMineCodeName = thisObject.node.parentNode.parentNode.parentNode.code.codeName + let botCodeName = thisObject.node.parentNode.parentNode.code.codeName + + let filePathRoot = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + dataMineCodeName + "/" + botCodeName; let filePath = filePathRoot + "/Output/" + pFolderPath; filePath += '/' + pFileName @@ -132,7 +121,7 @@ exports.newDataSet = function newDataSet(BOT, logger) { if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] createTextFile -> pFolderPath = " + pFolderPath); } if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] createTextFile -> pFileName = " + pFileName); } - let ownerId = dataDependencyNode.dataMine + "-" + dataDependencyNode.bot + "-" + dataDependencyNode.botVersion.major + "-" + dataDependencyNode.botVersion.minor + "-" + dataDependencyNode.dataSetVersion; + let ownerId = thisObject.node.dataMine + "-" + thisObject.node.bot + "-" + thisObject.node.botVersion.major + "-" + thisObject.node.botVersion.minor + "-" + thisObject.node.dataSetVersion; let botId = bot.dataMine + "-" + bot.codeName + "-" + bot.version.major + "-" + bot.version.minor + "-" + bot.dataSetVersion; if (ownerId !== botId) { @@ -146,7 +135,7 @@ exports.newDataSet = function newDataSet(BOT, logger) { return; } - let filePathRoot = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + dataDependencyNode.dataMine + "/" + dataDependencyNode.bot; + let filePathRoot = bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset + "/" + thisObject.node.dataMine + "/" + thisObject.node.bot; let filePath = filePathRoot + "/Output/" + pFolderPath + '/' + pFileName; fileStorage.createTextFile(filePath, pFileContent, onFileCreated); diff --git a/MultiPeriod.js b/MultiPeriod.js index 8b08acb0b..328df1718 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -15,8 +15,7 @@ let utilities = UTILITIES.newCloudUtilities(logger); let statusDependencies; - let dataDependencies; - let datasets = []; + let dataDependenciesModule; let dataFiles = new Map(); let multiPeriodDataFiles = new Map(); @@ -39,27 +38,9 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } statusDependencies = pStatusDependencies; - dataDependencies = pDataDependencies; + dataDependenciesModule = pDataDependencies; processConfig = pProcessConfig; - for (let i = 0; i < dataDependencies.nodeArray.length; i++) { - - let key; - let dataset; - let dependency = dataDependencies.nodeArray[i]; - - key = dependency.dataMine + "-" + - dependency.bot + "-" + - dependency.product + "-" + - dependency.dataSet + "-" + - dependency.dataSetVersion - - dataset = dataDependencies.dataSets.get(key); - - datasets.push(dataset); - - } - let USER_BOT_MODULE = require("./TradingBot") botInstance = USER_BOT_MODULE.newTradingBot(bot, logger, UTILITIES, FILE_STORAGE); @@ -72,11 +53,10 @@ } function finalize() { - datasets = undefined dataFiles = undefined multiPeriodDataFiles = undefined statusDependencies = undefined - dataDependencies = undefined + dataDependenciesModule = undefined botInstance.finalize() botInstance = undefined fileStorage = undefined @@ -199,7 +179,7 @@ contextVariables.lastFile = new Date(thisReport.lastFile); interExecutionMemoryArray = thisReport.interExecutionMemoryArray; - processTimeFramesMarketFiles(); + processSingleFiles(); return; } else { @@ -225,12 +205,12 @@ interExecutionMemoryArray = []; - for (let i = 0; i < global.dailyFiletimeFrames.length; i++) { + for (let i = 0; i < global.dailyFilePeriods.length; i++) { let emptyObject = {}; interExecutionMemoryArray.push(emptyObject); } - processTimeFramesMarketFiles(); + processSingleFiles(); } @@ -244,9 +224,124 @@ } } - function processTimeFramesMarketFiles() { + function processSingleFiles() { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> Entering function."); } + + try { + let dependencyIndex = 0; + dataFiles = new Map(); + + dependencyLoopBody(); + + function dependencyLoopBody() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyLoopBody -> Entering function."); } + + let dependency = dataDependenciesModule.nodeArray[dependencyIndex]; + let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]; + + getFile(); + + function getFile() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyLoopBody -> getFile -> Entering function."); } + + let fileName = "Data.json"; + + if (datasetModule.node.code.type !== "Single File") { + + dependencyControlLoop(); + return + } + + let filePath = datasetModule.node.parentNode.code.codeName + '/' + datasetModule.node.code.codeName; + + datasetModule.getTextFile(filePath, fileName, onFileReceived); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + if (err.message === 'File does not exist.') { + logger.write(MODULE_NAME, "[ERROR] The file " + filePath + '/' + fileName + ' does not exist and it is required to continue. This process will retry to read it in a while. In the meantime make yourself sure that the process that generates it has ran properly.'); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + } else { + logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message); + callBackFunction(err); + } + return; + } + + let dataFile = JSON.parse(text); + dataFiles.set(dependency.id, dataFile); + + dependencyControlLoop(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> dependencyLoopBody -> getFile -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> timeFramesLoop -> dependencyLoopBody -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function dependencyControlLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyControlLoop -> Entering function."); } + + dependencyIndex++; + + if (dependencyIndex < dataDependenciesModule.nodeArray.length) { + + dependencyLoopBody(); + + } else { + + let mapKey = "Single Files" + multiPeriodDataFiles.set(mapKey, dataFiles) + + processMarketFiles(); + + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> dependencyControlLoop -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processSingleFiles -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function processMarketFiles() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> Entering function."); } try { @@ -258,7 +353,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoop -> Entering function."); } /* @@ -272,7 +367,7 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -281,17 +376,17 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> Entering function."); } - const timeFrame = global.marketFilestimeFrames[n][0]; - const timeFrameLabel = global.marketFilestimeFrames[n][1]; + const timeFrame = global.marketFilesPeriods[n][0]; + const timeFrameLabel = global.marketFilesPeriods[n][1]; let dependencyIndex = 0; dataFiles = new Map(); if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) { - currentTimeFrame = global.marketFilestimeFrames[n][0]; - currentOutputPeriodName = global.marketFilestimeFrames[n][1]; + currentTimeFrame = global.marketFilesPeriods[n][0]; + currentOutputPeriodName = global.marketFilesPeriods[n][1]; } dependencyLoopBody(); @@ -299,10 +394,10 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> Entering function."); } - let dependency = dataDependencies.nodeArray[dependencyIndex]; - let dataset = datasets[dependencyIndex]; + let dependency = dataDependenciesModule.nodeArray[dependencyIndex]; + let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]; getFile(); @@ -310,33 +405,33 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> Entering function."); } let fileName = "Data.json"; - if (dependency.dataSet !== "Multi-Period-Market") { + if (dependency.referenceParent.code.codeName !== "Multi-Period-Market") { dependencyControlLoop(); return } - let filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel; + let filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel; - dataset.getTextFile(filePath, fileName, onFileReceived); + datasetModule.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { if (err.message === 'File does not exist.') { logger.write(MODULE_NAME, "[ERROR] The file " + filePath + '/' + fileName + ' does not exist and it is required to continue. This process will retry to read it in a while. In the meantime make yourself sure that the process that generates it has ran properly.'); callBackFunction(global.DEFAULT_RETRY_RESPONSE); } else { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message); callBackFunction(err); } return; @@ -349,20 +444,20 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoop -> dependencyLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoop -> dependencyLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -371,17 +466,17 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> dependencyControlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> dependencyControlLoop -> Entering function."); } dependencyIndex++; - if (dependencyIndex < dataDependencies.nodeArray.length) { + if (dependencyIndex < dataDependenciesModule.nodeArray.length) { dependencyLoopBody(); } else { - let mapKey = global.marketFilestimeFrames[n][1]; + let mapKey = global.marketFilesPeriods[n][1]; multiPeriodDataFiles.set(mapKey, dataFiles) timeFramesControlLoop(); @@ -389,13 +484,13 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> dependencyControlLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> dependencyControlLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -404,36 +499,36 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesMarketFiles -> timeFramesControlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesControlLoop -> Entering function."); } n++; - if (n < global.marketFilestimeFrames.length) { + if (n < global.marketFilesPeriods.length) { timeFramesLoopBody(); } else { - processTimeFramesDailyFiles(); + processDailyFiles(); } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> timeFramesControlLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> timeFramesControlLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesMarketFiles -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processMarketFiles -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } - function processTimeFramesDailyFiles() { + function processDailyFiles() { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> Entering function."); } try { @@ -452,20 +547,20 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> advanceTime -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> Entering function."); } bot.multiPeriodProcessDatetime = new Date(bot.multiPeriodProcessDatetime.valueOf() + ONE_DAY_IN_MILISECONDS); previousDay = new Date(bot.multiPeriodProcessDatetime.valueOf() - ONE_DAY_IN_MILISECONDS); - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> advanceTime -> bot.multiPeriodProcessDatetime = " + bot.multiPeriodProcessDatetime); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> advanceTime -> previousDay = " + previousDay); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> bot.multiPeriodProcessDatetime = " + bot.multiPeriodProcessDatetime); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> previousDay = " + previousDay); } /* Validation that we are not going past the user defined end date. */ if (bot.multiPeriodProcessDatetime.valueOf() >= bot.VALUES_TO_USE.timeRange.finalDatetime.valueOf()) { const logText = "User Defined End Datetime reached @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> advanceTime -> " + logText); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> " + logText); } bot.STOP_SESSION = true callBackFunction(global.DEFAULT_OK_RESPONSE); @@ -478,7 +573,7 @@ if (bot.multiPeriodProcessDatetime.valueOf() > contextVariables.dateEndOfMarket.valueOf() + ONE_DAY_IN_MILISECONDS - 1) { const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> advanceTime -> " + logText); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> advanceTime -> " + logText); } callBackFunction(global.DEFAULT_OK_RESPONSE); return; @@ -488,7 +583,7 @@ checkStopTaskGracefully(); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> advanceTime -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> advanceTime -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -497,7 +592,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> checkStopTaskGracefully -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> checkStopTaskGracefully -> Entering function."); } /* Validation that we dont need to stop. */ @@ -511,7 +606,7 @@ checkStopProcessing(); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> checkStopTaskGracefully -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> checkStopTaskGracefully -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -520,7 +615,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> checkStopProcessing -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> checkStopProcessing -> Entering function."); } /* Validation that we dont need to stop. */ @@ -534,7 +629,7 @@ timeFramesLoop(); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> checkStopProcessing -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> checkStopProcessing -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -543,7 +638,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoop -> Entering function."); } /* Telling the world we are alive and doing well */ let processingDate = bot.multiPeriodProcessDatetime.getUTCFullYear() + '-' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCMonth() + 1, 2) + '-' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCDate(), 2); @@ -561,7 +656,7 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -570,10 +665,10 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> Entering function."); } - const timeFrame = global.dailyFiletimeFrames[n][0]; - const timeFrameLabel = global.dailyFiletimeFrames[n][1]; + const timeFrame = global.dailyFilePeriods[n][0]; + const timeFrameLabel = global.dailyFilePeriods[n][1]; if (processConfig.framework.validtimeFrames !== undefined) { let validPeriod = false; @@ -582,15 +677,15 @@ if (period === timeFrameLabel) { validPeriod = true } } if (validPeriod === false) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> Discarding period for not being listed as a valid period. -> timeFrameLabel = " + timeFrameLabel); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> Discarding period for not being listed as a valid period. -> timeFrameLabel = " + timeFrameLabel); } timeFramesControlLoop(); return; } } if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) { - currentTimeFrame = global.dailyFiletimeFrames[n][0]; - currentOutputPeriodName = global.dailyFiletimeFrames[n][1]; + currentTimeFrame = global.dailyFilePeriods[n][0]; + currentOutputPeriodName = global.dailyFilePeriods[n][1]; } let dependencyIndex = 0; @@ -601,10 +696,10 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> Entering function."); } - let dependency = dataDependencies.nodeArray[dependencyIndex]; - let dataset = datasets[dependencyIndex]; + let dependency = dataDependenciesModule.nodeArray[dependencyIndex]; + let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]; let previousFile; let currentFile; @@ -620,9 +715,9 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } - if (dependency.dataSet === "Multi-Period-Market") { + if (dependency.referenceParent.code.codeName === "Multi-Period-Market") { dependencyControlLoop(); return @@ -631,21 +726,21 @@ let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2); let filePath - if (dependency.dataSet === "Multi-Period-Daily") { - filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel + "/" + dateForPath; + if (dependency.referenceParent.code.codeName === "Multi-Period-Daily") { + filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel + "/" + dateForPath; } else { - filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath; + filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + dateForPath; } let fileName = "Data.json"; - dataset.getTextFile(filePath, fileName, onFileReceived); + datasetModule.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); } if ((err.message === "File does not exist." && botNeverRan === true) || err.code === 'The specified key does not exist.') { @@ -656,7 +751,7 @@ After that, we will not accept more missing files on any of the dependencies, and if any is missing we will abort the processing. */ - logger.write(MODULE_NAME, "[WARN] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found."); + logger.write(MODULE_NAME, "[WARN] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found."); advanceTime(); return; @@ -671,7 +766,7 @@ if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); callBackFunction(err); return; } @@ -682,13 +777,13 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -697,9 +792,9 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } - if (dependency.dataSet === "Multi-Period-Market") { + if (dependency.referenceParent.code.codeName === "Multi-Period-Market") { dependencyControlLoop(); return @@ -707,21 +802,21 @@ let dateForPath = bot.multiPeriodProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCDate(), 2); let filePath - if (dependency.dataSet === "Multi-Period-Daily") { - filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel + "/" + dateForPath; + if (dependency.referenceParent.code.codeName === "Multi-Period-Daily") { + filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel + "/" + dateForPath; } else { - filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath; + filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + dateForPath; } let fileName = "Data.json"; - dataset.getTextFile(filePath, fileName, onFileReceived); + datasetModule.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); } if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { @@ -732,7 +827,7 @@ if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Not Ok -> err = " + err.code); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Not Ok -> err = " + err.code); callBackFunction(err); return; } @@ -747,19 +842,19 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoop -> dependencyLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoop -> dependencyLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -768,17 +863,17 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> dependencyControlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> dependencyControlLoop -> Entering function."); } dependencyIndex++; - if (dependencyIndex < dataDependencies.nodeArray.length) { + if (dependencyIndex < dataDependenciesModule.nodeArray.length) { dependencyLoopBody(); } else { - let mapKey = global.dailyFiletimeFrames[n][1]; + let mapKey = global.dailyFilePeriods[n][1]; multiPeriodDataFiles.set(mapKey, dataFiles) timeFramesControlLoop(); @@ -786,14 +881,14 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> dependencyControlLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> dependencyControlLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -802,11 +897,11 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> timeFramesControlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesControlLoop -> Entering function."); } n++; - if (n < global.dailyFiletimeFrames.length) { + if (n < global.dailyFilePeriods.length) { timeFramesLoopBody(); @@ -817,13 +912,13 @@ if (currentTimeFrame !== undefined) { callTheBot(); } else { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesControlLoop -> Time Frame not Recognized. Can not continue." ); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesControlLoop -> Time Frame not Recognized. Can not continue." ); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> timeFramesControlLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> timeFramesControlLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -832,7 +927,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> Entering function."); } botInstance.start( multiPeriodDataFiles, @@ -846,7 +941,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> onBotFinished -> Entering function."); } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { callBackFunction(err); @@ -855,7 +950,7 @@ botNeverRan = false; - if (currentTimeFrame > global.dailyFiletimeFrames[0][0]) { + if (currentTimeFrame > global.dailyFilePeriods[0][0]) { writeMarketStatusReport(onMarketStatusReport) } else { @@ -866,10 +961,10 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onWritten -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> onBotFinished -> onWritten -> Entering function."); } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); callBackFunction(err); return; } @@ -877,7 +972,7 @@ writeDailyStatusReport(bot.multiPeriodProcessDatetime, onDailyStatusReport); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -885,21 +980,21 @@ function onDailyStatusReport() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onDailyStatusReport -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> onBotFinished -> onDailyStatusReport -> Entering function."); } /* The next run we need the process to continue at the date it finished. */ processConfig.framework.startDate.resumeExecution = true; advanceTime(); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onDailyStatusReport -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> onBotFinished -> onDailyStatusReport -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } function onMarketStatusReport() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> Entering function."); } /* This is where we check if we need reached the user defined end datetime. */ @@ -907,7 +1002,7 @@ if (now.valueOf() >= bot.VALUES_TO_USE.timeRange.finalDatetime.valueOf()) { const logText = "User Defined End Datetime reached @ " + now.getUTCFullYear() + "/" + (now.getUTCMonth() + 1) + "/" + now.getUTCDate() + "."; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> " + logText); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> " + logText); } bot.STOP_SESSION = true callBackFunction(global.DEFAULT_OK_RESPONSE); @@ -917,25 +1012,25 @@ callBackFunction(global.DEFAULT_OK_RESPONSE); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> onBotFinished -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> onBotFinished -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> callTheBot -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> callTheBot -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimeFramesDailyFiles -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 054f9a37e..4e384081d 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -15,8 +15,7 @@ let utilities = UTILITIES.newCloudUtilities(logger); let statusDependencies; - let dataDependencies; - let datasets = []; + let dataDependenciesModule; let dataFiles = new Map; let botInstance; @@ -40,27 +39,9 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } statusDependencies = pStatusDependencies; - dataDependencies = pDataDependencies; + dataDependenciesModule = pDataDependencies; processConfig = pProcessConfig; - for (let i = 0; i < dataDependencies.nodeArray.length; i++) { - - let key; - let dataset; - let dependency = dataDependencies.nodeArray[i]; - - key = dependency.dataMine + "-" + - dependency.bot + "-" + - dependency.product + "-" + - dependency.dataSet + "-" + - dependency.dataSetVersion - - dataset = dataDependencies.dataSets.get(key); - - datasets.push(dataset); - - } - let USER_BOT_MODULE = require("./IndicatorBot") botInstance = USER_BOT_MODULE.newIndicatorBot(bot, logger, UTILITIES, FILE_STORAGE); @@ -73,10 +54,9 @@ } function finalize() { - datasets = undefined dataFiles = undefined statusDependencies = undefined - dataDependencies = undefined + dataDependenciesModule = undefined botInstance = undefined fileStorage = undefined processConfig = undefined @@ -413,7 +393,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } - let dependency = dataDependencies.nodeArray[dependencyIndex]; + let dependency = dataDependenciesModule.nodeArray[dependencyIndex]; if (dependency === undefined ) { @@ -422,7 +402,7 @@ return; } - let dataset = datasets[dependencyIndex]; + let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]; let previousFile; let currentFile; @@ -442,14 +422,14 @@ let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2); let filePath - if (dependency.dataSet === "Multi-Period-Daily") { - filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel + "/" + dateForPath; + if (dependency.referenceParent.code.codeName === "Multi-Period-Daily") { + filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel + "/" + dateForPath; } else { - filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath; + filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + dateForPath; } let fileName = "Data.json"; - dataset.getTextFile(filePath, fileName, onFileReceived); + datasetModule.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { @@ -512,14 +492,14 @@ let dateForPath = bot.multiPeriodDailyProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCDate(), 2); let filePath - if (dependency.dataSet === "Multi-Period-Daily") { - filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel + "/" + dateForPath; + if (dependency.referenceParent.code.codeName === "Multi-Period-Daily") { + filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel + "/" + dateForPath; } else { - filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath; + filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + dateForPath; } let fileName = "Data.json"; - dataset.getTextFile(filePath, fileName, onFileReceived); + datasetModule.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { @@ -588,7 +568,7 @@ dependencyIndex++; - if (dependencyIndex < dataDependencies.nodeArray.length) { + if (dependencyIndex < dataDependenciesModule.nodeArray.length) { dependencyLoopBody(); diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index a94bff1e7..3c8b10f07 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -13,8 +13,7 @@ let utilities = UTILITIES.newCloudUtilities(logger); let statusDependencies; - let dataDependencies; - let datasets = []; + let dataDependenciesModule; let dataFiles = new Map(); let botInstance; @@ -33,27 +32,9 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } statusDependencies = pStatusDependencies; - dataDependencies = pDataDependencies; + dataDependenciesModule = pDataDependencies; processConfig = pProcessConfig; - for (let i = 0; i < dataDependencies.nodeArray.length; i++) { - - let key; - let dataset; - let dependency = dataDependencies.nodeArray[i]; - - key = dependency.dataMine + "-" + - dependency.bot + "-" + - dependency.product + "-" + - dependency.dataSet + "-" + - dependency.dataSetVersion - - dataset = dataDependencies.dataSets.get(key); - - datasets.push(dataset); - - } - let USER_BOT_MODULE = require("./IndicatorBot") botInstance = USER_BOT_MODULE.newIndicatorBot(bot, logger, UTILITIES, FILE_STORAGE); @@ -66,10 +47,9 @@ } function finalize() { - datasets = undefined dataFiles = undefined statusDependencies = undefined - dataDependencies = undefined + dataDependenciesModule = undefined botInstance = undefined processConfig = undefined thisObject = undefined @@ -145,8 +125,8 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } - let dependency = dataDependencies.nodeArray[dependencyIndex]; - let dataset = datasets[dependencyIndex]; + let dependency = dataDependenciesModule.nodeArray[dependencyIndex]; + let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]; getFile(); @@ -160,12 +140,12 @@ let fileName = "Data.json"; let filePath - if (dependency.dataSet === "Multi-Period-Market") { - filePath = dependency.product + '/' + dependency.dataSet + "/" + timeFrameLabel; + if (dependency.referenceParent.code.codeName === "Multi-Period-Market") { + filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel; } else { - filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath; + filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + dateForPath; } - dataset.getTextFile(filePath, fileName, onFileReceived); + datasetModule.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { @@ -214,7 +194,7 @@ dependencyIndex++; - if (dependencyIndex < dataDependencies.nodeArray.length) { + if (dependencyIndex < dataDependenciesModule.nodeArray.length) { dependencyLoopBody(); diff --git a/Session.js b/Session.js index a2fc4d83a..8de5c2c7a 100644 --- a/Session.js +++ b/Session.js @@ -459,6 +459,7 @@ } function finalizeSocialBots() { + if (bot.SESSION.socialBots === undefined) {return} if (bot.SESSION.socialBots.bots !== undefined) { for (let i = 0; i < bot.SESSION.socialBots.bots.length; i++) { let socialBot = bot.SESSION.socialBots.bots[i] diff --git a/TradingBot.js b/TradingBot.js index 833b16693..1335f875c 100644 --- a/TradingBot.js +++ b/TradingBot.js @@ -55,13 +55,6 @@ let chart = {} let mainDependency = {} - let processingDailyFiles - - if (currentDay !== undefined) { - processingDailyFiles = true - } else { - processingDailyFiles = false - } /* The first phase here is about checking that we have everything we need at the definition level. */ let dataDependencies = bot.processNode.referenceParent.processDependencies.dataDependencies @@ -103,6 +96,21 @@ } } + /* Single Files */ + + let dataFiles = multiPeriodDataFiles.get('Single Files') + let products = {} + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Inflating Data Files from Single Files."); } + + if (dataFiles !== undefined) { + commons.inflateDatafiles(dataFiles, dataDependencies, products, mainDependency, timeFrame) + + let propertyName = 'atAnyTimeFrame'; + chart[propertyName] = products + } + + /* Simulation */ const TRADING_SIMULATION = require('./TradingSimulation.js'); let tradingSimulation = TRADING_SIMULATION.newTradingSimulation(bot, logger, UTILITIES); From c43f6e6727be825fa6285c070d057d9ad415eb13 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 17 Apr 2020 21:43:12 +0200 Subject: [PATCH 13/35] Single File Implementation finished --- TradingSimulation.js | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/TradingSimulation.js b/TradingSimulation.js index 471ae162d..a97fd1bc4 100644 --- a/TradingSimulation.js +++ b/TradingSimulation.js @@ -526,6 +526,7 @@ for (let k = 0; k < dataDependencies.length; k++) { let dataDependencyNode = dataDependencies[k] + if (dataDependencyNode.referenceParent.code.codeName !== 'Multi-Period-Daily') { continue } let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName let pluralVariableName = dataDependencyNode.referenceParent.parentNode.code.pluralVariableName let currentElement = getElement(thisChart[pluralVariableName], candle, 'Daily' + '-' + mapKey + '-' + pluralVariableName) @@ -534,6 +535,7 @@ } } + /* Finding the Current Element on Market Files */ for (let j = 0; j < global.marketFilesPeriods.length; j++) { let mapKey = marketFilesPeriods[j][1] @@ -542,6 +544,7 @@ for (let k = 0; k < dataDependencies.length; k++) { let dataDependencyNode = dataDependencies[k] + if (dataDependencyNode.referenceParent.code.codeName !== 'Multi-Period-Market') {continue} let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName let pluralVariableName = dataDependencyNode.referenceParent.parentNode.code.pluralVariableName let currentElement = getElement(thisChart[pluralVariableName], candle, 'Market' + '-' + mapKey + '-' + pluralVariableName) @@ -549,6 +552,29 @@ } } + /* Finding the Current Element on Single Files */ + + function isItInside(elementWithTimestamp, elementWithBeginEnd) { + if (elementWithTimestamp.timestamp >= elementWithBeginEnd.begin && elementWithTimestamp.timestamp <= elementWithBeginEnd.end) { + return true + } else { + return false + } + } + + let propertyName = 'atAnyTimeFrame' + let thisChart = chart[propertyName] + + for (let k = 0; k < dataDependencies.length; k++) { + let dataDependencyNode = dataDependencies[k] + if (dataDependencyNode.referenceParent.code.codeName !== 'Single-File') { continue } + let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName + let pluralVariableName = dataDependencyNode.referenceParent.parentNode.code.pluralVariableName + let elementArray = thisChart[pluralVariableName] + let currentElement = elementArray[elementArray.length - 1] + thisChart[singularVariableName] = currentElement + } + /* While we are processing the previous day. */ let positionedAtYesterday = false if (currentDay) { From 6ba783b74eded5c85a249a6264323febaf982f38 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 18 Apr 2020 08:03:26 +0200 Subject: [PATCH 14/35] Update MultiPeriod.js --- MultiPeriod.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MultiPeriod.js b/MultiPeriod.js index 328df1718..ff2c5d19c 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -253,7 +253,7 @@ let fileName = "Data.json"; - if (datasetModule.node.code.type !== "Single File") { + if (datasetModule.node.code.codeName !== "Single-File") { dependencyControlLoop(); return From 852c6a3d0fbfa3cf39f53ebf26cdae1388da5916 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 18 Apr 2020 13:02:28 +0200 Subject: [PATCH 15/35] Dependency Filters Implemented --- DataDependencies.js | 20 ++++++++++++++++-- MultiPeriod.js | 50 ++++++++++++++++++++++----------------------- Session.js | 1 + 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/DataDependencies.js b/DataDependencies.js index f5ce0e3d2..a833ad6a3 100644 --- a/DataDependencies.js +++ b/DataDependencies.js @@ -9,9 +9,12 @@ let thisObject = { nodeArray: undefined, dataSetsModulesArray: [], + isItADepenency: isItADepenency, initialize: initialize }; + let filter = new Map() + return thisObject; function initialize(callBackFunction) { @@ -42,10 +45,17 @@ callBackFunction(global.DEFAULT_OK_RESPONSE); return; } + + /* Session based dependency filters */ + if (bot.DEPENDENCY_FILTER !== undefined) { + for (let i = 0; i < bot.DEPENDENCY_FILTER.length; i++) { + let key = bot.DEPENDENCY_FILTER[i] + filter.set(key, true) + } + } + /* - For each dependency declared at the nodeArray, we will initialize a DataSet as part of this initialization process. - */ let alreadyCalledBack = false; let addCount = 0; @@ -92,4 +102,10 @@ callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } + + function isItADepenency(timeFrame, product) { + let key = timeFrame + '-' + product + + return filter.get(key) + } }; diff --git a/MultiPeriod.js b/MultiPeriod.js index ff2c5d19c..bc3d0b196 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -71,7 +71,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); } let currentTimeFrame - let currentOutputPeriodName + let currentTimeFrameLabel let market = bot.market; let botNeverRan = true; @@ -246,21 +246,21 @@ getFile(); function getFile() { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processSingleFiles -> dependencyLoopBody -> getFile -> Entering function."); } - let fileName = "Data.json"; - if (datasetModule.node.code.codeName !== "Single-File") { - dependencyControlLoop(); return } - let filePath = datasetModule.node.parentNode.code.codeName + '/' + datasetModule.node.code.codeName; + if (dataDependenciesModule.isItADepenency('atAnyTimeFrame', datasetModule.node.parentNode.code.singularVariableName) !== true) { + dependencyControlLoop(); + return + } + let fileName = "Data.json"; + let filePath = datasetModule.node.parentNode.code.codeName + '/' + datasetModule.node.code.codeName; datasetModule.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { @@ -356,9 +356,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoop -> Entering function."); } /* - We will iterate through all posible timeFrames. - */ n = 0 // loop Variable representing each possible period as defined at the timeFrames array. @@ -386,7 +384,7 @@ if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) { currentTimeFrame = global.marketFilesPeriods[n][0]; - currentOutputPeriodName = global.marketFilesPeriods[n][1]; + currentTimeFrameLabel = global.marketFilesPeriods[n][1]; } dependencyLoopBody(); @@ -402,21 +400,21 @@ getFile(); function getFile() { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processMarketFiles -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> Entering function."); } - let fileName = "Data.json"; - if (dependency.referenceParent.code.codeName !== "Multi-Period-Market") { - dependencyControlLoop(); return } - let filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel; + if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) { + dependencyControlLoop(); + return + } + let fileName = "Data.json"; + let filePath = dependency.referenceParent.parentNode.code.codeName + '/' + dependency.referenceParent.code.codeName + "/" + timeFrameLabel; datasetModule.getTextFile(filePath, fileName, onFileReceived); function onFileReceived(err, text) { @@ -685,7 +683,7 @@ if (bot.VALUES_TO_USE.timeFrame === timeFrameLabel) { currentTimeFrame = global.dailyFilePeriods[n][0]; - currentOutputPeriodName = global.dailyFilePeriods[n][1]; + currentTimeFrameLabel = global.dailyFilePeriods[n][1]; } let dependencyIndex = 0; @@ -712,13 +710,15 @@ } function getPreviousFile() { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } - if (dependency.referenceParent.code.codeName === "Multi-Period-Market") { + if (dependency.referenceParent.code.codeName !== "Multi-Period-Daily") { + dependencyControlLoop(); + return + } + if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) { dependencyControlLoop(); return } @@ -932,7 +932,7 @@ botInstance.start( multiPeriodDataFiles, currentTimeFrame, - currentOutputPeriodName, + currentTimeFrameLabel, bot.multiPeriodProcessDatetime, interExecutionMemoryArray[n], onBotFinished); @@ -954,7 +954,7 @@ writeMarketStatusReport(onMarketStatusReport) } else { - writeDataRanges(currentOutputPeriodName, onWritten); + writeDataRanges(currentTimeFrameLabel, onWritten); } function onWritten(err) { @@ -1035,7 +1035,7 @@ } } - function writeDataRanges(currentOutputPeriodName, callBack) { + function writeDataRanges(currentTimeFrameLabel, callBack) { try { @@ -1046,7 +1046,7 @@ function productLoopBody() { let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.code.codeName; - writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodProcessDatetime, productCodeName, currentOutputPeriodName, controlLoop); + writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodProcessDatetime, productCodeName, currentTimeFrameLabel, controlLoop); } function controlLoop() { @@ -1067,7 +1067,7 @@ } - function writeDataRange(pBegin, pEnd, productCodeName, currentOutputPeriodName, callBack) { + function writeDataRange(pBegin, pEnd, productCodeName, currentTimeFrameLabel, callBack) { try { diff --git a/Session.js b/Session.js index 8de5c2c7a..68a12b7f2 100644 --- a/Session.js +++ b/Session.js @@ -54,6 +54,7 @@ /* We are going to run the Definition comming at the event. */ bot.TRADING_SYSTEM = JSON.parse(message.event.tradingSystem) bot.SESSION = JSON.parse(message.event.session) + bot.DEPENDENCY_FILTER = JSON.parse(message.event.dependencyFilter) /* Set the folderName for logging, reports, context and data output */ let code From d5370220f562e695b25bafac52cec760da73eb27 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 18 Apr 2020 16:21:56 +0200 Subject: [PATCH 16/35] Dependencies filters tested --- MultiPeriod.js | 24 ++++++++++++++++-------- TradingSimulation.js | 10 ++++++++-- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/MultiPeriod.js b/MultiPeriod.js index bc3d0b196..9a54e8978 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -409,8 +409,10 @@ } if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) { - dependencyControlLoop(); - return + if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) { + dependencyControlLoop(); + return + } } let fileName = "Data.json"; @@ -719,8 +721,10 @@ } if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) { - dependencyControlLoop(); - return + if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) { + dependencyControlLoop(); + return + } } let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2); @@ -789,17 +793,21 @@ } function getCurrentFile() { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processDailyFiles -> timeFramesLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } - if (dependency.referenceParent.code.codeName === "Multi-Period-Market") { - + if (dependency.referenceParent.code.codeName !== "Multi-Period-Daily") { dependencyControlLoop(); return } + if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.code.singularVariableName) !== true) { + if (!(bot.VALUES_TO_USE.timeFrame === timeFrameLabel && datasetModule.node.parentNode.code.pluralVariableName === 'candles')) { + dependencyControlLoop(); + return + } + } + let dateForPath = bot.multiPeriodProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCDate(), 2); let filePath if (dependency.referenceParent.code.codeName === "Multi-Period-Daily") { diff --git a/TradingSimulation.js b/TradingSimulation.js index a97fd1bc4..4f6ce4397 100644 --- a/TradingSimulation.js +++ b/TradingSimulation.js @@ -571,7 +571,10 @@ let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName let pluralVariableName = dataDependencyNode.referenceParent.parentNode.code.pluralVariableName let elementArray = thisChart[pluralVariableName] - let currentElement = elementArray[elementArray.length - 1] + let currentElement + if (elementArray !== undefined) { + currentElement = elementArray[elementArray.length - 1] + } thisChart[singularVariableName] = currentElement } @@ -615,6 +618,7 @@ previousLoopingDay = loopingDay.valueOf() /* If any of the needed data dependencies is missing for this particular candle, then we jump the candle*/ + /* for (let k = 0; k < dataDependencies.length; k++) { let dataDependencyNode = dataDependencies[k] let singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName @@ -629,6 +633,7 @@ return } } + */ periods++; days = periods * timeFrame / ONE_DAY_IN_MILISECONDS; @@ -2905,7 +2910,7 @@ } function getElement(pArray, currentCandle, datasetName) { - + if (pArray === undefined) {return} try { let element; for (let i = 0; i < pArray.length; i++) { @@ -2938,6 +2943,7 @@ } catch (err) { logger.write(MODULE_NAME, "[ERROR] runSimulation -> getElement -> datasetName = " + datasetName); + logger.write(MODULE_NAME, "[ERROR] runSimulation -> getElement -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } From 7bb4e5b64ec2d4eaf240ba5b54be37b11b4d094d Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 20 Apr 2020 11:48:35 +0200 Subject: [PATCH 17/35] Update TradingBotProcessMainLoop.js --- TradingBotProcessMainLoop.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TradingBotProcessMainLoop.js b/TradingBotProcessMainLoop.js index 7d7ae1657..461b648a8 100644 --- a/TradingBotProcessMainLoop.js +++ b/TradingBotProcessMainLoop.js @@ -159,7 +159,7 @@ /* High level log entry */ console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 30) + " " + pad(bot.process, 30) - + " Main Loop # " + pad(Number(bot.loopCounter), 8)) + + " Main Loop # " + pad(Number(bot.loopCounter), 8) + " " + bot.processNode.session.type + " " + bot.processNode.session.name) /* Checking if we need to need to emit any event */ From ea519e2525a7580ca2d00fe25f5910a06aa057c8 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 20 Apr 2020 14:56:18 +0200 Subject: [PATCH 18/35] Move to Phase --- TradingSimulation.js | 206 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/TradingSimulation.js b/TradingSimulation.js index 4f6ce4397..9fa497537 100644 --- a/TradingSimulation.js +++ b/TradingSimulation.js @@ -873,6 +873,28 @@ } } } + + /* move to phase events */ + for (let n = 0; n < phase.moveToPhaseEvents.length; n++) { + let moveToPhaseEvent = phase.moveToPhaseEvents[n]; + if (moveToPhaseEvent !== undefined) { + + for (let k = 0; k < moveToPhaseEvent.situations.length; k++) { + + let situation = moveToPhaseEvent.situations[k]; + + for (let m = 0; m < situation.conditions.length; m++) { + + let condition = situation.conditions[m]; + let key = j + '-' + 'openStage' + '-' + 'initialDefinition' + '-' + 'stopLoss' + '-' + p + '-' + n + '-' + k + '-' + m; + + if (condition.javascriptCode !== undefined) { + newCondition(key, condition.javascriptCode, chart); + } + } + } + } + } } } @@ -939,6 +961,28 @@ } } } + + /* move to phase events */ + for (let n = 0; n < phase.moveToPhaseEvents.length; n++) { + let moveToPhaseEvent = phase.moveToPhaseEvents[n]; + if (moveToPhaseEvent !== undefined) { + + for (let k = 0; k < moveToPhaseEvent.situations.length; k++) { + + let situation = moveToPhaseEvent.situations[k]; + + for (let m = 0; m < situation.conditions.length; m++) { + + let condition = situation.conditions[m]; + let key = j + '-' + 'openStage' + '-' + 'initialDefinition' + '-' + 'takeProfit' + '-' + p + '-' + n + '-' + k + '-' + m; + + if (condition.javascriptCode !== undefined) { + newCondition(key, condition.javascriptCode, chart); + } + } + } + } + } } } } @@ -1015,6 +1059,28 @@ } } } + + /* move to phase events */ + for (let n = 0; n < phase.moveToPhaseEvents.length; n++) { + let moveToPhaseEvent = phase.moveToPhaseEvents[n]; + if (moveToPhaseEvent !== undefined) { + + for (let k = 0; k < moveToPhaseEvent.situations.length; k++) { + + let situation = moveToPhaseEvent.situations[k]; + + for (let m = 0; m < situation.conditions.length; m++) { + + let condition = situation.conditions[m]; + let key = j + '-' + 'manageStage' + '-' + 'stopLoss' + '-' + p + '-' + n + '-' + k + '-' + m; + + if (condition.javascriptCode !== undefined) { + newCondition(key, condition.javascriptCode, chart); + } + } + } + } + } } } @@ -1082,6 +1148,28 @@ } } } + + /* move to phase events */ + for (let n = 0; n < phase.moveToPhaseEvents.length; n++) { + let moveToPhaseEvent = phase.moveToPhaseEvents[n]; + if (moveToPhaseEvent !== undefined) { + + for (let k = 0; k < moveToPhaseEvent.situations.length; k++) { + + let situation = moveToPhaseEvent.situations[k]; + + for (let m = 0; m < situation.conditions.length; m++) { + + let condition = situation.conditions[m]; + let key = j + '-' + 'manageStage' + '-' + 'takeProfit' + '-' + p + '-' + n + '-' + k + '-' + m; + + if (condition.javascriptCode !== undefined) { + newCondition(key, condition.javascriptCode, chart); + } + } + } + } + } } } } @@ -1418,6 +1506,7 @@ if (parentNode !== undefined) { let phase = parentNode.stopLoss.phases[p]; + /* Check the next Phase Event. */ let nextPhaseEvent = phase.nextPhaseEvent; if (nextPhaseEvent !== undefined) { @@ -1461,6 +1550,64 @@ } } } + + /* Check the Move to Phase Events. */ + for (let n = 0; n < phase.moveToPhaseEvents.length; n++) { + let moveToPhaseEvent = phase.moveToPhaseEvents[n]; + if (moveToPhaseEvent !== undefined) { + + for (let k = 0; k < moveToPhaseEvent.situations.length; k++) { + + let situation = moveToPhaseEvent.situations[k]; + let passed = true; + + for (let m = 0; m < situation.conditions.length; m++) { + + let condition = situation.conditions[m]; + let key = j + '-' + stageKey + initialDefinitionKey + '-' + 'stopLoss' + '-' + p + '-' + n + '-' + k + '-' + m; + + let value = false + if (conditions.get(key) !== undefined) { + value = conditions.get(key).value; + } + + if (value === false) { passed = false; } + } + + if (passed) { + + let moveToPhase = moveToPhaseEvent.referenceParent + if (moveToPhase !== undefined) { + for (let q = 0; q < parentNode.stopLoss.phases.length; q++) { + if (parentNode.stopLoss.phases[q].id === moveToPhase.id) { + stopLossPhase = q + 1 + } + } + } else { + moveToPhaseEvent.error = "This Node needs to reference a Phase." + continue + } + + stopLossStage = 'Manage Stage' + if (takeProfitPhase > 0) { + strategyStage = 'Manage Stage' + checkAnnouncements(manageStage, 'Take Profit') + } + + if (processingDailyFiles) { + if (positionedAtYesterday) { + yesterday.stopLossPhase = stopLossPhase; + yesterday.stopLossStage = stopLossStage; + yesterday.strategyStage = strategyStage; + } + } + + checkAnnouncements(moveToPhaseEvent) + return; + } + } + } + } } } @@ -1553,6 +1700,7 @@ let phase = parentNode.takeProfit.phases[p]; if (phase === undefined) {return} // trying to jump to a phase that does not exists. + /* Check the next Phase Event. */ let nextPhaseEvent = phase.nextPhaseEvent; if (nextPhaseEvent !== undefined) { @@ -1596,6 +1744,64 @@ } } } + + /* Check the Move to Phase Events. */ + for (let n = 0; n < phase.moveToPhaseEvents.length; n++) { + let moveToPhaseEvent = phase.moveToPhaseEvents[n]; + if (moveToPhaseEvent !== undefined) { + + for (let k = 0; k < moveToPhaseEvent.situations.length; k++) { + + let situation = moveToPhaseEvent.situations[k]; + let passed = true; + + for (let m = 0; m < situation.conditions.length; m++) { + + let condition = situation.conditions[m]; + let key = j + '-' + stageKey + initialDefinitionKey + '-' + 'takeProfit' + '-' + p + '-' + n + '-' + k + '-' + m; + + let value = false + if (conditions.get(key) !== undefined) { + value = conditions.get(key).value; + } + + if (value === false) { passed = false; } + } + + if (passed) { + + let moveToPhase = moveToPhaseEvent.referenceParent + if (moveToPhase !== undefined) { + for (let q = 0; q < parentNode.takeProfit.phases.length; q++) { + if (parentNode.takeProfit.phases[q].id === moveToPhase.id) { + takeProfitPhase = q + 1 + } + } + } else { + moveToPhaseEvent.error = "This Node needs to reference a Phase." + continue + } + + takeProfitStage = 'Manage Stage' + if (stopLossPhase > 0) { + strategyStage = 'Manage Stage' + checkAnnouncements(manageStage, 'Stop') + } + + if (processingDailyFiles) { + if (positionedAtYesterday) { + yesterday.takeProfitPhase = takeProfitPhase; + yesterday.takeProfitStage = takeProfitStage; + yesterday.strategyStage = strategyStage; + } + } + + checkAnnouncements(moveToPhaseEvent) + return; + } + } + } + } } } From 480ab1b78e95f6d3663ae3f5288fd1189e9732fc Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 23 Apr 2020 13:38:00 +0200 Subject: [PATCH 19/35] fetching data from networknodes --- DataSet.js | 39 ++++++++++++++++++++++++++++++++-- FileStorage.js | 57 +++++++++++++++++++++++++++++++++++++++++++------- server.js | 6 ++++-- 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/DataSet.js b/DataSet.js index 434a86ce1..66ded33de 100644 --- a/DataSet.js +++ b/DataSet.js @@ -6,6 +6,7 @@ exports.newDataSet = function newDataSet(BOT, logger) { let thisObject = { node: undefined, + networkNode: undefined, initialize: initialize, getTextFile: getTextFile, createTextFile: createTextFile @@ -14,7 +15,7 @@ exports.newDataSet = function newDataSet(BOT, logger) { /* Storage account to be used here. */ const FILE_STORAGE = require('./FileStorage.js'); - let fileStorage = FILE_STORAGE.newFileStorage(logger); + let fileStorage return thisObject; @@ -76,7 +77,41 @@ exports.newDataSet = function newDataSet(BOT, logger) { return } - callBackFunction(global.DEFAULT_OK_RESPONSE); + /* Now we will see where do we need to fetch this data from. */ + let network = global.TASK_NETWORK.parentNode.parentNode.parentNode.parentNode.parentNode + let datasetProductDefinition = thisObject.node.parentNode + let datasetNetworkNode + + for (let i = 0; i < network.networkNodes.length; i++) { + let networkNode = network.networkNodes[i] + if (networkNode.dataStorage !== undefined) { + if (networkNode.dataStorage.sessionIndependentData !== undefined) { + for (let j = 0; i < networkNode.dataStorage.sessionIndependentData.exchangeDataProducts.length; i++) { + let exchangeDataProduct = exchangeDataProducts[j] + for (let k = 0; k < exchangeDataProduct.singleMarketData.length; k++) { + let singleMarketData = exchangeDataProduct.singleMarketData[k] + for (let m = 0; m < singleMarketData.dataProducts.length; m++) { + let dataProduct = singleMarketData.dataProducts[m] + if (dataProduct.referenceParent !== undefined) { + let productDefinition = dataProduct.referenceParent + if (datasetProductDefinition.id === productDefinition.id) { + + /* We found where the data is located on the network. */ + thisObject.networkNode = networkNode + fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return + } + } + } + } + } + } + } + } + + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this dataset is located within the network. Check the logs for more info."); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack); diff --git a/FileStorage.js b/FileStorage.js index 625c78fd5..2778bc11c 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -1,7 +1,7 @@ const path = require('path') -exports.newFileStorage = function newFileStorage(logger) { +exports.newFileStorage = function newFileStorage(logger, host, port) { const MODULE_NAME = 'FileStorage' const MAX_RETRY = 30 @@ -51,9 +51,15 @@ exports.newFileStorage = function newFileStorage(logger) { logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> fileLocation: ' + fileLocation) - /* Here we actually write the file. */ - const fs = require('fs') - fs.readFile(fileLocation, onFileRead) + /* Here we actually reaad the file. */ + if (host === undefined) { + /* We read the file from the local file system. */ + const fs = require('fs') + fs.readFile(fileLocation, onFileRead) + } else { + /* We read the file via a web server over http */ + getFileViaHTTP(fileLocation, onFileRead) + } function onFileRead(err, text) { let retryTimeToUse = FAST_RETRY_TIME_IN_MILISECONDS @@ -390,9 +396,8 @@ exports.newFileStorage = function newFileStorage(logger) { } } - /* Function to create folders of missing folders at any path. */ - - function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) { + /* Function to create folders of missing folders at any path. */ + function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) { const sep = '/'; const initDir = path.isAbsolute(targetDir) ? sep : ''; const baseDir = isRelativeToScript ? __dirname : '.'; @@ -420,5 +425,41 @@ exports.newFileStorage = function newFileStorage(logger) { return curDir; }, initDir); - } + } + + function getFileViaHTTP(filePath, callback) { + try { + + let http = require('http'); + let url = 'http://' + host + + ':' + port + + '/Storage/' + + filePath + + http.get(url, onResponse); + + function onResponse(response) { + + const chunks = [] + + response.on('data', onMessegesArrived) + response.on('end', onEnd) + + function onMessegesArrived(chunk) { + chunks.push(chunk) + } + + function onEnd() { + let fileContent = Buffer.concat(chunks).toString('utf8') + + callback(global.DEFAULT_OK_RESPONSE, fileContent) + } + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + abort = true + } + } } diff --git a/server.js b/server.js index 631ed3edb..310145b83 100644 --- a/server.js +++ b/server.js @@ -125,7 +125,8 @@ function preLoader() { global.EVENT_SERVER_CLIENT.raiseEvent('Task Manager - ' + taskId, 'Nodejs Process Ready for Task') function eventReceived(message) { global.TASK_NODE = message - global.TASK_NODE = JSON.parse(message.event.definition) + global.TASK_NODE = JSON.parse(message.event.taskDefinition) + global.TASK_NETWORK = JSON.parse(message.event.networkDefinition) bootLoader() } } catch (err) { @@ -139,7 +140,8 @@ function preLoader() { global.EVENT_SERVER_CLIENT.listenToEvent('Task Server', 'Debug Task Started', undefined, 'Task Server', undefined, startDebugging) function startDebugging(message) { global.TASK_NODE = message - global.TASK_NODE = JSON.parse(message.event.definition) + global.TASK_NODE = JSON.parse(message.event.taskDefinition) + global.TASK_NETWORK = JSON.parse(message.event.networkDefinition) bootLoader() } } catch (err) { From 2c5bcefc39f84cfd181975ccaa5cd093000cdc89 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 24 Apr 2020 12:03:46 +0200 Subject: [PATCH 20/35] Remote Data Implemented --- DataDependencies.js | 13 +++++- DataSet.js | 43 ++++++++++++------ EventServerClient.js | 13 ++++-- FileStorage.js | 13 ++++-- IndicatorBotProcessMainLoop.js | 10 +++-- ProcessExecutionEvents.js | 82 +++++++++++++++++++++++++++++++--- SensorBotProcessMainLoop.js | 5 ++- StatusDependencies.js | 9 ++++ StatusReport.js | 72 +++++++++++++++++++++++------ TradingBotProcessMainLoop.js | 7 ++- server.js | 4 +- 11 files changed, 223 insertions(+), 48 deletions(-) diff --git a/DataDependencies.js b/DataDependencies.js index a833ad6a3..4f646a80c 100644 --- a/DataDependencies.js +++ b/DataDependencies.js @@ -10,7 +10,8 @@ nodeArray: undefined, dataSetsModulesArray: [], isItADepenency: isItADepenency, - initialize: initialize + initialize: initialize, + finalize: finalize }; let filter = new Map() @@ -103,6 +104,16 @@ } } + function finalize() { + for (let i = 0; i < dataSetsModulesArray.length; i++) { + dataSetsModule = dataSetsModulesArray[i] + dataSetModule.finalize() + } + filter = undefined + bot = undefined + thisObject = undefined + } + function isItADepenency(timeFrame, product) { let key = timeFrame + '-' + product diff --git a/DataSet.js b/DataSet.js index 66ded33de..066e6f7fc 100644 --- a/DataSet.js +++ b/DataSet.js @@ -8,6 +8,7 @@ exports.newDataSet = function newDataSet(BOT, logger) { node: undefined, networkNode: undefined, initialize: initialize, + finalize: finalize, getTextFile: getTextFile, createTextFile: createTextFile }; @@ -78,29 +79,36 @@ exports.newDataSet = function newDataSet(BOT, logger) { } /* Now we will see where do we need to fetch this data from. */ - let network = global.TASK_NETWORK.parentNode.parentNode.parentNode.parentNode.parentNode + let network = global.TASK_NETWORK let datasetProductDefinition = thisObject.node.parentNode - let datasetNetworkNode for (let i = 0; i < network.networkNodes.length; i++) { let networkNode = network.networkNodes[i] if (networkNode.dataStorage !== undefined) { if (networkNode.dataStorage.sessionIndependentData !== undefined) { - for (let j = 0; i < networkNode.dataStorage.sessionIndependentData.exchangeDataProducts.length; i++) { - let exchangeDataProduct = exchangeDataProducts[j] + for (let j = 0; j < networkNode.dataStorage.sessionIndependentData.exchangeDataProducts.length; j++) { + let exchangeDataProduct = networkNode.dataStorage.sessionIndependentData.exchangeDataProducts[j] for (let k = 0; k < exchangeDataProduct.singleMarketData.length; k++) { let singleMarketData = exchangeDataProduct.singleMarketData[k] - for (let m = 0; m < singleMarketData.dataProducts.length; m++) { - let dataProduct = singleMarketData.dataProducts[m] - if (dataProduct.referenceParent !== undefined) { - let productDefinition = dataProduct.referenceParent - if (datasetProductDefinition.id === productDefinition.id) { + if (singleMarketData.referenceParent !== undefined) { + let market = singleMarketData.referenceParent + let currentProcessMarket = bot.processNode.marketReference.referenceParent - /* We found where the data is located on the network. */ - thisObject.networkNode = networkNode - fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort); - callBackFunction(global.DEFAULT_OK_RESPONSE); - return + if (currentProcessMarket.id === market.id) { + for (let m = 0; m < singleMarketData.dataProducts.length; m++) { + let dataProduct = singleMarketData.dataProducts[m] + if (dataProduct.referenceParent !== undefined) { + let productDefinition = dataProduct.referenceParent + if (datasetProductDefinition.id === productDefinition.id) { + + /* We found where the data is located on the network. */ + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving data from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); } + + fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return + } + } } } } @@ -119,6 +127,13 @@ exports.newDataSet = function newDataSet(BOT, logger) { } } + function finalize() { + fileStorage = undefined + thisObject.networkNode = undefined + bot = undefined + thisObject = undefined + } + function getTextFile(pFolderPath, pFileName, callBackFunction) { try { diff --git a/EventServerClient.js b/EventServerClient.js index 3695225b2..a5de99f5a 100644 --- a/EventServerClient.js +++ b/EventServerClient.js @@ -1,4 +1,4 @@ -exports.newEventsServerClient = function newEventsServerClient() { +exports.newEventsServerClient = function newEventsServerClient(host, port) { const MODULE = "Task Server " const ERROR_LOG = true @@ -19,7 +19,14 @@ let WEB_SOCKETS_CLIENT const WEB_SOCKET = require('ws') - let port = process.env.WEB_SOCKETS_SERVER_PORT + + if (host === undefined) { + host = 'localhost' + } + if (port === undefined) { + port = process.env.WEB_SOCKETS_SERVER_PORT + } + return thisObject @@ -32,7 +39,7 @@ function setuptWebSockets(callBackFunction) { try { - WEB_SOCKETS_CLIENT = new WEB_SOCKET('ws://localhost:' + port ) + WEB_SOCKETS_CLIENT = new WEB_SOCKET('ws://' + host + ':' + port ) WEB_SOCKETS_CLIENT.onerror = error => { console.log('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> On connection error -> error = ' + JSON.stringify(error)) diff --git a/FileStorage.js b/FileStorage.js index 2778bc11c..7dff0ea37 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -52,7 +52,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> fileLocation: ' + fileLocation) /* Here we actually reaad the file. */ - if (host === undefined) { + if (host === undefined || host === 'localhost') { /* We read the file from the local file system. */ const fs = require('fs') fs.readFile(fileLocation, onFileRead) @@ -124,7 +124,14 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { if (canUsePrevious === true) { logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Could read the file, but could not parse it as it is not a valid JSON. Will try to read the PREVIOUS version instead. -> file = ' + fileLocation) - fs.readFile(fileLocation + '.Previous.json', onPreviousFileRead) + if (host === undefined || host === 'localhost') { + /* We read the file from the local file system. */ + const fs = require('fs') + fs.readFile(fileLocation + '.Previous.json', onPreviousFileRead) + } else { + /* We read the file via a web server over http */ + getFileViaHTTP(fileLocation + '.Previous.json', onPreviousFileRead) + } function onPreviousFileRead(err, text) { @@ -452,7 +459,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { function onEnd() { let fileContent = Buffer.concat(chunks).toString('utf8') - callback(global.DEFAULT_OK_RESPONSE, fileContent) + callback(null, fileContent) } } diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index bf61953d3..f74c6f1af 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -776,10 +776,13 @@ function onFinished(err) { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> onFinished -> Entering function."); } processFramework.finalize() processFramework = undefined + dataDependencies.finalize() + dataDependencies = undefined + statusDependencies.finalize() + statusDependencies = undefined switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { @@ -876,11 +879,12 @@ processExecutionEvents.finish(onFinished); function onFinished(err) { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Entering function."); } + processExecutionEvents.finalize() + processExecutionEvents = undefined + switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Execution finished well."); diff --git a/ProcessExecutionEvents.js b/ProcessExecutionEvents.js index 18ff8a70b..ba44bf7f2 100644 --- a/ProcessExecutionEvents.js +++ b/ProcessExecutionEvents.js @@ -8,27 +8,33 @@ */ const MODULE_NAME = "Process Execution Events"; + logger.fileName = MODULE_NAME let bot = BOT; let thisObject = { + networkNode: undefined, initialize, initialize, + finalize: finalize, start: start, finish: finish }; let processThisDependsOn; // This is the process this process needs to wait for it to finishes in order to start. - let currentProcessKey + let EVENT_SERVER_CLIENT = require('./EventServerClient.js'); + return thisObject; function initialize(callBackFunction) { try { + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } let name = 'Not Depends on any Process' + logger.fileName = MODULE_NAME + "." + name; if (bot.processNode.referenceParent !== undefined) { @@ -48,9 +54,63 @@ } } } + + /* We need to find at which network node is running the process that we need to hear from when it finished. */ + let network = global.TASK_NETWORK + + for (let i = 0; i < network.networkNodes.length; i++) { + let networkNode = network.networkNodes[i] + if (networkNode.dataMining !== undefined) { + for (let j = 0; j < networkNode.dataMining.exchangeTasks.length; j++) { + let exchangeTasks = networkNode.dataMining.exchangeTasks[j] + for (let k = 0; k < exchangeTasks.taskManagers.length; k++) { + let taskManager = exchangeTasks.taskManagers[k] + for (let m = 0; m < taskManager.tasks.length; m++) { + let task = taskManager.tasks[m] + if (task.bot !== undefined) { + for (let n = 0; n < task.bot.processes.length; n++) { + let process = task.bot.processes[n] + if (process.marketReference !== undefined) { + if (process.marketReference.referenceParent !== undefined) { + let market = process.marketReference.referenceParent + let currentProcessMarket = bot.processNode.marketReference.referenceParent + + if (currentProcessMarket.id === market.id) { + if (process.referenceParent !== undefined) { + let processDefinition = process.referenceParent + if (processThisDependsOn.id === processDefinition.id) { + + /* We found where the task that runs the process definition we are waiting for is located on the network. */ + + thisObject.networkNode = networkNode + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Connecting to Websockets Server " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webSocketsPort + '.'); } + + EVENT_SERVER_CLIENT = EVENT_SERVER_CLIENT.newEventsServerClient(networkNode.code.host, networkNode.code.webSocketsPort) + EVENT_SERVER_CLIENT.initialize(onConnected) + + function onConnected() { + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Connected to Websockets Server " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webSocketsPort + '.'); } + + callBackFunction(global.DEFAULT_OK_RESPONSE); + return + } + return + } + } + } + } + } + } + } + } + } + } + } + } + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the task that runs the Process Definition we are waiting for is located within the network. Check the logs for more info."); + return } - - logger.fileName = MODULE_NAME + "." + name; callBackFunction(global.DEFAULT_OK_RESPONSE); } catch (err) { @@ -59,6 +119,16 @@ } } + function finalize() { + EVENT_SERVER_CLIENT.finalize() + EVENT_SERVER_CLIENT = undefined + processThisDependsOn = undefined + currentProcessKey = undefined + thisObject.networkNode = undefined + bot = undefined + thisObject = undefined + } + function start(callBackFunction) { try { @@ -81,9 +151,9 @@ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> waitForDependantProcess -> key = " + key); } if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> waitForDependantProcess -> callerId = " + callerId); } - global.EVENT_SERVER_CLIENT.listenToEvent(key, 'Process Execution Finished', undefined, callerId, responseCallBack, eventsCallBack) + EVENT_SERVER_CLIENT.listenToEvent(key, 'Process Execution Finished', undefined, callerId, responseCallBack, eventsCallBack) - bot.processHeartBeat(undefined, undefined, "Waiting for " + processThisDependsOn.parentNode.parentNode.name + "->" + processThisDependsOn.parentNode.code.codeName + "->" + processThisDependsOn.code.codeName) + bot.processHeartBeat(undefined, undefined, "Waiting for " + thisObject.networkNode.name + "->" + processThisDependsOn.parentNode.parentNode.name + "->" + processThisDependsOn.parentNode.code.codeName + "->" + processThisDependsOn.code.codeName) function responseCallBack(message) { if (message.result !== global.DEFAULT_OK_RESPONSE.result) { @@ -102,7 +172,7 @@ /* We continue the normal flow after we learn the dependent process has finished its execution. */ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> eventsCallBack -> stopListening to Process Execution Finished. "); } if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> eventsCallBack -> subscriptionId = " + subscriptionId); } - global.EVENT_SERVER_CLIENT.stopListening(key, 'Process Execution Finished', subscriptionId) + EVENT_SERVER_CLIENT.stopListening(key, 'Process Execution Finished', subscriptionId) if (message.event.err.result === global.DEFAULT_OK_RESPONSE.result) { /* We emit our own event that the process started. */ diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index 5a5d55444..47c7a4361 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -584,11 +584,12 @@ processExecutionEvents.finish(onFinished); function onFinished(err) { - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Entering function."); } + processExecutionEvents.finalize() + processExecutionEvents = undefined + switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Execution finished well."); diff --git a/StatusDependencies.js b/StatusDependencies.js index 19e7485ca..8dcc1bd67 100644 --- a/StatusDependencies.js +++ b/StatusDependencies.js @@ -11,6 +11,7 @@ statusReports: new Map(), reportsByMainUtility: new Map(), initialize: initialize, + finalize: finalize, keys: [] }; @@ -165,4 +166,12 @@ } } + function finalize() { + statusReports.forEach(forEachStatusDependency) + function forEachStatusDependency(statusDependency) { + statusDependency.finalize() + } + bot = undefined + thisObject = undefined + } }; diff --git a/StatusReport.js b/StatusReport.js index 66839dac9..99fb2fc28 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -13,9 +13,11 @@ let bot = BOT; let thisObject = { + networkNode: undefined, mainUtility: undefined, file: undefined, // Here we have the JSON object representing the file content. initialize: initialize, + finalize: finalize, load: load, save: save, status: undefined @@ -30,7 +32,7 @@ /* Storage account to be used here. */ const FILE_STORAGE = require('./FileStorage.js'); - let fileStorage = FILE_STORAGE.newFileStorage(logger); + let fileStorage let sessionPath = '' @@ -93,17 +95,6 @@ statusDependencyNode.bottype = statusDependencyNode.referenceParent.parentNode.parentNode.type statusDependencyNode.dataMine = statusDependencyNode.referenceParent.parentNode.parentNode.parentNode.code.codeName - /* Let's see if the process is run monthly or not. */ - if (statusDependencyNode.referenceParent.parentNode.code.startMode !== undefined) { - if (statusDependencyNode.referenceParent.parentNode.code.startMode.allMonths !== undefined) { - if (statusDependencyNode.referenceParent.parentNode.code.startMode.oneMonth !== undefined) { - if (statusDependencyNode.referenceParent.parentNode.code.startMode.allMonths.run === "true" || statusDependencyNode.referenceParent.parentNode.code.startMode.oneMonth.run === "true") { - statusDependencyNode.processRunMonthly = true - } - } - } - } - /* We retrieve the report main utility */ if (statusDependencyNode.code !== undefined) { if (statusDependencyNode.code.mainUtility !== undefined) { @@ -122,7 +113,55 @@ sessionPath = bot.SESSION.folderName + "/" } - callBackFunction(global.DEFAULT_OK_RESPONSE); + /* Now we will see where do we need to fetch this status report from. */ + let network = global.TASK_NETWORK + let processThisDependsOn = statusDependencyNode.referenceParent.parentNode + + for (let i = 0; i < network.networkNodes.length; i++) { + let networkNode = network.networkNodes[i] + if (networkNode.dataMining !== undefined) { + for (let j = 0; j < networkNode.dataMining.exchangeTasks.length; j++) { + let exchangeTasks = networkNode.dataMining.exchangeTasks[j] + for (let k = 0; k < exchangeTasks.taskManagers.length; k++) { + let taskManager = exchangeTasks.taskManagers[k] + for (let m = 0; m < taskManager.tasks.length; m++) { + let task = taskManager.tasks[m] + if (task.bot !== undefined) { + for (let n = 0; n < task.bot.processes.length; n++) { + let process = task.bot.processes[n] + if (process.marketReference !== undefined) { + if (process.marketReference.referenceParent !== undefined) { + let market = process.marketReference.referenceParent + let currentProcessMarket = bot.processNode.marketReference.referenceParent + if (currentProcessMarket.id === market.id) { + if (process.referenceParent !== undefined) { + let processDefinition = process.referenceParent + if (processThisDependsOn.id === processDefinition.id) { + + /* We found where the task that runs the process definition this status report depends on and where it is located on the network. */ + + thisObject.networkNode = networkNode + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving status report from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); } + + fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return + } + } + } + } + } + } + } + } + } + } + } + } + + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this status report is located within the network. Check the logs for more info."); + } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack); @@ -130,6 +169,13 @@ } } + function finalize() { + fileStorage = undefined + thisObject.networkNode = undefined + bot = undefined + thisObject = undefined + } + function load(callBackFunction) { try { diff --git a/TradingBotProcessMainLoop.js b/TradingBotProcessMainLoop.js index 461b648a8..463d16fee 100644 --- a/TradingBotProcessMainLoop.js +++ b/TradingBotProcessMainLoop.js @@ -694,6 +694,10 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> onFinished -> Entering function."); } processFramework.finalize() processFramework = undefined + dataDependencies.finalize() + dataDependencies = undefined + statusDependencies.finalize() + statusDependencies = undefined switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { @@ -792,8 +796,9 @@ processExecutionEvents.finish(onFinished); function onFinished(err) { - try { + processExecutionEvents.finalize() + processExecutionEvents = undefined if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Entering function."); } diff --git a/server.js b/server.js index 310145b83..f32a3bc13 100644 --- a/server.js +++ b/server.js @@ -131,7 +131,7 @@ function preLoader() { } } catch (err) { console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE -> ' + err.stack) - console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE)) + console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE).substring(0, 1000)) } } else { /* This process was started not by the Task Manager, but independently (most likely for debugging purposes). In this case we listen to an event with the Task Info that should be emitted at the UI */ @@ -146,7 +146,7 @@ function preLoader() { } } catch (err) { console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE -> ' + err.stack) - console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE)) + console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE).substring(0, 1000)) } } } From fe537dff3b219c7d3246f04b6357001aea855802 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 24 Apr 2020 13:27:20 +0200 Subject: [PATCH 21/35] bugs fixed --- DataDependencies.js | 5 +++-- FileStorage.js | 14 ++++++++++++-- StatusDependencies.js | 3 ++- StatusReport.js | 1 + 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/DataDependencies.js b/DataDependencies.js index 4f646a80c..3ea9ea759 100644 --- a/DataDependencies.js +++ b/DataDependencies.js @@ -105,10 +105,11 @@ } function finalize() { - for (let i = 0; i < dataSetsModulesArray.length; i++) { - dataSetsModule = dataSetsModulesArray[i] + for (let i = 0; i < thisObject.dataSetsModulesArray.length; i++) { + let dataSetModule = thisObject.dataSetsModulesArray[i] dataSetModule.finalize() } + thisObject.dataSetsModulesArray = undefined filter = undefined bot = undefined thisObject = undefined diff --git a/FileStorage.js b/FileStorage.js index 7dff0ea37..199183aa3 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -149,6 +149,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { } else { logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Could read the file, but could not parse it as it is not a valid JSON.') logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> err = ' + err.stack) + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> text.toString() = ' + text.toString()) setTimeout(retry, retryTimeToUse) return } @@ -437,6 +438,9 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { function getFileViaHTTP(filePath, callback) { try { + /* The filePath received is the one that is needed to fetch data from with fs. To do it via http we need to remove the prefix that includes this: ./Data-Storage/ */ + filePath = filePath.substring(15, filePath.length) + let http = require('http'); let url = 'http://' + host + ':' + port + @@ -458,8 +462,14 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { function onEnd() { let fileContent = Buffer.concat(chunks).toString('utf8') - - callback(null, fileContent) + let err = null + if (fileContent === 'The specified key does not exist.') { + err = { + code: "ENOENT" // This is how fs would have returned upon this situation. + } + fileContent = undefined + } + callback(err, fileContent) } } diff --git a/StatusDependencies.js b/StatusDependencies.js index 8dcc1bd67..48b5659a9 100644 --- a/StatusDependencies.js +++ b/StatusDependencies.js @@ -167,10 +167,11 @@ } function finalize() { - statusReports.forEach(forEachStatusDependency) + thisObject.statusReports.forEach(forEachStatusDependency) function forEachStatusDependency(statusDependency) { statusDependency.finalize() } + thisObject.statusReports = undefined bot = undefined thisObject = undefined } diff --git a/StatusReport.js b/StatusReport.js index 99fb2fc28..4648c1a94 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -258,6 +258,7 @@ */ logger.write(MODULE_NAME, "[ERROR] load -> onFileReceived -> Error Parsing the Status report. -> Err = " + err.message); + logger.write(MODULE_NAME, "[WARN] load -> onFileReceived -> Error Parsing the Status report. -> text = " + text); let customFail = { result: global.CUSTOM_FAIL_RESPONSE.result, From 5a2c317e5edd4a27c19129c2a1049b541fe0c6dc Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 24 Apr 2020 13:39:31 +0200 Subject: [PATCH 22/35] Stop gracefully improved --- IndicatorBotProcessMainLoop.js | 6 ++++++ SensorBotProcessMainLoop.js | 6 ++++++ TradingBotProcessMainLoop.js | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index f74c6f1af..2c2401a15 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -252,6 +252,12 @@ switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well."); + + if (global.STOP_TASK_GRACEFULLY === true) { + loopControl() + return + } + initializeStatusDependencies(); return; } diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index 47c7a4361..e90924148 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -254,6 +254,12 @@ switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well."); + + if (global.STOP_TASK_GRACEFULLY === true) { + loopControl() + return + } + initializeStatusDependencies(); return; } diff --git a/TradingBotProcessMainLoop.js b/TradingBotProcessMainLoop.js index 463d16fee..ab3bee5ad 100644 --- a/TradingBotProcessMainLoop.js +++ b/TradingBotProcessMainLoop.js @@ -288,6 +288,12 @@ switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well."); + + if (global.STOP_TASK_GRACEFULLY === true) { + loopControl() + return + } + initializeStatusDependencies(); return; } From c5466103ca03e00654cc7584a8ab91772b30083c Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 24 Apr 2020 15:25:29 +0200 Subject: [PATCH 23/35] Update ProcessExecutionEvents.js --- ProcessExecutionEvents.js | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/ProcessExecutionEvents.js b/ProcessExecutionEvents.js index ba44bf7f2..83a2cab8f 100644 --- a/ProcessExecutionEvents.js +++ b/ProcessExecutionEvents.js @@ -37,26 +37,32 @@ logger.fileName = MODULE_NAME + "." + name; if (bot.processNode.referenceParent !== undefined) { + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because the Process Instance needs to have a Reference Parent."); + return + } - let market = bot.market.baseAsset + '/' + bot.market.quotedAsset - currentProcessKey = bot.processNode.referenceParent.name + "-" + bot.processNode.referenceParent.type + "-" + bot.processNode.referenceParent.id + "-" + bot.exchange + "-" + market + let market = bot.market.baseAsset + '/' + bot.market.quotedAsset + currentProcessKey = bot.processNode.referenceParent.name + "-" + bot.processNode.referenceParent.type + "-" + bot.processNode.referenceParent.id + "-" + bot.exchange + "-" + market - if (bot.processNode.referenceParent.executionStartedEvent !== undefined) { - if (bot.processNode.referenceParent.executionStartedEvent.referenceParent !== undefined) { - if (bot.processNode.referenceParent.executionStartedEvent.referenceParent.parentNode !== undefined) { - processThisDependsOn = bot.processNode.referenceParent.executionStartedEvent.referenceParent.parentNode + if (bot.processNode.referenceParent.executionStartedEvent !== undefined) { + if (bot.processNode.referenceParent.executionStartedEvent.referenceParent !== undefined) { + if (bot.processNode.referenceParent.executionStartedEvent.referenceParent.parentNode !== undefined) { + processThisDependsOn = bot.processNode.referenceParent.executionStartedEvent.referenceParent.parentNode - name = processThisDependsOn.name - if (processThisDependsOn.code.monthlyInstances === true) { - logger.fileName = MODULE_NAME + "." + name; - } - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> " + currentProcessKey + " depends on " + name); } + name = processThisDependsOn.name + if (processThisDependsOn.code.monthlyInstances === true) { + logger.fileName = MODULE_NAME + "." + name; } + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> " + currentProcessKey + " depends on " + name); } } } + } + + if (processThisDependsOn !== undefined) { /* We need to find at which network node is running the process that we need to hear from when it finished. */ - let network = global.TASK_NETWORK + let network = global.TASK_NETWORK for (let i = 0; i < network.networkNodes.length; i++) { let networkNode = network.networkNodes[i] @@ -75,7 +81,7 @@ let market = process.marketReference.referenceParent let currentProcessMarket = bot.processNode.marketReference.referenceParent - if (currentProcessMarket.id === market.id) { + if (currentProcessMarket.id === market.id) { if (process.referenceParent !== undefined) { let processDefinition = process.referenceParent if (processThisDependsOn.id === processDefinition.id) { @@ -98,7 +104,7 @@ } } } - } + } } } } @@ -112,7 +118,7 @@ return } callBackFunction(global.DEFAULT_OK_RESPONSE); - + } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); From 7d6700934c971d6cde6afd51591e7a4533e01357 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 24 Apr 2020 15:50:38 +0200 Subject: [PATCH 24/35] Update ProcessExecutionEvents.js --- ProcessExecutionEvents.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ProcessExecutionEvents.js b/ProcessExecutionEvents.js index 83a2cab8f..8731bf55a 100644 --- a/ProcessExecutionEvents.js +++ b/ProcessExecutionEvents.js @@ -24,6 +24,7 @@ let currentProcessKey let EVENT_SERVER_CLIENT = require('./EventServerClient.js'); + let eventServerClient return thisObject; @@ -36,7 +37,7 @@ let name = 'Not Depends on any Process' logger.fileName = MODULE_NAME + "." + name; - if (bot.processNode.referenceParent !== undefined) { + if (bot.processNode.referenceParent === undefined) { callBackFunction(global.DEFAULT_FAIL_RESPONSE); logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because the Process Instance needs to have a Reference Parent."); return @@ -91,8 +92,8 @@ thisObject.networkNode = networkNode if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Connecting to Websockets Server " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webSocketsPort + '.'); } - EVENT_SERVER_CLIENT = EVENT_SERVER_CLIENT.newEventsServerClient(networkNode.code.host, networkNode.code.webSocketsPort) - EVENT_SERVER_CLIENT.initialize(onConnected) + eventServerClient = EVENT_SERVER_CLIENT.newEventsServerClient(networkNode.code.host, networkNode.code.webSocketsPort) + eventServerClient.initialize(onConnected) function onConnected() { if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Connected to Websockets Server " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webSocketsPort + '.'); } @@ -126,8 +127,11 @@ } function finalize() { - EVENT_SERVER_CLIENT.finalize() - EVENT_SERVER_CLIENT = undefined + if (eventServerClient !== undefined) { + eventServerClient.finalize() + eventServerClient = undefined + } + processThisDependsOn = undefined currentProcessKey = undefined thisObject.networkNode = undefined @@ -157,7 +161,7 @@ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> waitForDependantProcess -> key = " + key); } if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> waitForDependantProcess -> callerId = " + callerId); } - EVENT_SERVER_CLIENT.listenToEvent(key, 'Process Execution Finished', undefined, callerId, responseCallBack, eventsCallBack) + eventServerClient.listenToEvent(key, 'Process Execution Finished', undefined, callerId, responseCallBack, eventsCallBack) bot.processHeartBeat(undefined, undefined, "Waiting for " + thisObject.networkNode.name + "->" + processThisDependsOn.parentNode.parentNode.name + "->" + processThisDependsOn.parentNode.code.codeName + "->" + processThisDependsOn.code.codeName) @@ -178,7 +182,7 @@ /* We continue the normal flow after we learn the dependent process has finished its execution. */ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> eventsCallBack -> stopListening to Process Execution Finished. "); } if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] start -> eventsCallBack -> subscriptionId = " + subscriptionId); } - EVENT_SERVER_CLIENT.stopListening(key, 'Process Execution Finished', subscriptionId) + eventServerClient.stopListening(key, 'Process Execution Finished', subscriptionId) if (message.event.err.result === global.DEFAULT_OK_RESPONSE.result) { /* We emit our own event that the process started. */ From ecc8248af713638f40eea4d7cdce2ee054b93250 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 24 Apr 2020 16:31:55 +0200 Subject: [PATCH 25/35] bug fixed --- DataDependencies.js | 2 +- DataSet.js | 2 +- StatusReport.js | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/DataDependencies.js b/DataDependencies.js index 3ea9ea759..731495e8d 100644 --- a/DataDependencies.js +++ b/DataDependencies.js @@ -70,7 +70,7 @@ function onInitilized(err) { if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] initialize -> onInitilized -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] initialize -> onInitilized -> err = "+ JSON.stringify(err)); alreadyCalledBack = true; callBackFunction(err); diff --git a/DataSet.js b/DataSet.js index 066e6f7fc..a55029ab6 100644 --- a/DataSet.js +++ b/DataSet.js @@ -118,8 +118,8 @@ exports.newDataSet = function newDataSet(BOT, logger) { } } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this dataset is located within the network. Check the logs for more info."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack); diff --git a/StatusReport.js b/StatusReport.js index 4648c1a94..888ec2e67 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -159,9 +159,8 @@ } } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this status report is located within the network. Check the logs for more info."); - + callBackFunction(global.DEFAULT_FAIL_RESPONSE); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack); From 8cccb243b214b216aeb7faabd4ef7b03f2bdc938 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 24 Apr 2020 17:34:44 +0200 Subject: [PATCH 26/35] Update StatusReport.js --- StatusReport.js | 58 +++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/StatusReport.js b/StatusReport.js index 888ec2e67..6f95de543 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -120,32 +120,39 @@ for (let i = 0; i < network.networkNodes.length; i++) { let networkNode = network.networkNodes[i] if (networkNode.dataMining !== undefined) { - for (let j = 0; j < networkNode.dataMining.exchangeTasks.length; j++) { - let exchangeTasks = networkNode.dataMining.exchangeTasks[j] - for (let k = 0; k < exchangeTasks.taskManagers.length; k++) { - let taskManager = exchangeTasks.taskManagers[k] - for (let m = 0; m < taskManager.tasks.length; m++) { - let task = taskManager.tasks[m] - if (task.bot !== undefined) { - for (let n = 0; n < task.bot.processes.length; n++) { - let process = task.bot.processes[n] - if (process.marketReference !== undefined) { - if (process.marketReference.referenceParent !== undefined) { - let market = process.marketReference.referenceParent - let currentProcessMarket = bot.processNode.marketReference.referenceParent - if (currentProcessMarket.id === market.id) { - if (process.referenceParent !== undefined) { - let processDefinition = process.referenceParent - if (processThisDependsOn.id === processDefinition.id) { - /* We found where the task that runs the process definition this status report depends on and where it is located on the network. */ + if (checkThisBranch(networkNode.dataMining) === true) { return } + if (checkThisBranch(networkNode.testingEnvironment) === true) {return} + if (checkThisBranch(networkNode.productionEnvironment) === true) { return } - thisObject.networkNode = networkNode - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving status report from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); } + function checkThisBranch() { + for (let j = 0; j < networkNode.dataMining.exchangeTasks.length; j++) { + let exchangeTasks = networkNode.dataMining.exchangeTasks[j] + for (let k = 0; k < exchangeTasks.taskManagers.length; k++) { + let taskManager = exchangeTasks.taskManagers[k] + for (let m = 0; m < taskManager.tasks.length; m++) { + let task = taskManager.tasks[m] + if (task.bot !== undefined) { + for (let n = 0; n < task.bot.processes.length; n++) { + let process = task.bot.processes[n] + if (process.marketReference !== undefined) { + if (process.marketReference.referenceParent !== undefined) { + let market = process.marketReference.referenceParent + let currentProcessMarket = bot.processNode.marketReference.referenceParent + if (currentProcessMarket.id === market.id) { + if (process.referenceParent !== undefined) { + let processDefinition = process.referenceParent + if (processThisDependsOn.id === processDefinition.id) { - fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort); - callBackFunction(global.DEFAULT_OK_RESPONSE); - return + /* We found where the task that runs the process definition this status report depends on and where it is located on the network. */ + + thisObject.networkNode = networkNode + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving status report from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); } + + fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return true + } } } } @@ -160,6 +167,11 @@ } logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this status report is located within the network. Check the logs for more info."); + logger.write(MODULE_NAME, "[ERROR] initialize -> bot = " + statusDependencyNode.bot); + logger.write(MODULE_NAME, "[ERROR] initialize -> process = " + statusDependencyNode.process); + logger.write(MODULE_NAME, "[ERROR] initialize -> bottype = " + statusDependencyNode.bottype); + logger.write(MODULE_NAME, "[ERROR] initialize -> dataMine = " + statusDependencyNode.dataMine); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); } catch (err) { From 1b7a622b2c68a081033a2babed5472a02a0281bd Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 25 Apr 2020 09:08:42 +0200 Subject: [PATCH 27/35] Bugs Fixed --- DataSet.js | 33 +++++++++++++++++++++++++++++++++ StatusReport.js | 16 +++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/DataSet.js b/DataSet.js index a55029ab6..aa0c6bb1c 100644 --- a/DataSet.js +++ b/DataSet.js @@ -94,6 +94,37 @@ exports.newDataSet = function newDataSet(BOT, logger) { let market = singleMarketData.referenceParent let currentProcessMarket = bot.processNode.marketReference.referenceParent + if (currentProcessMarket.id === market.id) { + for (let m = 0; m < singleMarketData.dataProducts.length; m++) { + let dataProduct = singleMarketData.dataProducts[m] + if (dataProduct.referenceParent !== undefined) { + let productDefinition = dataProduct.referenceParent + if (datasetProductDefinition.id === productDefinition.id) { + + /* We found where the data is located on the network. */ + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving data from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); } + + fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return + } + } + } + } + } + } + } + } + if (networkNode.dataStorage.sessionBasedData !== undefined) { + for (let j = 0; j < networkNode.dataStorage.sessionBasedData.exchangeDataProducts.length; j++) { + let exchangeDataProduct = networkNode.dataStorage.sessionBasedData.exchangeDataProducts[j] + for (let s = 0; s < exchangeDataProduct.sessionReferences.length; s++) { + let sessionReference = exchangeDataProduct.sessionReferences[s] + let singleMarketData = sessionReference.singleMarketData + if (singleMarketData.referenceParent !== undefined) { + let market = singleMarketData.referenceParent + let currentProcessMarket = bot.processNode.marketReference.referenceParent + if (currentProcessMarket.id === market.id) { for (let m = 0; m < singleMarketData.dataProducts.length; m++) { let dataProduct = singleMarketData.dataProducts[m] @@ -119,6 +150,8 @@ exports.newDataSet = function newDataSet(BOT, logger) { } logger.write(MODULE_NAME, "[ERROR] initialize -> Initialization Failed because we could not find where the data of this dataset is located within the network. Check the logs for more info."); + logger.write(MODULE_NAME, "[ERROR] initialize -> Could not find where " + datasetProductDefinition.name + " for " + bot.exchange + " " + bot.market.baseAsset + "/" + bot.market.quotedAsset + " is stored within the network."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); } catch (err) { diff --git a/StatusReport.js b/StatusReport.js index 6f95de543..267889a75 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -125,9 +125,10 @@ if (checkThisBranch(networkNode.testingEnvironment) === true) {return} if (checkThisBranch(networkNode.productionEnvironment) === true) { return } - function checkThisBranch() { - for (let j = 0; j < networkNode.dataMining.exchangeTasks.length; j++) { - let exchangeTasks = networkNode.dataMining.exchangeTasks[j] + function checkThisBranch(branch) { + if (branch === undefined) {return} + for (let j = 0; j < branch.exchangeTasks.length; j++) { + let exchangeTasks = branch.exchangeTasks[j] for (let k = 0; k < exchangeTasks.taskManagers.length; k++) { let taskManager = exchangeTasks.taskManagers[k] for (let m = 0; m < taskManager.tasks.length; m++) { @@ -143,6 +144,15 @@ if (process.referenceParent !== undefined) { let processDefinition = process.referenceParent if (processThisDependsOn.id === processDefinition.id) { + if (process.type === 'Trading Process Instance') { + if (process.session !== undefined) { + if (bot.processNode.session.id !== process.session.id) { + continue + } + } else { + continue + } + } /* We found where the task that runs the process definition this status report depends on and where it is located on the network. */ From 466abf5b4914cca2f0310f8bd69bcec9f8f0c069 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 25 Apr 2020 10:13:37 +0200 Subject: [PATCH 28/35] Update StatusReport.js --- StatusReport.js | 76 ++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/StatusReport.js b/StatusReport.js index 267889a75..0ae6bc288 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -119,50 +119,48 @@ for (let i = 0; i < network.networkNodes.length; i++) { let networkNode = network.networkNodes[i] - if (networkNode.dataMining !== undefined) { + + if (checkThisBranch(networkNode.dataMining) === true) { return } + if (checkThisBranch(networkNode.testingEnvironment) === true) {return} + if (checkThisBranch(networkNode.productionEnvironment) === true) { return } - if (checkThisBranch(networkNode.dataMining) === true) { return } - if (checkThisBranch(networkNode.testingEnvironment) === true) {return} - if (checkThisBranch(networkNode.productionEnvironment) === true) { return } - - function checkThisBranch(branch) { - if (branch === undefined) {return} - for (let j = 0; j < branch.exchangeTasks.length; j++) { - let exchangeTasks = branch.exchangeTasks[j] - for (let k = 0; k < exchangeTasks.taskManagers.length; k++) { - let taskManager = exchangeTasks.taskManagers[k] - for (let m = 0; m < taskManager.tasks.length; m++) { - let task = taskManager.tasks[m] - if (task.bot !== undefined) { - for (let n = 0; n < task.bot.processes.length; n++) { - let process = task.bot.processes[n] - if (process.marketReference !== undefined) { - if (process.marketReference.referenceParent !== undefined) { - let market = process.marketReference.referenceParent - let currentProcessMarket = bot.processNode.marketReference.referenceParent - if (currentProcessMarket.id === market.id) { - if (process.referenceParent !== undefined) { - let processDefinition = process.referenceParent - if (processThisDependsOn.id === processDefinition.id) { - if (process.type === 'Trading Process Instance') { - if (process.session !== undefined) { - if (bot.processNode.session.id !== process.session.id) { - continue - } - } else { + function checkThisBranch(branch) { + if (branch === undefined) {return} + for (let j = 0; j < branch.exchangeTasks.length; j++) { + let exchangeTasks = branch.exchangeTasks[j] + for (let k = 0; k < exchangeTasks.taskManagers.length; k++) { + let taskManager = exchangeTasks.taskManagers[k] + for (let m = 0; m < taskManager.tasks.length; m++) { + let task = taskManager.tasks[m] + if (task.bot !== undefined) { + for (let n = 0; n < task.bot.processes.length; n++) { + let process = task.bot.processes[n] + if (process.marketReference !== undefined) { + if (process.marketReference.referenceParent !== undefined) { + let market = process.marketReference.referenceParent + let currentProcessMarket = bot.processNode.marketReference.referenceParent + if (currentProcessMarket.id === market.id) { + if (process.referenceParent !== undefined) { + let processDefinition = process.referenceParent + if (processThisDependsOn.id === processDefinition.id) { + if (process.type === 'Trading Process Instance') { + if (process.session !== undefined) { + if (bot.processNode.session.id !== process.session.id) { continue } + } else { + continue } - - /* We found where the task that runs the process definition this status report depends on and where it is located on the network. */ - - thisObject.networkNode = networkNode - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving status report from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); } - - fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort); - callBackFunction(global.DEFAULT_OK_RESPONSE); - return true } + + /* We found where the task that runs the process definition this status report depends on and where it is located on the network. */ + + thisObject.networkNode = networkNode + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Retrieving status report from " + networkNode.name + " -> host = " + networkNode.code.host + ' -> port = ' + networkNode.code.webPort + '.'); } + + fileStorage = FILE_STORAGE.newFileStorage(logger, networkNode.code.host, networkNode.code.webPort); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return true } } } From 106256ca64d2cd2404237a30ef020bc02e51621b Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 25 Apr 2020 11:50:34 +0200 Subject: [PATCH 29/35] bug fix --- DataDependencies.js | 1 - FileStorage.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/DataDependencies.js b/DataDependencies.js index 731495e8d..918def0bf 100644 --- a/DataDependencies.js +++ b/DataDependencies.js @@ -112,7 +112,6 @@ thisObject.dataSetsModulesArray = undefined filter = undefined bot = undefined - thisObject = undefined } function isItADepenency(timeFrame, product) { diff --git a/FileStorage.js b/FileStorage.js index 199183aa3..30b69242e 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -463,7 +463,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { function onEnd() { let fileContent = Buffer.concat(chunks).toString('utf8') let err = null - if (fileContent === 'The specified key does not exist.') { + if (fileContent.indexOf('does not exist') >= 0) { err = { code: "ENOENT" // This is how fs would have returned upon this situation. } From 34dba0d0559e993ca2e03f0752057af041c41a7c Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 26 Apr 2020 07:45:31 +0200 Subject: [PATCH 30/35] Update MultiPeriod.js --- MultiPeriod.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MultiPeriod.js b/MultiPeriod.js index 9a54e8978..217e49077 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -124,7 +124,7 @@ thisReport = statusReport.file; if (thisReport.lastFile === undefined) { - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> thisReport = " + JSON.stringify(thisReport)); let customOK = { result: global.CUSTOM_OK_RESPONSE.result, From 57685ff222698c6486d41e11d8d32633a9a7df65 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 26 Apr 2020 16:04:54 +0200 Subject: [PATCH 31/35] Update ExchangeAPI.js --- ExchangeAPI.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ExchangeAPI.js b/ExchangeAPI.js index 4677a2c45..54c3c52e7 100644 --- a/ExchangeAPI.js +++ b/ExchangeAPI.js @@ -38,7 +38,7 @@ 'secret': secret, 'timeout': 30000, 'enableRateLimit': true, - verbose: true, + verbose: false, options: options } From d435c0e158c1caa91cc1b80bb13ed957450ffdd9 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Apr 2020 08:57:12 +0200 Subject: [PATCH 32/35] Improved Handling of Loggers while process is shutting down --- DebugLog.js | 8 +++++++- FileStorage.js | 2 ++ IndicatorBotProcessMainLoop.js | 4 ++++ Root.js | 3 +++ SensorBotProcessMainLoop.js | 4 ++++ TradingBotProcessMainLoop.js | 5 ++++- server.js | 19 +++++++++++++++++-- 7 files changed, 41 insertions(+), 4 deletions(-) diff --git a/DebugLog.js b/DebugLog.js index efb51d469..bcaf567a3 100644 --- a/DebugLog.js +++ b/DebugLog.js @@ -17,7 +17,8 @@ exports.newDebugLog = function newDebugLog() { write: write, newInternalLoop: newInternalLoop, persist: persist, // This method is executed at the end of each Main Loop. - initialize: initialize + initialize: initialize, + finalize: finalize }; let accumulatedLog = "["; @@ -37,6 +38,10 @@ exports.newDebugLog = function newDebugLog() { } } + function finalize() { + persist() + } + function newInternalLoop(pBot, pProcess, date, percentage) { if (percentage === undefined) { percentage = "" @@ -64,6 +69,7 @@ exports.newDebugLog = function newDebugLog() { /* Here we actually write the content of the in-memory log to a file */ try { + if (accumulatedLog === "[") {return} // nothing to persist at the moment. internalLoopCounter++; diff --git a/FileStorage.js b/FileStorage.js index 30b69242e..a220960c5 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -447,6 +447,8 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { '/Storage/' + filePath + logger.write(MODULE_NAME, '[INFO] FileStorage -> getFileViaHTTP -> url = ' + url) + http.get(url, onResponse); function onResponse(response) { diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 2c2401a15..90892dd72 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -116,7 +116,11 @@ /* For each loop we want to create a new log file. */ + if (logger !== undefined) { + logger.finalize() + } logger = DEBUG_MODULE.newDebugLog(); + global.LOGGER_MAP.set(MODULE_NAME, logger) logger.bot = bot; logger.initialize(); diff --git a/Root.js b/Root.js index bd1a498aa..3a0dec4cd 100644 --- a/Root.js +++ b/Root.js @@ -391,6 +391,7 @@ let logger; logger = DEBUG_MODULE.newDebugLog(); + global.LOGGER_MAP.set('runSensorBot', logger) logger.bot = pBotConfig; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runSensorBot -> Entering function."); } @@ -455,6 +456,7 @@ let logger; logger = DEBUG_MODULE.newDebugLog(); + global.LOGGER_MAP.set('runIndicatorBot', logger) logger.bot = pBotConfig; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runIndicatorBot -> Entering function."); } @@ -517,6 +519,7 @@ let logger; logger = DEBUG_MODULE.newDebugLog(); + global.LOGGER_MAP.set('runTradingBot', logger) logger.bot = pBotConfig; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runTradingBot -> Entering function."); } diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index e90924148..558d9ffa4 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -121,7 +121,11 @@ /* For each loop we want to create a new log file. */ + if (logger !== undefined) { + logger.finalize() + } logger = DEBUG_MODULE.newDebugLog(); + global.LOGGER_MAP.set(MODULE_NAME, logger) logger.bot = bot; logger.initialize(); diff --git a/TradingBotProcessMainLoop.js b/TradingBotProcessMainLoop.js index ab3bee5ad..579de4f33 100644 --- a/TradingBotProcessMainLoop.js +++ b/TradingBotProcessMainLoop.js @@ -124,8 +124,11 @@ } /* For each loop we want to create a new log file. */ - + if (logger !== undefined) { + logger.finalize() + } logger = DEBUG_MODULE.newDebugLog(); + global.LOGGER_MAP.set(MODULE_NAME, logger) logger.bot = bot; logger.initialize(); diff --git a/server.js b/server.js index f32a3bc13..2f08161cd 100644 --- a/server.js +++ b/server.js @@ -26,19 +26,28 @@ global.CUSTOM_FAIL_RESPONSE = { }; const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; +global.LOGGER_MAP = new Map() process.on('uncaughtException', function (err) { console.log('[ERROR] Task Server -> server -> uncaughtException -> err.message = ' + err.message) console.log('[ERROR] Task Server -> server -> uncaughtException -> err.stack = ' + err.stack) - process.exit(1) + global.EXIT_NODE_PROCESS() }) process.on('unhandledRejection', (reason, p) => { console.log('[ERROR] Task Server -> server -> unhandledRejection -> reason = ' + JSON.stringify(reason)) console.log('[ERROR] Task Server -> server -> unhandledRejection -> p = ' + JSON.stringify(p)) - process.exit(1) + global.EXIT_NODE_PROCESS() }) +function finalizeLoggers() { + global.LOGGER_MAP.forEach(forEachLogger) + + function forEachLogger(logger) { + logger.finalize() + } +} + process.on('exit', function (code) { if (global.TASK_NODE !== undefined) { @@ -69,8 +78,13 @@ process.on('message', message => { } }); + +let shuttingDownProcess = false global.EXIT_NODE_PROCESS = function exitProcess() { + if (shuttingDownProcess === true) { return } + shuttingDownProcess = true + /* Cleaning Before Exiting. */ clearInterval(global.HEARTBEAT_INTERVAL_HANDLER) @@ -82,6 +96,7 @@ global.EXIT_NODE_PROCESS = function exitProcess() { global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Stopped') // Meaning Process Stopped } + finalizeLoggers() //console.log("[INFO] Task Server -> " + global.TASK_NODE.name + " -> EXIT_NODE_PROCESS -> Task Server will stop in 10 seconds."); setTimeout(process.exit, 10000) // We will give 10 seconds to logs be written on file From d47a54c03fd22f57b0afa6b178b9a41418b733e8 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Apr 2020 09:24:36 +0200 Subject: [PATCH 33/35] Stopping Sessions correctly when crash happens --- FileStorage.js | 10 +++++----- Session.js | 1 + server.js | 12 ++++++++++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/FileStorage.js b/FileStorage.js index a220960c5..947b41dfa 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -58,7 +58,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { fs.readFile(fileLocation, onFileRead) } else { /* We read the file via a web server over http */ - getFileViaHTTP(fileLocation, onFileRead) + getFileViaHTTP(fileLocation, onFileRead, callBackFunction) } function onFileRead(err, text) { @@ -130,7 +130,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { fs.readFile(fileLocation + '.Previous.json', onPreviousFileRead) } else { /* We read the file via a web server over http */ - getFileViaHTTP(fileLocation + '.Previous.json', onPreviousFileRead) + getFileViaHTTP(fileLocation + '.Previous.json', onPreviousFileRead, callBackFunction) } function onPreviousFileRead(err, text) { @@ -435,7 +435,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { }, initDir); } - function getFileViaHTTP(filePath, callback) { + function getFileViaHTTP(filePath, callback, callBackFunction) { try { /* The filePath received is the one that is needed to fetch data from with fs. To do it via http we need to remove the prefix that includes this: ./Data-Storage/ */ @@ -477,8 +477,8 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { } catch (err) { logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - abort = true + logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> Failed to fetch file via HTTP. Will retry later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); } } } diff --git a/Session.js b/Session.js index 68a12b7f2..9ae1f99f4 100644 --- a/Session.js +++ b/Session.js @@ -39,6 +39,7 @@ /* Listen to event to start or stop the session. */ bot.sessionKey = bot.processNode.session.name + '-' + bot.processNode.session.type + '-' + bot.processNode.session.id + global.SESSION_MAP.set(bot.sessionKey, bot.sessionKey) global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Run Session', undefined, bot.sessionKey, undefined, runSession) global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Stop Session', undefined, bot.sessionKey, undefined, stopSession) diff --git a/server.js b/server.js index 2f08161cd..80ac46a62 100644 --- a/server.js +++ b/server.js @@ -27,6 +27,7 @@ global.CUSTOM_FAIL_RESPONSE = { const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; global.LOGGER_MAP = new Map() +global.SESSION_MAP = new Map() process.on('uncaughtException', function (err) { console.log('[ERROR] Task Server -> server -> uncaughtException -> err.message = ' + err.message) @@ -48,6 +49,14 @@ function finalizeLoggers() { } } +function finalizeSessions() { + global.SESSION_MAP.forEach(forEachSession) + + function forEachSession(session) { + global.EVENT_SERVER_CLIENT.raiseEvent(session, 'Stopped') + } +} + process.on('exit', function (code) { if (global.TASK_NODE !== undefined) { @@ -85,6 +94,9 @@ global.EXIT_NODE_PROCESS = function exitProcess() { if (shuttingDownProcess === true) { return } shuttingDownProcess = true + /* Signal that all sessions are stopping. */ + finalizeSessions() + /* Cleaning Before Exiting. */ clearInterval(global.HEARTBEAT_INTERVAL_HANDLER) From cad50d0d802165566278c431ad3bde30b2566aa4 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Apr 2020 10:12:23 +0200 Subject: [PATCH 34/35] bug fixed handling http exceptions --- FileStorage.js | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/FileStorage.js b/FileStorage.js index 947b41dfa..22966c706 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -449,32 +449,43 @@ exports.newFileStorage = function newFileStorage(logger, host, port) { logger.write(MODULE_NAME, '[INFO] FileStorage -> getFileViaHTTP -> url = ' + url) - http.get(url, onResponse); + let request = http.get(url, onResponse); + + request.on('error', function (err) { + logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onError -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onError -> Failed to fetch file via HTTP. Will retry later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + }) function onResponse(response) { - const chunks = [] + try { + const chunks = [] - response.on('data', onMessegesArrived) - response.on('end', onEnd) + response.on('data', onMessegesArrived) + response.on('end', onEnd) - function onMessegesArrived(chunk) { - chunks.push(chunk) - } - - function onEnd() { - let fileContent = Buffer.concat(chunks).toString('utf8') - let err = null - if (fileContent.indexOf('does not exist') >= 0) { - err = { - code: "ENOENT" // This is how fs would have returned upon this situation. - } - fileContent = undefined + function onMessegesArrived(chunk) { + chunks.push(chunk) } - callback(err, fileContent) + + function onEnd() { + let fileContent = Buffer.concat(chunks).toString('utf8') + let err = null + if (fileContent.indexOf('does not exist') >= 0) { + err = { + code: "ENOENT" // This is how fs would have returned upon this situation. + } + fileContent = undefined + } + callback(err, fileContent) + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onResponse -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onResponse -> Failed to fetch file via HTTP. Will retry later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); } } - } catch (err) { logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> err = " + err.stack); logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> Failed to fetch file via HTTP. Will retry later. "); From f93729486e16bd1ef829519cb767cee4c674eb3f Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 2 May 2020 09:20:36 +0200 Subject: [PATCH 35/35] Bug fix when Base Asset y swapped --- TradingBot.js | 18 ++++++++++++++++-- TradingSimulation.js | 2 ++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/TradingBot.js b/TradingBot.js index 1335f875c..1a2852487 100644 --- a/TradingBot.js +++ b/TradingBot.js @@ -213,6 +213,8 @@ record.maximumBalanceB + "," + record.baseAsset + "," + record.quotedAsset + "," + + record.marketBaseAsset + "," + + record.marketQuotedAsset + "," + record.positionPeriods + "," + record.positionDays + "," + record.distanceToLastTriggerOn + "," + @@ -559,8 +561,14 @@ record.maximumBalanceB + "," + record.baseAsset + "," + record.quotedAsset + "," + + record.marketBaseAsset + "," + + record.marketQuotedAsset + "," + record.positionPeriods + "," + - record.positionDays + "]"; + record.positionDays + "," + + record.distanceToLastTriggerOn + "," + + record.distanceToLastTriggerOff + "," + + record.distanceToLastTakePosition + "," + + record.distanceToLastClosePosition + "]"; if (separator === "") { separator = ","; } @@ -664,8 +672,14 @@ record.maximumBalanceB + "," + record.baseAsset + "," + record.quotedAsset + "," + + record.marketBaseAsset + "," + + record.marketQuotedAsset + "," + record.positionPeriods + "," + - record.positionDays + "]"; + record.positionDays + "," + + record.distanceToLastTriggerOn + "," + + record.distanceToLastTriggerOff + "," + + record.distanceToLastTakePosition + "," + + record.distanceToLastClosePosition + "]"; if (separator === "") { separator = ","; } diff --git a/TradingSimulation.js b/TradingSimulation.js index 9fa497537..1e924e89c 100644 --- a/TradingSimulation.js +++ b/TradingSimulation.js @@ -2798,6 +2798,8 @@ maximumBalanceB: bot.VALUES_TO_USE.maximumBalanceB, baseAsset: quotedBaseAsset, quotedAsset: quotedQuotedAsset, + marketBaseAsset: '"' + bot.market.baseAsset + '"', + marketQuotedAsset: '"' + bot.market.quotedAsset + '"' , positionPeriods: positionPeriods, positionDays: positionDays, distanceToLastTriggerOn: distanceToLast.triggerOn,