From fc59aa290cac449b51407c96f3996e6b0e77d034 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 11 Mar 2019 14:11:45 +0100 Subject: [PATCH 01/18] Failure to update Clones Module does not stop trading bot execution. --- TradingBotProcessMainLoop.js | 55 ++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/TradingBotProcessMainLoop.js b/TradingBotProcessMainLoop.js index 85ac5cab1..9d6e6cab5 100644 --- a/TradingBotProcessMainLoop.js +++ b/TradingBotProcessMainLoop.js @@ -904,28 +904,12 @@ } } - async function saveContext() { + function saveContext() { try { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> saveContext -> Entering function."); } - if (global.CURRENT_EXECUTION_AT === "Cloud") { - // Update operations module with latest context - let lastExecution = context.newHistoryRecord - let date = lastExecution.date / 1000 | 0 - let buyAvgRate = lastExecution.buyAvgRate - let sellAvgRate = lastExecution.sellAvgRate - let marketRate = lastExecution.marketRate - let combinedProfitsA = lastExecution.combinedProfitsA - let combinedProfitsB = lastExecution.combinedProfitsB - - await operationsIntegration.updateExecutionResults(date, buyAvgRate, sellAvgRate, - marketRate, combinedProfitsA, combinedProfitsB) - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> saveContext -> Execution Results updated."); } - } - context.saveThemAll(onFinished); function onFinished(err) { @@ -938,7 +922,7 @@ case global.DEFAULT_OK_RESPONSE.result: { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> saveContext -> onFinished -> Execution finished well."); } nextWaitTime = 'Normal'; - loopControl(nextWaitTime); + updateClonesModule(); return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. @@ -997,6 +981,41 @@ } } + async function updateClonesModule() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> updateClonesModule -> Entering function."); } + + if (global.CURRENT_EXECUTION_AT === "Cloud") { + + // Update clones module with latest context informations + + let lastExecution = context.newHistoryRecord + let date = lastExecution.date / 1000 | 0 + let buyAvgRate = lastExecution.buyAvgRate + let sellAvgRate = lastExecution.sellAvgRate + let marketRate = lastExecution.marketRate + let combinedProfitsA = lastExecution.combinedProfitsA + let combinedProfitsB = lastExecution.combinedProfitsB + + await operationsIntegration.updateExecutionResults(date, buyAvgRate, sellAvgRate, + marketRate, combinedProfitsA, combinedProfitsB) + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> updateClonesModule -> Clones Module Updated."); } + } + + nextWaitTime = 'Normal'; + loopControl(nextWaitTime); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> updateClonesModule -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> updateClonesModule -> Execution will continue anyways. "); + nextWaitTime = 'Normal'; + loopControl(nextWaitTime); + } + } + function loopControl(nextWaitTime) { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Entering function."); } From 1324881882135368f9ed65fbb79010e06e493a96 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 16 Mar 2019 08:51:30 +0100 Subject: [PATCH 02/18] Data Dependencies implemented on Indicator Loop --- DataDependencies.js | 7 +++ IndicatorBotProcessMainLoop.js | 81 +++++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/DataDependencies.js b/DataDependencies.js index db08c5a77..0660226b9 100644 --- a/DataDependencies.js +++ b/DataDependencies.js @@ -23,6 +23,13 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } + if (pDataDependenciesConfig === undefined) { + + // We allow old indicators not to declare their data dependencies. + + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + } /* For each dependency declared at the bot config, we will initialize a DataSet as part of this initialization process. diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 5ed0969df..88347779d 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -175,6 +175,8 @@ const UTILITIES = require(ROOT_DIR + 'CloudUtilities'); const STATUS_REPORT = require(ROOT_DIR + 'StatusReport'); const STATUS_DEPENDENCIES = require(ROOT_DIR + 'StatusDependencies'); + const DATA_DEPENDENCIES = require(ROOT_DIR + 'DataDependencies'); + const DATA_SET = require(ROOT_DIR + 'DataSet'); /* We define the datetime for the process that we are running now. This will be the official processing time for both the infraestructure and the bot. */ @@ -189,6 +191,7 @@ let userBot; let statusDependencies; + let dataDependencies; let nextWaitTime; @@ -213,7 +216,7 @@ switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeStatusDependencies -> onInizialized -> Execution finished well."); - initializeUserBot(); + initializeDataDependencies(); return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. @@ -269,6 +272,80 @@ } } + function initializeDataDependencies() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> Entering function."); } + + dataDependencies = DATA_DEPENDENCIES.newDataDependencies(bot, logger, DATA_SET, BLOB_STORAGE, UTILITIES); + + dataDependencies.initialize(processConfig.dataDependencies, onInizialized); + + function onInizialized(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> onInizialized -> Entering function."); } + + switch (err.result) { + case global.DEFAULT_OK_RESPONSE.result: { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> onInizialized -> Execution finished well."); } + initializeUserBot(); + return; + } + case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); + nextWaitTime = 'Retry'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Operation Failed. Aborting the process."); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + return; + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Unhandled err.result received. -> err.message = " + err.message); + + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> err = " + err.message); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + } + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> err = " + err.message); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + } + } + function initializeUserBot() { try { @@ -277,7 +354,7 @@ usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE); - usertBot.initialize(statusDependencies, pMonth, pYear, onInizialized); + usertBot.initialize(statusDependencies, pMonth, pYear, onInizialized, dataDependencies); function onInizialized(err) { From 9c92069b5d07d753dbbcebdc2d6ae7a686950045 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 16 Mar 2019 09:50:23 +0100 Subject: [PATCH 03/18] Framework for Multi Period Market Processes added. INitializar Function ready --- Multi-Period-Market.js | 924 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 924 insertions(+) create mode 100644 Multi-Period-Market.js diff --git a/Multi-Period-Market.js b/Multi-Period-Market.js new file mode 100644 index 000000000..75e4236c7 --- /dev/null +++ b/Multi-Period-Market.js @@ -0,0 +1,924 @@ +exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, FILE_STORAGE) { + + const FULL_LOG = true; + const LOG_FILE_CONTENT = false; + + const MODULE_NAME = "User Bot"; + + const EXCHANGE_NAME = "Poloniex"; + + const LRC_FOLDER_NAME = "LRC-Points"; + const PERCENTAGE_BANDWIDTH_FOLDER_NAME = "Percentage-Bandwidth"; + const BOLLINGER_BANDS_FOLDER_NAME = "Bollinger-Bands"; + const BOLLINGER_CHANNELS_FOLDER_NAME = "Bollinger-Channels"; + const BOLLINGER_SUB_CHANNELS_FOLDER_NAME = "Bollinger-Sub-Channels"; + const CANDLES_FOLDER_NAME = "Candles"; + const SIMULATED_RECORDS_FOLDER_NAME = "Trading-Simulation"; + const CONDITIONS_FOLDER_NAME = "Simulation-Conditions"; + + const commons = COMMONS.newCommons(bot, logger, UTILITIES); + + thisObject = { + initialize: initialize, + start: start + }; + + let gaussStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + let chrisStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + let oliviaStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + let jasonStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + + let utilities = UTILITIES.newCloudUtilities(bot, logger); + + let statusDependencies; + let dataDependencies; + let storages = []; + + return thisObject; + + function initialize(pStatusDependencies, pDataDependencies, pMonth, pYear, callBackFunction) { + + try { + + logger.fileName = MODULE_NAME; + logger.initialize(); + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } + + statusDependencies = pStatusDependencies; + dataDependencies = pDataDependencies; + + for (let i = 0; i < dataDependencies.length; i++) { + + let key; + let storage; + + key = dataDependencies.devTeam + "-" + + dataDependencies.bot + "-" + + dataDependencies.product + "-" + + dataDependencies.dataSet + "-" + + dataDependencies.dataSetVersion + + storage = dataDependencies.dataSets.get(key); + + storages.push(storage); + + } + + callBackFunction(global.DEFAULT_OK_RESPONSE); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + + + function start(callBackFunction) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); } + + let market = global.MARKET; + + buildRecords(); + + function buildRecords() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> Entering function."); } + + try { + + let n; + + periodsLoop(); + + function periodsLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> periodsLoop -> Entering function."); } + + /* + + We will iterate through all posible periods. + + */ + + n = 0 // loop Variable representing each possible period as defined at the periods array. + + loopBody(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> periodsLoop -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function loopBody() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> Entering function."); } + + const outputPeriod = global.marketFilesPeriods[n][0]; + const timePeriod = global.marketFilesPeriods[n][1]; + + let marketFile; + + + + + + + + let LRCMap = new Map(); + let percentageBandwidthMap = new Map(); + let bollingerBandsMap = new Map(); + let bollingerChannelsArray = []; + let bollingerSubChannelsArray = []; + + let candles = []; + let recordsArray = []; + let conditionsArray = []; + let simulationLogic = {}; + + nextLRC(); + + function nextLRC() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextLRC -> Entering function."); } + + let fileName = market.assetA + '_' + market.assetB + ".json"; + + let filePathRoot = "AAVikings" + "/" + "AAGauss" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; + let filePath = filePathRoot + "/Output/" + LRC_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; + + gaussStorage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextLRC -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextLRC -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextLRC -> onFileReceived -> err = " + err.message); + callBackFunction(err); + return; + + } + + marketFile = JSON.parse(text); + + buildLRC(); + + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextLRC -> onFileReceived -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextLRC -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function buildLRC() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildLRC -> Entering function."); } + + try { + + let previous; + + for (let i = 0; i < marketFile.length; i++) { + + let LRC = { + begin: marketFile[i][0], + end: marketFile[i][1], + _15: marketFile[i][2], + _30: marketFile[i][3], + _60: marketFile[i][4] + }; + + if (previous !== undefined) { + + if (previous._15 > LRC._15) { LRC.direction15 = 'down'; } + if (previous._15 < LRC._15) { LRC.direction15 = 'up'; } + if (previous._15 === LRC._15) { LRC.direction15 = 'side'; } + + if (previous._30 > LRC._30) { LRC.direction30 = 'down'; } + if (previous._30 < LRC._30) { LRC.direction30 = 'up'; } + if (previous._30 === LRC._30) { LRC.direction30 = 'side'; } + + if (previous._60 > LRC._60) { LRC.direction60 = 'down'; } + if (previous._60 < LRC._60) { LRC.direction60 = 'up'; } + if (previous._60 === LRC._60) { LRC.direction60 = 'side'; } + + } + + LRC.previous = previous; + + LRCMap.set(LRC.begin, LRC); + + previous = LRC; + } + + nextPercentageBandwidth(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildLRC -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function nextPercentageBandwidth() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> Entering function."); } + + let fileName = market.assetA + '_' + market.assetB + ".json"; + + let filePathRoot = bot.devTeam + "/" + "AAChris" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; + let filePath = filePathRoot + "/Output/" + PERCENTAGE_BANDWIDTH_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; + + chrisStorage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> onFileReceived -> err = " + err.message); + callBackFunction(err); + return; + + } + + marketFile = JSON.parse(text); + + buildPercentageBandwidthMap(); + + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> onFileReceived -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function buildPercentageBandwidthMap() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildPercentageBandwidthMap -> Entering function."); } + + try { + + let previous; + + for (let i = 0; i < marketFile.length; i++) { + + let percentageBandwidth = { + begin: marketFile[i][0], + end: marketFile[i][1], + value: marketFile[i][2], + movingAverage: marketFile[i][3], + bandwidth: marketFile[i][4] + }; + + if (previous !== undefined) { + + if (previous.movingAverage > percentageBandwidth.movingAverage) { percentageBandwidth.direction = 'down'; } + if (previous.movingAverage < percentageBandwidth.movingAverage) { percentageBandwidth.direction = 'up'; } + if (previous.movingAverage === percentageBandwidth.movingAverage) { percentageBandwidth.direction = 'side'; } + + } + + percentageBandwidth.previous = previous; + + percentageBandwidthMap.set(percentageBandwidth.begin, percentageBandwidth); + + previous = percentageBandwidth; + } + + nextBollingerBands(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildPercentageBandwidthMap -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function nextBollingerBands() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerBands -> Entering function."); } + + let fileName = market.assetA + '_' + market.assetB + ".json"; + + let filePathRoot = bot.devTeam + "/" + "AAChris" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; + let filePath = filePathRoot + "/Output/" + BOLLINGER_BANDS_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; + + chrisStorage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerBands -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerBands -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerBands -> onFileReceived -> err = " + err.message); + callBackFunction(err); + return; + + } + + marketFile = JSON.parse(text); + + buildBollingerBandsMap(); + + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerBands -> onFileReceived -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerBands -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function buildBollingerBandsMap() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildBollingerBandsMap -> Entering function."); } + + try { + + let previous; + + for (let i = 0; i < marketFile.length; i++) { + + let bollingerBand = { + begin: marketFile[i][0], + end: marketFile[i][1], + movingAverage: marketFile[i][2], + standardDeviation: marketFile[i][3], + deviation: marketFile[i][4] + }; + + if (previous !== undefined) { + + if (previous.movingAverage > bollingerBand.movingAverage) { bollingerBand.direction = 'down'; } + if (previous.movingAverage < bollingerBand.movingAverage) { bollingerBand.direction = 'up'; } + if (previous.movingAverage === bollingerBand.movingAverage) { bollingerBand.direction = 'side'; } + + } + + bollingerBand.previous = previous; + + bollingerBandsMap.set(bollingerBand.begin, bollingerBand); + + previous = bollingerBand; + } + + nextBollingerChannels(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildBollingerBandsMap -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function nextBollingerChannels() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerChannels -> Entering function."); } + + let fileName = market.assetA + '_' + market.assetB + ".json"; + + let filePathRoot = bot.devTeam + "/" + "AAPaula" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; + let filePath = filePathRoot + "/Output/" + BOLLINGER_CHANNELS_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; + + chrisStorage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerChannels -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerChannels -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerChannels -> onFileReceived -> err = " + err.message); + callBackFunction(err); + return; + + } + + marketFile = JSON.parse(text); + + buildBollingerChannelsArray(); + + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerChannels -> onFileReceived -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerChannels -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function buildBollingerChannelsArray() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildBollingerChannelsArray -> Entering function."); } + + try { + + let previous; + + for (let i = 0; i < marketFile.length; i++) { + + let bollingerChannel = { + begin: marketFile[i][0], + end: marketFile[i][1], + direction: marketFile[i][2], + period: marketFile[i][3], + firstMovingAverage: marketFile[i][4], + lastMovingAverage: marketFile[i][5], + firstDeviation: marketFile[i][6], + lastDeviation: marketFile[i][7] + }; + + bollingerChannel.previous = previous; + + bollingerChannelsArray.push(bollingerChannel); + + previous = bollingerChannel; + } + + nextBollingerSubChannels(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildBollingerChannelsArray -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function nextBollingerSubChannels() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> Entering function."); } + + let fileName = market.assetA + '_' + market.assetB + ".json"; + + let filePathRoot = bot.devTeam + "/" + "AAPaula" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; + let filePath = filePathRoot + "/Output/" + BOLLINGER_SUB_CHANNELS_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; + + chrisStorage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> onFileReceived -> err = " + err.message); + callBackFunction(err); + return; + + } + + marketFile = JSON.parse(text); + + buildBollingerSubChannelsArray(); + + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> onFileReceived -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function buildBollingerSubChannelsArray() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildBollingerSubChannelsArray -> Entering function."); } + + try { + + let previous; + + for (let i = 0; i < marketFile.length; i++) { + + let bollingerSubChannel = { + begin: marketFile[i][0], + end: marketFile[i][1], + direction: marketFile[i][2], + slope: marketFile[i][3], + period: marketFile[i][4], + firstMovingAverage: marketFile[i][5], + lastMovingAverage: marketFile[i][6], + firstDeviation: marketFile[i][7], + lastDeviation: marketFile[i][8] + }; + + bollingerSubChannel.previous = previous; + + bollingerSubChannelsArray.push(bollingerSubChannel); + + previous = bollingerSubChannel; + } + + nextCandlesFile(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildBollingerSubChannelsArray -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function nextCandlesFile() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextCandlesFile -> Entering function."); } + + let fileName = market.assetA + '_' + market.assetB + ".json"; + + let filePathRoot = bot.devTeam + "/" + "AAOlivia" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; + let filePath = filePathRoot + "/Output/" + CANDLES_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; + + oliviaStorage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextCandlesFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextCandlesFile -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextCandlesFile -> onFileReceived -> err = " + err.message); + callBackFunction(err); + return; + + } + + marketFile = JSON.parse(text); + + buildCandles(); + + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextCandlesFile -> onFileReceived -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextCandlesFile -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function buildCandles() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildCandles -> Entering function."); } + + try { + + let previous; + + for (let i = 0; i < marketFile.length; i++) { + + let candle = { + open: undefined, + close: undefined, + min: 10000000000000, + max: 0, + begin: undefined, + end: undefined, + direction: undefined + }; + + candle.min = marketFile[i][0]; + candle.max = marketFile[i][1]; + + candle.open = marketFile[i][2]; + candle.close = marketFile[i][3]; + + candle.begin = marketFile[i][4]; + candle.end = marketFile[i][5]; + + if (candle.open > candle.close) { candle.direction = 'down'; } + if (candle.open < candle.close) { candle.direction = 'up'; } + if (candle.open === candle.close) { candle.direction = 'side'; } + + candle.previous = previous; + + candles.push(candle); + + previous = candle; + } + + commons.runSimulation( + candles, + bollingerBandsMap, + percentageBandwidthMap, + LRCMap, + bollingerChannelsArray, + bollingerSubChannelsArray, + recordsArray, + conditionsArray, + simulationLogic, + outputPeriod, + writeRecordsFile) + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildCandles -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function writeRecordsFile() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeRecordsFile -> Entering function."); } + + let separator = ""; + let fileRecordCounter = 0; + + let fileContent = ""; + + for (let i = 0; i < recordsArray.length; i++) { + + let record = recordsArray[i]; + + fileContent = fileContent + separator + '[' + + record.begin + "," + + record.end + "," + + record.type + "," + + record.rate + "," + + record.amount + "," + + record.balanceA + "," + + record.balanceB + "," + + record.profit + "," + + record.lastProfit + "," + + record.stopLoss + "," + + record.roundtrips + "," + + record.hits + "," + + record.fails + "," + + record.hitRatio + "," + + record.ROI + "," + + record.periods + "," + + record.days + "," + + record.anualizedRateOfReturn + "," + + record.sellRate + "," + + record.lastProfitPercent + "," + + record.strategy + "," + + record.strategyPhase + "," + + record.buyOrder + "," + + record.stopLossPhase + "," + + record.buyOrderPhase + "]"; + + if (separator === "") { separator = ","; } + + fileRecordCounter++; + + } + + fileContent = "[" + fileContent + "]"; + + let fileName = '' + market.assetA + '_' + market.assetB + '.json'; + + let filePathRoot = bot.devTeam + "/" + bot.codeName + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; + let filePath = filePathRoot + "/Output/" + SIMULATED_RECORDS_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; + + jasonStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + + function onFileCreated(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> fileContent = " + fileContent); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> filePath = " + filePath); + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> market = " + market.assetA + "_" + market.assetB); + + callBackFunction(err); + return; + + } + + writeConditionsFile(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeRecordsFile -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function writeConditionsFile() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeConditionsFile -> Entering function."); } + + let separator = ""; + let fileRecordCounter = 0; + + let fileContent = ""; + + for (let i = 0; i < conditionsArray.length; i++) { + + let record = conditionsArray[i]; + + let conditions = ""; + let conditionsSeparator = ""; + + for (let j = 0; j < record.length - 1; j++) { + conditions = conditions + conditionsSeparator + record[j]; + if (conditionsSeparator === "") { conditionsSeparator = ","; } + } + + conditions = conditions + conditionsSeparator + '[' + record[record.length - 1] + ']'; // The last item contains an Array of condition values. + + fileContent = fileContent + separator + '[' + conditions + ']'; + + if (separator === "") { separator = ","; } + + fileRecordCounter++; + + } + + fileContent = "[" + fileContent + "]"; + fileContent = "[" + JSON.stringify(simulationLogic) + "," + fileContent + "]"; + let fileName = '' + market.assetA + '_' + market.assetB + '.json'; + + let filePathRoot = bot.devTeam + "/" + bot.codeName + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; + let filePath = filePathRoot + "/Output/" + CONDITIONS_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; + + jasonStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + + function onFileCreated(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> fileContent = " + fileContent); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> filePath = " + filePath); + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> market = " + market.assetA + "_" + market.assetB); + + callBackFunction(err); + return; + + } + + controlLoop(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeConditionsFile -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function controlLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> controlLoop -> Entering function."); } + + n++; + + if (n < global.marketFilesPeriods.length) { + + loopBody(); + + } else { + + writeStatusReport(callBackFunction); + + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> controlLoop -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function writeStatusReport(callBack) { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> Entering function."); } + + try { + + let reportKey = "AAMasters" + "-" + "AAJason" + "-" + "Multi-Period-Market" + "-" + "dataSet.V1"; + let thisReport = statusDependencies.statusReports.get(reportKey); + + thisReport.file.lastExecution = bot.processDatetime; + thisReport.save(callBack); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } +}; From fa5f4cb2f5060d4cbcc372f25521e9c156d2f290 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 16 Mar 2019 10:30:31 +0100 Subject: [PATCH 04/18] Get Files Function --- Multi-Period-Market.js | 342 +++++++++-------------------------------- 1 file changed, 70 insertions(+), 272 deletions(-) diff --git a/Multi-Period-Market.js b/Multi-Period-Market.js index 75e4236c7..71dc66beb 100644 --- a/Multi-Period-Market.js +++ b/Multi-Period-Market.js @@ -33,6 +33,7 @@ let statusDependencies; let dataDependencies; let storages = []; + let marketFiles = []; return thisObject; @@ -52,12 +53,13 @@ let key; let storage; + let dependency = dataDependencies[i]; - key = dataDependencies.devTeam + "-" + - dataDependencies.bot + "-" + - dataDependencies.product + "-" + - dataDependencies.dataSet + "-" + - dataDependencies.dataSetVersion + key = dependency.devTeam + "-" + + dependency.bot + "-" + + dependency.product + "-" + + dependency.dataSet + "-" + + dependency.dataSetVersion storage = dataDependencies.dataSets.get(key); @@ -126,12 +128,73 @@ const outputPeriod = global.marketFilesPeriods[n][0]; const timePeriod = global.marketFilesPeriods[n][1]; - - let marketFile; + let fileCount = 0; + + for (let i = 0; i < dataDependencies.length; i++) { + + let dependency = dataDependencies[i]; + let storage = storages[i]; + + getFile(); + + function getFile() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> getFile -> Entering function."); } + + let fileName = market.assetA + '_' + market.assetB + ".json"; + + let filePathRoot = dependency.devTeam + "/" + dependency.bot + "." + dependency.botVersion.major + "." + dependency.botVersion.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + dependency.dataSetVersion; + let filePath = filePathRoot + "/Output/" + dependency.product + "/" + "Multi-Period-Market" + "/" + timePeriod; + + storage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> getFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> getFile -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> getFile -> onFileReceived -> err = " + err.message); + callBackFunction(err); + return; + + } + + let marketFile = JSON.parse(text); + marketFiles.push(marketFile); + fileCount++; + + callTheBot(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> getFile -> onFileReceived -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> getFile -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + } + + function callTheBot() { + + if (fileCount === dataDependencies.length + 1) { + } + } @@ -148,51 +211,7 @@ nextLRC(); - function nextLRC() { - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextLRC -> Entering function."); } - - let fileName = market.assetA + '_' + market.assetB + ".json"; - - let filePathRoot = "AAVikings" + "/" + "AAGauss" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + LRC_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; - - gaussStorage.getTextFile(filePath, fileName, onFileReceived, true); - - function onFileReceived(err, text) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextLRC -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextLRC -> onFileReceived -> text = " + text); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextLRC -> onFileReceived -> err = " + err.message); - callBackFunction(err); - return; - - } - - marketFile = JSON.parse(text); - - buildLRC(); - - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextLRC -> onFileReceived -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextLRC -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } function buildLRC() { @@ -244,51 +263,7 @@ } } - function nextPercentageBandwidth() { - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> Entering function."); } - - let fileName = market.assetA + '_' + market.assetB + ".json"; - - let filePathRoot = bot.devTeam + "/" + "AAChris" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + PERCENTAGE_BANDWIDTH_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; - - chrisStorage.getTextFile(filePath, fileName, onFileReceived, true); - - function onFileReceived(err, text) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> onFileReceived -> text = " + text); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> onFileReceived -> err = " + err.message); - callBackFunction(err); - return; - - } - - marketFile = JSON.parse(text); - - buildPercentageBandwidthMap(); - - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> onFileReceived -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextPercentageBandwidth -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } function buildPercentageBandwidthMap() { @@ -332,51 +307,7 @@ } } - function nextBollingerBands() { - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerBands -> Entering function."); } - - let fileName = market.assetA + '_' + market.assetB + ".json"; - - let filePathRoot = bot.devTeam + "/" + "AAChris" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + BOLLINGER_BANDS_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; - - chrisStorage.getTextFile(filePath, fileName, onFileReceived, true); - - function onFileReceived(err, text) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerBands -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerBands -> onFileReceived -> text = " + text); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerBands -> onFileReceived -> err = " + err.message); - callBackFunction(err); - return; - - } - - marketFile = JSON.parse(text); - - buildBollingerBandsMap(); - - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerBands -> onFileReceived -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerBands -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } function buildBollingerBandsMap() { @@ -420,51 +351,7 @@ } } - function nextBollingerChannels() { - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerChannels -> Entering function."); } - - let fileName = market.assetA + '_' + market.assetB + ".json"; - - let filePathRoot = bot.devTeam + "/" + "AAPaula" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + BOLLINGER_CHANNELS_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; - - chrisStorage.getTextFile(filePath, fileName, onFileReceived, true); - - function onFileReceived(err, text) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerChannels -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerChannels -> onFileReceived -> text = " + text); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerChannels -> onFileReceived -> err = " + err.message); - callBackFunction(err); - return; - - } - - marketFile = JSON.parse(text); - - buildBollingerChannelsArray(); - - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerChannels -> onFileReceived -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerChannels -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } function buildBollingerChannelsArray() { @@ -503,51 +390,7 @@ } } - function nextBollingerSubChannels() { - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> Entering function."); } - - let fileName = market.assetA + '_' + market.assetB + ".json"; - - let filePathRoot = bot.devTeam + "/" + "AAPaula" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + BOLLINGER_SUB_CHANNELS_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; - - chrisStorage.getTextFile(filePath, fileName, onFileReceived, true); - - function onFileReceived(err, text) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> onFileReceived -> text = " + text); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> onFileReceived -> err = " + err.message); - callBackFunction(err); - return; - - } - - marketFile = JSON.parse(text); - - buildBollingerSubChannelsArray(); - - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> onFileReceived -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextBollingerSubChannels -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } function buildBollingerSubChannelsArray() { @@ -587,51 +430,6 @@ } } - function nextCandlesFile() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextCandlesFile -> Entering function."); } - - let fileName = market.assetA + '_' + market.assetB + ".json"; - - let filePathRoot = bot.devTeam + "/" + "AAOlivia" + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + CANDLES_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; - - oliviaStorage.getTextFile(filePath, fileName, onFileReceived, true); - - function onFileReceived(err, text) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextCandlesFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> nextCandlesFile -> onFileReceived -> text = " + text); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextCandlesFile -> onFileReceived -> err = " + err.message); - callBackFunction(err); - return; - - } - - marketFile = JSON.parse(text); - - buildCandles(); - - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextCandlesFile -> onFileReceived -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> nextCandlesFile -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } function buildCandles() { From 2e476b39b61a423295eae76eabf53a931e20ced7 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 16 Mar 2019 10:57:38 +0100 Subject: [PATCH 05/18] Initialize Function done --- Multi-Period-Market.js | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/Multi-Period-Market.js b/Multi-Period-Market.js index 71dc66beb..bce2f7c45 100644 --- a/Multi-Period-Market.js +++ b/Multi-Period-Market.js @@ -1,4 +1,4 @@ -exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, FILE_STORAGE) { +exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, FILE_STORAGE, USER_BOT_MODULE, COMMONS_MODULE) { const FULL_LOG = true; const LOG_FILE_CONTENT = false; @@ -7,15 +7,6 @@ const EXCHANGE_NAME = "Poloniex"; - const LRC_FOLDER_NAME = "LRC-Points"; - const PERCENTAGE_BANDWIDTH_FOLDER_NAME = "Percentage-Bandwidth"; - const BOLLINGER_BANDS_FOLDER_NAME = "Bollinger-Bands"; - const BOLLINGER_CHANNELS_FOLDER_NAME = "Bollinger-Channels"; - const BOLLINGER_SUB_CHANNELS_FOLDER_NAME = "Bollinger-Sub-Channels"; - const CANDLES_FOLDER_NAME = "Candles"; - const SIMULATED_RECORDS_FOLDER_NAME = "Trading-Simulation"; - const CONDITIONS_FOLDER_NAME = "Simulation-Conditions"; - const commons = COMMONS.newCommons(bot, logger, UTILITIES); thisObject = { @@ -23,11 +14,6 @@ start: start }; - let gaussStorage = BLOB_STORAGE.newBlobStorage(bot, logger); - let chrisStorage = BLOB_STORAGE.newBlobStorage(bot, logger); - let oliviaStorage = BLOB_STORAGE.newBlobStorage(bot, logger); - let jasonStorage = BLOB_STORAGE.newBlobStorage(bot, logger); - let utilities = UTILITIES.newCloudUtilities(bot, logger); let statusDependencies; @@ -35,6 +21,8 @@ let storages = []; let marketFiles = []; + let usertBot; + return thisObject; function initialize(pStatusDependencies, pDataDependencies, pMonth, pYear, callBackFunction) { @@ -67,7 +55,8 @@ } - callBackFunction(global.DEFAULT_OK_RESPONSE); + usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE); + usertBot.initialize(dataDependencies, pMonth, pYear, callBackFunction); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); From 07a766296c013b204698d62e05c76bef7a8375f2 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 16 Mar 2019 11:40:36 +0100 Subject: [PATCH 06/18] Update Multi-Period-Market.js --- Multi-Period-Market.js | 501 +++-------------------------------------- 1 file changed, 35 insertions(+), 466 deletions(-) diff --git a/Multi-Period-Market.js b/Multi-Period-Market.js index bce2f7c45..49b62d192 100644 --- a/Multi-Period-Market.js +++ b/Multi-Period-Market.js @@ -64,8 +64,6 @@ } } - - function start(callBackFunction) { try { @@ -74,11 +72,11 @@ let market = global.MARKET; - buildRecords(); + processTimePeriods(); - function buildRecords() { + function processTimePeriods() { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> Entering function."); } try { @@ -90,7 +88,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> periodsLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoop -> Entering function."); } /* @@ -104,7 +102,7 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> periodsLoop -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoop -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -113,7 +111,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> Entering function."); } const outputPeriod = global.marketFilesPeriods[n][0]; const timePeriod = global.marketFilesPeriods[n][1]; @@ -131,7 +129,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> getFile -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> getFile -> Entering function."); } let fileName = market.assetA + '_' + market.assetB + ".json"; @@ -144,12 +142,12 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> getFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> getFile -> onFileReceived -> text = " + text); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> getFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> getFile -> onFileReceived -> text = " + text); } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> getFile -> onFileReceived -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> getFile -> onFileReceived -> err = " + err.message); callBackFunction(err); return; @@ -163,13 +161,13 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> getFile -> onFileReceived -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> getFile -> onFileReceived -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> getFile -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> getFile -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -178,476 +176,47 @@ function callTheBot() { - if (fileCount === dataDependencies.length + 1) { - - - - } - } - - - - let LRCMap = new Map(); - let percentageBandwidthMap = new Map(); - let bollingerBandsMap = new Map(); - let bollingerChannelsArray = []; - let bollingerSubChannelsArray = []; - - let candles = []; - let recordsArray = []; - let conditionsArray = []; - let simulationLogic = {}; - - nextLRC(); - - - - function buildLRC() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildLRC -> Entering function."); } - try { - let previous; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> callTheBot -> Entering function."); } - for (let i = 0; i < marketFile.length; i++) { + if (fileCount === dataDependencies.length + 1) { - let LRC = { - begin: marketFile[i][0], - end: marketFile[i][1], - _15: marketFile[i][2], - _30: marketFile[i][3], - _60: marketFile[i][4] - }; + const outputPeriod = global.marketFilesPeriods[n][0]; + const timePeriod = global.marketFilesPeriods[n][1]; - if (previous !== undefined) { + usertBot.initialize(marketFiles, outputPeriod, timePeriod, callBackFunction); - if (previous._15 > LRC._15) { LRC.direction15 = 'down'; } - if (previous._15 < LRC._15) { LRC.direction15 = 'up'; } - if (previous._15 === LRC._15) { LRC.direction15 = 'side'; } + function onBotFinished(err) { - if (previous._30 > LRC._30) { LRC.direction30 = 'down'; } - if (previous._30 < LRC._30) { LRC.direction30 = 'up'; } - if (previous._30 === LRC._30) { LRC.direction30 = 'side'; } + try { - if (previous._60 > LRC._60) { LRC.direction60 = 'down'; } - if (previous._60 < LRC._60) { LRC.direction60 = 'up'; } - if (previous._60 === LRC._60) { LRC.direction60 = 'side'; } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> callTheBot -> onBotFinished -> Entering function."); } - } + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - LRC.previous = previous; - - LRCMap.set(LRC.begin, LRC); - - previous = LRC; - } - - nextPercentageBandwidth(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildLRC -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - - - function buildPercentageBandwidthMap() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildPercentageBandwidthMap -> Entering function."); } - - try { - - let previous; - - for (let i = 0; i < marketFile.length; i++) { - - let percentageBandwidth = { - begin: marketFile[i][0], - end: marketFile[i][1], - value: marketFile[i][2], - movingAverage: marketFile[i][3], - bandwidth: marketFile[i][4] - }; - - if (previous !== undefined) { - - if (previous.movingAverage > percentageBandwidth.movingAverage) { percentageBandwidth.direction = 'down'; } - if (previous.movingAverage < percentageBandwidth.movingAverage) { percentageBandwidth.direction = 'up'; } - if (previous.movingAverage === percentageBandwidth.movingAverage) { percentageBandwidth.direction = 'side'; } - - } - - percentageBandwidth.previous = previous; - - percentageBandwidthMap.set(percentageBandwidth.begin, percentageBandwidth); - - previous = percentageBandwidth; - } - - nextBollingerBands(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildPercentageBandwidthMap -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - - - function buildBollingerBandsMap() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildBollingerBandsMap -> Entering function."); } - - try { - - let previous; - - for (let i = 0; i < marketFile.length; i++) { - - let bollingerBand = { - begin: marketFile[i][0], - end: marketFile[i][1], - movingAverage: marketFile[i][2], - standardDeviation: marketFile[i][3], - deviation: marketFile[i][4] - }; - - if (previous !== undefined) { - - if (previous.movingAverage > bollingerBand.movingAverage) { bollingerBand.direction = 'down'; } - if (previous.movingAverage < bollingerBand.movingAverage) { bollingerBand.direction = 'up'; } - if (previous.movingAverage === bollingerBand.movingAverage) { bollingerBand.direction = 'side'; } - - } - - bollingerBand.previous = previous; - - bollingerBandsMap.set(bollingerBand.begin, bollingerBand); - - previous = bollingerBand; - } - - nextBollingerChannels(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildBollingerBandsMap -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - - - function buildBollingerChannelsArray() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildBollingerChannelsArray -> Entering function."); } - - try { - - let previous; - - for (let i = 0; i < marketFile.length; i++) { - - let bollingerChannel = { - begin: marketFile[i][0], - end: marketFile[i][1], - direction: marketFile[i][2], - period: marketFile[i][3], - firstMovingAverage: marketFile[i][4], - lastMovingAverage: marketFile[i][5], - firstDeviation: marketFile[i][6], - lastDeviation: marketFile[i][7] - }; - - bollingerChannel.previous = previous; - - bollingerChannelsArray.push(bollingerChannel); - - previous = bollingerChannel; - } - - nextBollingerSubChannels(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildBollingerChannelsArray -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - - - function buildBollingerSubChannelsArray() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildBollingerSubChannelsArray -> Entering function."); } - - try { - - let previous; - - for (let i = 0; i < marketFile.length; i++) { - - let bollingerSubChannel = { - begin: marketFile[i][0], - end: marketFile[i][1], - direction: marketFile[i][2], - slope: marketFile[i][3], - period: marketFile[i][4], - firstMovingAverage: marketFile[i][5], - lastMovingAverage: marketFile[i][6], - firstDeviation: marketFile[i][7], - lastDeviation: marketFile[i][8] - }; - - bollingerSubChannel.previous = previous; - - bollingerSubChannelsArray.push(bollingerSubChannel); - - previous = bollingerSubChannel; - } - - nextCandlesFile(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildBollingerSubChannelsArray -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - - function buildCandles() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> buildCandles -> Entering function."); } - - try { - - let previous; - - for (let i = 0; i < marketFile.length; i++) { - - let candle = { - open: undefined, - close: undefined, - min: 10000000000000, - max: 0, - begin: undefined, - end: undefined, - direction: undefined - }; - - candle.min = marketFile[i][0]; - candle.max = marketFile[i][1]; - - candle.open = marketFile[i][2]; - candle.close = marketFile[i][3]; - - candle.begin = marketFile[i][4]; - candle.end = marketFile[i][5]; - - if (candle.open > candle.close) { candle.direction = 'down'; } - if (candle.open < candle.close) { candle.direction = 'up'; } - if (candle.open === candle.close) { candle.direction = 'side'; } - - candle.previous = previous; - - candles.push(candle); - - previous = candle; - } - - commons.runSimulation( - candles, - bollingerBandsMap, - percentageBandwidthMap, - LRCMap, - bollingerChannelsArray, - bollingerSubChannelsArray, - recordsArray, - conditionsArray, - simulationLogic, - outputPeriod, - writeRecordsFile) - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> buildCandles -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function writeRecordsFile() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeRecordsFile -> Entering function."); } - - let separator = ""; - let fileRecordCounter = 0; - - let fileContent = ""; - - for (let i = 0; i < recordsArray.length; i++) { - - let record = recordsArray[i]; - - fileContent = fileContent + separator + '[' + - record.begin + "," + - record.end + "," + - record.type + "," + - record.rate + "," + - record.amount + "," + - record.balanceA + "," + - record.balanceB + "," + - record.profit + "," + - record.lastProfit + "," + - record.stopLoss + "," + - record.roundtrips + "," + - record.hits + "," + - record.fails + "," + - record.hitRatio + "," + - record.ROI + "," + - record.periods + "," + - record.days + "," + - record.anualizedRateOfReturn + "," + - record.sellRate + "," + - record.lastProfitPercent + "," + - record.strategy + "," + - record.strategyPhase + "," + - record.buyOrder + "," + - record.stopLossPhase + "," + - record.buyOrderPhase + "]"; - - if (separator === "") { separator = ","; } - - fileRecordCounter++; - - } - - fileContent = "[" + fileContent + "]"; - - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePathRoot = bot.devTeam + "/" + bot.codeName + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + SIMULATED_RECORDS_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; - - jasonStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); - - function onFileCreated(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> fileContent = " + fileContent); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> err = " + err.message); - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> filePath = " + filePath); - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> market = " + market.assetA + "_" + market.assetB); - - callBackFunction(err); - return; + callBackFunction(err); + return; + } + controlLoop(); } - - writeConditionsFile(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeRecordsFile -> onFileCreated -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeRecordsFile -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function writeConditionsFile() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeConditionsFile -> Entering function."); } - - let separator = ""; - let fileRecordCounter = 0; - - let fileContent = ""; - - for (let i = 0; i < conditionsArray.length; i++) { - - let record = conditionsArray[i]; - - let conditions = ""; - let conditionsSeparator = ""; - - for (let j = 0; j < record.length - 1; j++) { - conditions = conditions + conditionsSeparator + record[j]; - if (conditionsSeparator === "") { conditionsSeparator = ","; } - } - - conditions = conditions + conditionsSeparator + '[' + record[record.length - 1] + ']'; // The last item contains an Array of condition values. - - fileContent = fileContent + separator + '[' + conditions + ']'; - - if (separator === "") { separator = ","; } - - fileRecordCounter++; - - } - - fileContent = "[" + fileContent + "]"; - fileContent = "[" + JSON.stringify(simulationLogic) + "," + fileContent + "]"; - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePathRoot = bot.devTeam + "/" + bot.codeName + "." + bot.version.major + "." + bot.version.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + CONDITIONS_FOLDER_NAME + "/" + "Multi-Period-Market" + "/" + timePeriod; - - jasonStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); - - function onFileCreated(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> fileContent = " + fileContent); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> err = " + err.message); - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> filePath = " + filePath); - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> market = " + market.assetA + "_" + market.assetB); - - callBackFunction(err); - return; - + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> callTheBot -> onBotFinished -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); } - - controlLoop(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeConditionsFile -> onFileCreated -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } + } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> writeConditionsFile -> err = " + err.message); + catch(err){ + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> callTheBot -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> loopBody -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -656,7 +225,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildRecords -> controlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> controlLoop -> Entering function."); } n++; @@ -671,14 +240,14 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> controlLoop -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> controlLoop -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> buildRecords -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } From fdbfad1e6753799abc9446298e77fda883302e51 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 16 Mar 2019 12:38:13 +0100 Subject: [PATCH 07/18] Debugging, Data Confirg added at data Dependencies --- DataDependencies.js | 13 +- IndicatorBotProcessMainLoop.js | 246 +++++++++++++++++- ...i-Period-Market.js => MultiPeriodMarket.js | 6 +- Project.njsproj | 3 + 4 files changed, 258 insertions(+), 10 deletions(-) rename Multi-Period-Market.js => MultiPeriodMarket.js (97%) diff --git a/DataDependencies.js b/DataDependencies.js index 0660226b9..df98108cc 100644 --- a/DataDependencies.js +++ b/DataDependencies.js @@ -9,6 +9,7 @@ let ownerBot; // This is the bot owner of the Data Set. let thisObject = { + config: undefined, dataSets: new Map(), initialize: initialize, keys: [] @@ -23,7 +24,9 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } - if (pDataDependenciesConfig === undefined) { + thisObject.config = pDataDependenciesConfig; + + if (thisObject.config === undefined) { // We allow old indicators not to declare their data dependencies. @@ -38,11 +41,11 @@ let alreadyCalledBack = false; let addCount = 0; - for (let i = 0; i < pDataDependenciesConfig.length; i++) { + for (let i = 0; i < thisObject.config.length; i++) { let dataSetModule = DATA_SET.newDataSet(BOT, logger, BLOB_STORAGE, UTILITIES); - dataSetModule.initialize(pDataDependenciesConfig[i], onInitilized); + dataSetModule.initialize(thisObject.config[i], onInitilized); function onInitilized(err) { @@ -65,14 +68,14 @@ let key; - key = pDataDependenciesConfig[i].devTeam + "-" + pDataDependenciesConfig[i].bot + "-" + pDataDependenciesConfig[i].product + "-" + pDataDependenciesConfig[i].dataSet + "-" + pDataDependenciesConfig[i].dataSetVersion; + key = thisObject.config[i].devTeam + "-" + thisObject.config[i].bot + "-" + thisObject.config[i].product + "-" + thisObject.config[i].dataSet + "-" + thisObject.config[i].dataSetVersion; thisObject.keys.push(key); thisObject.dataSets.set(key, dataSetModule); if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> addDataSet -> DataSet added to Map. -> key = " + key); } - if (addCount === pDataDependenciesConfig.length) { + if (addCount === thisObject.config.length) { if (alreadyCalledBack === false) { callBackFunction(global.DEFAULT_OK_RESPONSE); return; diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 88347779d..0b1b0013a 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -9,6 +9,7 @@ let COMMONS_MODULE; const BLOB_STORAGE = require(ROOT_DIR + 'BlobStorage'); + const MULTI_PERIOD_MARKET = require(ROOT_DIR + 'MultiPeriodMarket'); const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); let logger; // We need this here in order for the loopHealth function to work and be able to rescue the loop when it gets in trouble. @@ -190,6 +191,7 @@ /* We will prepare first the infraestructure needed for the bot to run. There are 3 modules we need to sucessfullly initialize first. */ let userBot; + let multiPeriodMarket; let statusDependencies; let dataDependencies; @@ -291,7 +293,16 @@ switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> onInizialized -> Execution finished well."); } - initializeUserBot(); + + /* If the process is configured to run inside a framework, we continue there, otherwise we run the bot directly. */ + + switch (processConfig.framework) { + case 'Multi-Period-Market': { + intitializeMultiPeriodMarket(); + break; + } + default: { initializeUserBot();} + } return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. @@ -354,7 +365,7 @@ usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE); - usertBot.initialize(statusDependencies, pMonth, pYear, onInizialized, dataDependencies); + usertBot.initialize(statusDependencies, pMonth, pYear, onInizialized); function onInizialized(err) { @@ -577,6 +588,237 @@ } } + function intitializeMultiPeriodMarket() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeMultiPeriodMarket -> Entering function."); } + + multiPeriodMarket = MULTI_PERIOD_MARKET.newMultiPeriodMarket(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE); + + multiPeriodMarket.initialize(statusDependencies, dataDependencies, pMonth, pYear, onInizialized, dataDependencies); + + function onInizialized(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Entering function."); } + + switch (err.result) { + case global.DEFAULT_OK_RESPONSE.result: { + logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Execution finished well."); + startMultiPeriodMarket(); + return; + } + case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Retry Later. Requesting Execution Retry."); + nextWaitTime = 'Retry'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Operation Failed. Aborting the process."); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + return; + } + case global.CUSTOM_OK_RESPONSE.result: { + + switch (err.message) { + case "Too far in the future.": { + logger.write(MODULE_NAME, "[WARN] run -> loop -> intitializeMultiPeriodMarket -> onInizialized > Too far in the future. This Loop will enter in coma."); + nextWaitTime = 'Coma'; + loopControl(nextWaitTime); + return; + } + case "Not needed now, but soon.": { + logger.write(MODULE_NAME, "[WARN] run -> loop -> intitializeMultiPeriodMarket -> onInizialized > Not needed now, but soon. This Loop will continue with Normal wait time."); + nextWaitTime = 'Normal'; + loopControl(nextWaitTime); + return; + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized > Unhandled custom response received. -> err.message = " + err.message); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + return; + } + } + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Unhandled err.result received. -> err.message = " + err.message); + + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> err = " + err.message); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + } + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> err = " + err.message); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + } + } + + function startMultiPeriodMarket() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startMultiPeriodMarket -> Entering function."); } + + multiPeriodMarket.start(onFinished); + + function onFinished(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startMultiPeriodMarket -> onFinished -> Entering function."); } + + switch (err.result) { + case global.DEFAULT_OK_RESPONSE.result: { + logger.write(MODULE_NAME, "[INFO] run -> loop -> startMultiPeriodMarket -> onFinished -> Execution finished well."); + nextWaitTime = 'Normal'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> Retry Later. Requesting Execution Retry."); + nextWaitTime = 'Retry'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> Operation Failed. Aborting the process."); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + return; + } + case global.CUSTOM_OK_RESPONSE.result: { + + switch (err.message) { + case "Dependency does not exist.": { + logger.write(MODULE_NAME, "[WARN] run -> loop -> startMultiPeriodMarket -> onFinished -> Dependency does not exist. This Loop will go to sleep."); + nextWaitTime = 'Sleep'; + loopControl(nextWaitTime); + return; + } + case "Dependency not ready.": { + logger.write(MODULE_NAME, "[WARN] run -> loop -> startMultiPeriodMarket -> onFinished -> Dependency not ready. This Loop will go to sleep."); + nextWaitTime = 'Sleep'; + loopControl(nextWaitTime); + return; + } + case "Month before it is needed.": { + logger.write(MODULE_NAME, "[WARN] run -> loop -> startMultiPeriodMarket -> onFinished -> Month before it is needed. This Loop will be terminated."); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + } + case "Month fully processed.": { + logger.write(MODULE_NAME, "[WARN] run -> loop -> startMultiPeriodMarket -> onFinished -> Month fully processed. This Loop will be terminated."); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + } + case "End of the month reached.": { + logger.write(MODULE_NAME, "[WARN] run -> loop -> startMultiPeriodMarket -> onFinished -> End of the month reached. This Loop will be terminated."); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> Unhandled custom response received. -> err.message = " + err.message); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + return; + } + } + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> Unhandled err.result received. -> err.message = " + err.message); + + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> err = " + err.message); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + } + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> err = " + err.message); + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + clearTimeout(checkLoopHealthHandle); + bot.enableCheckLoopHealth = false; + callBackFunction(err); + } + } + function loopControl(nextWaitTime) { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Entering function."); } diff --git a/Multi-Period-Market.js b/MultiPeriodMarket.js similarity index 97% rename from Multi-Period-Market.js rename to MultiPeriodMarket.js index 49b62d192..719ea5683 100644 --- a/Multi-Period-Market.js +++ b/MultiPeriodMarket.js @@ -1,9 +1,9 @@ -exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, FILE_STORAGE, USER_BOT_MODULE, COMMONS_MODULE) { +exports.newMultiPeriodMarket = function newMultiPeriodMarket(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, FILE_STORAGE, USER_BOT_MODULE, COMMONS_MODULE) { const FULL_LOG = true; const LOG_FILE_CONTENT = false; - const MODULE_NAME = "User Bot"; + const MODULE_NAME = "Multi Period Market"; const EXCHANGE_NAME = "Poloniex"; @@ -37,7 +37,7 @@ statusDependencies = pStatusDependencies; dataDependencies = pDataDependencies; - for (let i = 0; i < dataDependencies.length; i++) { + for (let i = 0; i < dataDependencies.dataSets.Entries.length; i++) { let key; let storage; diff --git a/Project.njsproj b/Project.njsproj index 9edd19db5..3c981ef93 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -33,6 +33,9 @@ Code + + Code + Code From 03ea78988a79d2a3777d9f04469b8a6a28bdbbf5 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 16 Mar 2019 13:12:23 +0100 Subject: [PATCH 08/18] For transformed into a Loop --- DataSet.js | 1 - IndicatorBotProcessMainLoop.js | 2 +- MultiPeriodMarket.js | 168 +++++++++++++++++++-------------- 3 files changed, 100 insertions(+), 71 deletions(-) diff --git a/DataSet.js b/DataSet.js index 074135d0a..215137939 100644 --- a/DataSet.js +++ b/DataSet.js @@ -5,7 +5,6 @@ let bot = BOT; let thisObject = { - file: undefined, // Here we have the JSON object representing the file content. initialize: initialize, getTextFile: getTextFile, createTextFile: createTextFile diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 0b1b0013a..0aa0046e0 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -594,7 +594,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeMultiPeriodMarket -> Entering function."); } - multiPeriodMarket = MULTI_PERIOD_MARKET.newMultiPeriodMarket(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE); + multiPeriodMarket = MULTI_PERIOD_MARKET.newMultiPeriodMarket(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE); multiPeriodMarket.initialize(statusDependencies, dataDependencies, pMonth, pYear, onInizialized, dataDependencies); diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index 719ea5683..e3840bfb4 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -1,4 +1,4 @@ -exports.newMultiPeriodMarket = function newMultiPeriodMarket(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, FILE_STORAGE, USER_BOT_MODULE, COMMONS_MODULE) { +exports.newMultiPeriodMarket = function newMultiPeriodMarket(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE) { const FULL_LOG = true; const LOG_FILE_CONTENT = false; @@ -37,11 +37,11 @@ statusDependencies = pStatusDependencies; dataDependencies = pDataDependencies; - for (let i = 0; i < dataDependencies.dataSets.Entries.length; i++) { + for (let i = 0; i < dataDependencies.config.length; i++) { let key; let storage; - let dependency = dataDependencies[i]; + let dependency = dataDependencies.config[i]; key = dependency.devTeam + "-" + dependency.bot + "-" + @@ -98,7 +98,7 @@ n = 0 // loop Variable representing each possible period as defined at the periods array. - loopBody(); + periodsLoopBody(); } catch (err) { @@ -107,131 +107,161 @@ } } - function loopBody() { + function periodsLoopBody() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Entering function."); } const outputPeriod = global.marketFilesPeriods[n][0]; const timePeriod = global.marketFilesPeriods[n][1]; - let fileCount = 0; + let dependencyIndex = 0; + dependencyLoopBody(); - for (let i = 0; i < dataDependencies.length; i++) { + function dependencyLoopBody() { - let dependency = dataDependencies[i]; - let storage = storages[i]; + try { - getFile(); + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } - function getFile() { + let dependency = dataDependencies.config[dependencyIndex]; + let storage = storages[dependencyIndex]; - try { + getFile(); - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> getFile -> Entering function."); } + function getFile() { - let fileName = market.assetA + '_' + market.assetB + ".json"; + try { - let filePathRoot = dependency.devTeam + "/" + dependency.bot + "." + dependency.botVersion.major + "." + dependency.botVersion.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + dependency.dataSetVersion; - let filePath = filePathRoot + "/Output/" + dependency.product + "/" + "Multi-Period-Market" + "/" + timePeriod; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getFile -> Entering function."); } - storage.getTextFile(filePath, fileName, onFileReceived, true); + let fileName = market.assetA + '_' + market.assetB + ".json"; - function onFileReceived(err, text) { + let filePathRoot = dependency.devTeam + "/" + dependency.bot + "." + dependency.botVersion.major + "." + dependency.botVersion.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + dependency.dataSetVersion; + let filePath = filePathRoot + "/Output/" + dependency.product + "/" + "Multi-Period-Market" + "/" + timePeriod; - try { + storage.getTextFile(filePath, fileName, onFileReceived, true); - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> getFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> getFile -> onFileReceived -> text = " + text); } + function onFileReceived(err, text) { - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + try { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> getFile -> onFileReceived -> err = " + err.message); - callBackFunction(err); - return; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message); + callBackFunction(err); + return; + + } + + let marketFile = JSON.parse(text); + marketFiles.push(marketFile); + fileCount++; + + callTheBot(); } - - let marketFile = JSON.parse(text); - marketFiles.push(marketFile); - fileCount++; - - callTheBot(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> getFile -> onFileReceived -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } } } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getFile -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> getFile -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoop -> dependencyLoopBody -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } } + function dependencyControlLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> dependencyControlLoop -> Entering function."); } + + dependencyIndex++; + + if (dependencyIndex < dataDependencies.config.length) { + + dependencyLoopBody(); + + } else { + + callTheBot(); + + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> dependencyControlLoop -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + function callTheBot() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> callTheBot -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> Entering function."); } - if (fileCount === dataDependencies.length + 1) { + const outputPeriod = global.marketFilesPeriods[n][0]; + const timePeriod = global.marketFilesPeriods[n][1]; - const outputPeriod = global.marketFilesPeriods[n][0]; - const timePeriod = global.marketFilesPeriods[n][1]; + usertBot.initialize(marketFiles, outputPeriod, timePeriod, callBackFunction); - usertBot.initialize(marketFiles, outputPeriod, timePeriod, callBackFunction); + function onBotFinished(err) { - function onBotFinished(err) { + try { - try { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> onBotFinished -> Entering function."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> loopBody -> callTheBot -> onBotFinished -> Entering function."); } + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - callBackFunction(err); - return; - } - - controlLoop(); - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> callTheBot -> onBotFinished -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); + callBackFunction(err); + return; } + + periodsControlLoop(); + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> onBotFinished -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } - } catch(err){ - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> callTheBot -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> loopBody -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } - function controlLoop() { + function periodsControlLoop() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> controlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsControlLoop -> Entering function."); } n++; if (n < global.marketFilesPeriods.length) { - loopBody(); + periodsLoopBody(); } else { @@ -240,7 +270,7 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> controlLoop -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsControlLoop -> err = " + err.message); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -258,7 +288,7 @@ try { - let reportKey = "AAMasters" + "-" + "AAJason" + "-" + "Multi-Period-Market" + "-" + "dataSet.V1"; + let reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period-Market" + "-" + "dataSet.V1"; let thisReport = statusDependencies.statusReports.get(reportKey); thisReport.file.lastExecution = bot.processDatetime; From d2a88e2b5435296085ccc2a9bbe393a66ca90bdb Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 16 Mar 2019 14:02:48 +0100 Subject: [PATCH 09/18] Multi Period Market Framework, ready! --- MultiPeriodMarket.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index e3840bfb4..b360236be 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -138,8 +138,7 @@ let fileName = market.assetA + '_' + market.assetB + ".json"; - let filePathRoot = dependency.devTeam + "/" + dependency.bot + "." + dependency.botVersion.major + "." + dependency.botVersion.minor + "/" + global.PLATFORM_CONFIG.codeName + "." + global.PLATFORM_CONFIG.version.major + "." + global.PLATFORM_CONFIG.version.minor + "/" + global.EXCHANGE_NAME + "/" + dependency.dataSetVersion; - let filePath = filePathRoot + "/Output/" + dependency.product + "/" + "Multi-Period-Market" + "/" + timePeriod; + let filePath = dependency.product + "/" + "Multi-Period-Market" + "/" + timePeriod; storage.getTextFile(filePath, fileName, onFileReceived, true); @@ -153,16 +152,16 @@ if (err.result !== global.DEFAULT_OK_RESPONSE.result) { logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> filePath = " + filePath); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> fileName = " + fileName); callBackFunction(err); return; - } let marketFile = JSON.parse(text); marketFiles.push(marketFile); - fileCount++; - callTheBot(); + dependencyControlLoop(); } catch (err) { @@ -217,7 +216,7 @@ const outputPeriod = global.marketFilesPeriods[n][0]; const timePeriod = global.marketFilesPeriods[n][1]; - usertBot.initialize(marketFiles, outputPeriod, timePeriod, callBackFunction); + usertBot.start(marketFiles, outputPeriod, timePeriod, callBackFunction); function onBotFinished(err) { From 07144f0385042fcb44c2cb63c150125d96c132a6 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 16 Mar 2019 15:36:59 +0100 Subject: [PATCH 10/18] dataFiles --- MultiPeriodMarket.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index b360236be..53d4a3776 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -19,7 +19,7 @@ let statusDependencies; let dataDependencies; let storages = []; - let marketFiles = []; + let dataFiles = []; let usertBot; @@ -159,7 +159,7 @@ } let marketFile = JSON.parse(text); - marketFiles.push(marketFile); + dataFiles.push(marketFile); dependencyControlLoop(); @@ -216,7 +216,7 @@ const outputPeriod = global.marketFilesPeriods[n][0]; const timePeriod = global.marketFilesPeriods[n][1]; - usertBot.start(marketFiles, outputPeriod, timePeriod, callBackFunction); + usertBot.start(dataFiles, outputPeriod, timePeriod, callBackFunction); function onBotFinished(err) { From 882bfe91393caa3520301429a39502d34344d13a Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 16 Mar 2019 16:01:42 +0100 Subject: [PATCH 11/18] Bug, array not being cleanned. --- MultiPeriodMarket.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index 53d4a3776..a331941b8 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -117,6 +117,8 @@ const timePeriod = global.marketFilesPeriods[n][1]; let dependencyIndex = 0; + dataFiles = []; + dependencyLoopBody(); function dependencyLoopBody() { @@ -158,8 +160,8 @@ return; } - let marketFile = JSON.parse(text); - dataFiles.push(marketFile); + let dataFile = JSON.parse(text); + dataFiles.push(dataFile); dependencyControlLoop(); @@ -216,7 +218,7 @@ const outputPeriod = global.marketFilesPeriods[n][0]; const timePeriod = global.marketFilesPeriods[n][1]; - usertBot.start(dataFiles, outputPeriod, timePeriod, callBackFunction); + usertBot.start(dataFiles, outputPeriod, timePeriod, onBotFinished); function onBotFinished(err) { From a9d386a0607f9b53bddbe7d9175b36b5764d1a03 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Mar 2019 08:35:30 +0100 Subject: [PATCH 12/18] Multi Period Daily implemented --- MultiPeriodDaily.js | 657 ++++++++++++++++++++++++++++++++++++++++++++ Project.njsproj | 3 + 2 files changed, 660 insertions(+) create mode 100644 MultiPeriodDaily.js diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js new file mode 100644 index 000000000..9f2ef1466 --- /dev/null +++ b/MultiPeriodDaily.js @@ -0,0 +1,657 @@ +exports.newMultiPeriodDaily = function newMultiPeriodDaily(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE) { + + const FULL_LOG = true; + const LOG_FILE_CONTENT = false; + + const MODULE_NAME = "Multi Period Daily"; + + const EXCHANGE_NAME = "Poloniex"; + + const commons = COMMONS.newCommons(bot, logger, UTILITIES); + + thisObject = { + initialize: initialize, + start: start + }; + + let utilities = UTILITIES.newCloudUtilities(bot, logger); + + let statusDependencies; + let dataDependencies; + let storages = []; + let dataFiles = []; + + let usertBot; + let currentBotStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + + return thisObject; + + function initialize(pStatusDependencies, pDataDependencies, pMonth, pYear, callBackFunction) { + + try { + + logger.fileName = MODULE_NAME; + logger.initialize(); + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } + + statusDependencies = pStatusDependencies; + dataDependencies = pDataDependencies; + + for (let i = 0; i < dataDependencies.config.length; i++) { + + let key; + let storage; + let dependency = dataDependencies.config[i]; + + key = dependency.devTeam + "-" + + dependency.bot + "-" + + dependency.product + "-" + + dependency.dataSet + "-" + + dependency.dataSetVersion + + storage = dataDependencies.dataSets.get(key); + + storages.push(storage); + + } + + currentBotStorage.initialize(bot.devTeam, onStorageInizialized); + + function onStorageInizialized(err) { + + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + + usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE); + usertBot.initialize(dataDependencies, pMonth, pYear, callBackFunction); + + } else { + logger.write(MODULE_NAME, "[ERROR] initialize -> onStorageInizialized -> err = " + err.message); + callBackFunction(err); + } + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function start(callBackFunction) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); } + + let market = global.MARKET; + + /* Context Variables */ + + let contextVariables = { + lastFile: undefined, // Datetime of the last file files sucessfully produced by this process. + dateBeginingOfMarket: undefined, // Datetime of the first trade file in the whole market history. + dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process. + }; + + let previousDay; // Holds the date of the previous day relative to the processing date. + let currentDate; // Holds the processing date. + + getContextVariables(); + + function getContextVariables() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Entering function."); } + + let thisReport; + let reportKey; + let statusReport; + + /* We look first for Charly in order to get when the market starts. */ + + reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Poloniex-Historic-Trades" + "-" + "dataSet.V1"; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } + + statusReport = statusDependencies.statusReports.get(reportKey); + + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + thisReport = statusDependencies.statusReports.get(reportKey).file; + + if (thisReport.lastFile === undefined) { + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); + logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet."); + + let customOK = { + result: global.CUSTOM_OK_RESPONSE.result, + message: "Dependency does not exist." + } + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); + callBackFunction(customOK); + return; + } + + contextVariables.dateBeginingOfMarket = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS); + + /* Second, we get the report from Olivia, to know when the marted ends. */ + + reportKey = "AAMasters" + "-" + "AAOlivia" + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } + + statusReport = statusDependencies.statusReports.get(reportKey); + + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + thisReport = statusDependencies.statusReports.get(reportKey).file; + + if (thisReport.lastFile === undefined) { + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); + + let customOK = { + result: global.CUSTOM_OK_RESPONSE.result, + message: "Dependency not ready." + } + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); + callBackFunction(customOK); + return; + } + + contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf()); + + /* Finally we get our own Status Report. */ + + reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } + + statusReport = statusDependencies.statusReports.get(reportKey); + + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + + thisReport = statusDependencies.statusReports.get(reportKey).file; + + if (thisReport.lastFile !== undefined) { + + contextVariables.lastFile = new Date(thisReport.lastFile); + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> thisReport.lastFile !== undefined"); } + + processTimePeriods(); + return; + + } else { + + /* + In the case when there is no status report, we take the date of the file with the first trades as the begining of the market. Then we will + go one day further in time, so that the previous day does fine a file at the begining of the market. + */ + + contextVariables.lastFile = new Date(contextVariables.dateBeginingOfMarket.getUTCFullYear() + "-" + (contextVariables.dateBeginingOfMarket.getUTCMonth() + 1) + "-" + contextVariables.dateBeginingOfMarket.getUTCDate() + " " + "00:00" + GMT_SECONDS); + contextVariables.lastFile = new Date(contextVariables.lastFile.valueOf() + ONE_DAY_IN_MILISECONDS); + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> thisReport.lastFile === undefined"); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> contextVariables.lastFile = " + contextVariables.lastFile); } + + processTimePeriods(); + return; + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> err = " + err.message); + if (err.message === "Cannot read property 'file' of undefined") { + logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Check the bot configuration to see if all of its statusDependencies declarations are correct. "); + logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys)); + } + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function processTimePeriods() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> Entering function."); } + + try { + + let n; + + advanceTime(); + + function advanceTime() { + + try { + + logger.newInternalLoop(bot.codeName, bot.process); + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> Entering function."); } + + currentDate = new Date(currentDate.valueOf() + ONE_DAY_IN_MILISECONDS); + previousDay = new Date(currentDate.valueOf() - ONE_DAY_IN_MILISECONDS); + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> currentDate = " + currentDate); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> previousDay = " + previousDay); } + + /* Validation that we are not going past the head of the market. */ + + if (currentDate.valueOf() > contextVariables.dateEndOfMarket.valueOf()) { + + const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> " + logText); } + + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + + } + + periodsLoop(); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> advanceTime -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoop -> Entering function."); } + + /* + + We will iterate through all posible periods. + + */ + + n = 0 // loop Variable representing each possible period as defined at the periods array. + + periodsLoopBody(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoop -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsLoopBody() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Entering function."); } + + const outputPeriod = global.dailyFilesPeriods[n][0]; + const timePeriod = global.dailyFilesPeriods[n][1]; + + let dependencyIndex = 0; + dataFiles = []; + + dependencyLoopBody(); + + function dependencyLoopBody() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } + + let dependency = dataDependencies.config[dependencyIndex]; + let storage = storages[dependencyIndex]; + + let previousFile; + let currentFile; + + getPreviousFile(); + + function getPreviousFile() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } + + let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2); + let filePath = dependency.product + '/' + "Multi-Period-Daily" + "/" + timePeriod + "/" + dateForPath; + let fileName = market.assetA + '_' + market.assetB + ".json"; + + storage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> filePath = " + filePath); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> fileName = " + fileName); + callBackFunction(err); + return; + } + + previousFile = JSON.parse(text); + + getCurrentFile(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function getCurrentFile() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } + + let dateForPath = currentDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2); + let filePath = dependency.product + '/' + "Multi-Period-Daily" + "/" + timePeriod + "/" + dateForPath; + let fileName = market.assetA + '_' + market.assetB + ".json"; + + storage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> filePath = " + filePath); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> fileName = " + fileName); + callBackFunction(err); + return; + } + + currentFile = JSON.parse(text); + + let datafile = previousFile.concat(currentFile); + + dataFiles.push(dataFile); + dependencyControlLoop(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoop -> dependencyLoopBody -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function dependencyControlLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> dependencyControlLoop -> Entering function."); } + + dependencyIndex++; + + if (dependencyIndex < dataDependencies.config.length) { + + dependencyLoopBody(); + + } else { + + callTheBot(); + + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> dependencyControlLoop -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function callTheBot() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> Entering function."); } + + const outputPeriod = global.dailyFilesPeriods[n][0]; + const timePeriod = global.dailyFilesPeriods[n][1]; + + usertBot.start(dataFiles, outputPeriod, timePeriod, onBotFinished); + + function onBotFinished(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> onBotFinished -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + callBackFunction(err); + return; + } + + periodsControlLoop(); + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> onBotFinished -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsControlLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsControlLoop -> Entering function."); } + + n++; + + if (n < global.dailyFilesPeriods.length) { + + periodsLoopBody(); + + } else { + + n = 0; + + writeDataRanges(onWritten); + + function onWritten(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> controlLoop -> onWritten -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> controlLoop -> onWritten -> err = " + err.message); + callBackFunction(err); + return; + } + + writeStatusReport(currentDate, advanceTime); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> controlLoop -> onWritten -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsControlLoop -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function writeDataRanges(callBack) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRanges -> Entering function."); } + + let productIndex = 0; + productLoopBody(); + + function productLoopBody() { + + let folderName = bot.products[productIndex].codeName; + + writeDataRange(contextVariables.dateBeginingOfMarket, currentDate, folderName, controlLoop); + } + + function controlLoop() { + + productIndex++; + + if (productIndex < bot.products.length) { + productLoopBody(); + } else { + callBack(global.DEFAULT_OK_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> writeDataRanges -> err = " + err.message); + callBack(global.DEFAULT_FAIL_RESPONSE); + } + + } + + function writeDataRange(pBegin, pEnd, pProductFolder, callBack) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> Entering function."); } + + let dataRange = { + begin: pBegin.valueOf(), + end: pEnd.valueOf() + }; + + let fileContent = JSON.stringify(dataRange); + + let fileName = 'Data.Range.' + market.assetA + '_' + market.assetB + '.json'; + let filePath = bot.filePathRoot + "/Output/" + pProductFolder + "/" + bot.process; + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> fileName = " + fileName); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> filePath = " + filePath); } + + currentBotStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + + function onFileCreated(err) { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> onFileCreated -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write(MODULE_NAME, "[ERROR] start -> writeDataRange -> onFileCreated -> err = " + err.message); + callBack(err); + return; + } + + if (LOG_FILE_CONTENT === true) { + logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> onFileCreated -> Content written = " + fileContent); + } + + callBack(global.DEFAULT_OK_RESPONSE); + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> writeDataRange -> err = " + err.message); + callBack(global.DEFAULT_FAIL_RESPONSE); + } + } + + function writeStatusReport(lastFileDate, callBack) { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> lastFileDate = " + lastFileDate); } + + try { + + let reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; + let thisReport = statusDependencies.statusReports.get(reportKey); + + thisReport.file.lastExecution = bot.currentDatetime; + thisReport.file.lastFile = lastFileDate; + thisReport.save(callBack); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } +}; diff --git a/Project.njsproj b/Project.njsproj index 3c981ef93..c9cb27126 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -33,6 +33,9 @@ Code + + Code + Code From a3ddec4831d3572e91f3a0c3e9b72faa7328be91 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Mar 2019 08:47:59 +0100 Subject: [PATCH 13/18] processFramework Implemented --- IndicatorBotProcessMainLoop.js | 77 ++++++++++++++++++---------------- MultiPeriodDaily.js | 4 +- MultiPeriodMarket.js | 4 +- 3 files changed, 45 insertions(+), 40 deletions(-) diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 0aa0046e0..58f34ccce 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -10,6 +10,7 @@ const BLOB_STORAGE = require(ROOT_DIR + 'BlobStorage'); const MULTI_PERIOD_MARKET = require(ROOT_DIR + 'MultiPeriodMarket'); + const MULTI_PERIOD_DAILY = require(ROOT_DIR + 'MultiPeriodDaily'); const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); let logger; // We need this here in order for the loopHealth function to work and be able to rescue the loop when it gets in trouble. @@ -191,7 +192,7 @@ /* We will prepare first the infraestructure needed for the bot to run. There are 3 modules we need to sucessfullly initialize first. */ let userBot; - let multiPeriodMarket; + let processFramework; let statusDependencies; let dataDependencies; @@ -298,7 +299,13 @@ switch (processConfig.framework) { case 'Multi-Period-Market': { - intitializeMultiPeriodMarket(); + processFramework = MULTI_PERIOD_MARKET.newMultiPeriodMarket(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE); + intitializeProcessFramework(); + break; + } + case 'Multi-Period-Daily': { + processFramework = MULTI_PERIOD_DAILY.newMultiPeriodDaily(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE); + intitializeProcessFramework(); break; } default: { initializeUserBot();} @@ -588,36 +595,34 @@ } } - function intitializeMultiPeriodMarket() { + function intitializeProcessFramework() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeMultiPeriodMarket -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeProcessFramework -> Entering function."); } - multiPeriodMarket = MULTI_PERIOD_MARKET.newMultiPeriodMarket(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE); - - multiPeriodMarket.initialize(statusDependencies, dataDependencies, pMonth, pYear, onInizialized, dataDependencies); + processFramework.initialize(statusDependencies, dataDependencies, onInizialized, dataDependencies); function onInizialized(err) { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeProcessFramework -> onInizialized -> Entering function."); } switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { - logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Execution finished well."); - startMultiPeriodMarket(); + logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeProcessFramework -> onInizialized -> Execution finished well."); + startProcessFramework(); return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Operation Failed. Aborting the process."); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -630,19 +635,19 @@ switch (err.message) { case "Too far in the future.": { - logger.write(MODULE_NAME, "[WARN] run -> loop -> intitializeMultiPeriodMarket -> onInizialized > Too far in the future. This Loop will enter in coma."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> intitializeProcessFramework -> onInizialized > Too far in the future. This Loop will enter in coma."); nextWaitTime = 'Coma'; loopControl(nextWaitTime); return; } case "Not needed now, but soon.": { - logger.write(MODULE_NAME, "[WARN] run -> loop -> intitializeMultiPeriodMarket -> onInizialized > Not needed now, but soon. This Loop will continue with Normal wait time."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> intitializeProcessFramework -> onInizialized > Not needed now, but soon. This Loop will continue with Normal wait time."); nextWaitTime = 'Normal'; loopControl(nextWaitTime); return; } default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized > Unhandled custom response received. -> err.message = " + err.message); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized > Unhandled custom response received. -> err.message = " + err.message); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -654,8 +659,8 @@ } } default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> Unhandled err.result received. -> err.message = " + err.message); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> Unhandled err.result received. -> err.message = " + err.message); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -668,7 +673,7 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> onInizialized -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> err = " + err.message); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -679,7 +684,7 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeMultiPeriodMarket -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> err = " + err.message); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -689,35 +694,35 @@ } } - function startMultiPeriodMarket() { + function startProcessFramework() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startMultiPeriodMarket -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> Entering function."); } - multiPeriodMarket.start(onFinished); + processFramework.start(onFinished); function onFinished(err) { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startMultiPeriodMarket -> onFinished -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> onFinished -> Entering function."); } switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { - logger.write(MODULE_NAME, "[INFO] run -> loop -> startMultiPeriodMarket -> onFinished -> Execution finished well."); + logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> onFinished -> Execution finished well."); nextWaitTime = 'Normal'; loopControl(nextWaitTime); return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> Operation Failed. Aborting the process."); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -730,19 +735,19 @@ switch (err.message) { case "Dependency does not exist.": { - logger.write(MODULE_NAME, "[WARN] run -> loop -> startMultiPeriodMarket -> onFinished -> Dependency does not exist. This Loop will go to sleep."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> startProcessFramework -> onFinished -> Dependency does not exist. This Loop will go to sleep."); nextWaitTime = 'Sleep'; loopControl(nextWaitTime); return; } case "Dependency not ready.": { - logger.write(MODULE_NAME, "[WARN] run -> loop -> startMultiPeriodMarket -> onFinished -> Dependency not ready. This Loop will go to sleep."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> startProcessFramework -> onFinished -> Dependency not ready. This Loop will go to sleep."); nextWaitTime = 'Sleep'; loopControl(nextWaitTime); return; } case "Month before it is needed.": { - logger.write(MODULE_NAME, "[WARN] run -> loop -> startMultiPeriodMarket -> onFinished -> Month before it is needed. This Loop will be terminated."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> startProcessFramework -> onFinished -> Month before it is needed. This Loop will be terminated."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -752,7 +757,7 @@ return; } case "Month fully processed.": { - logger.write(MODULE_NAME, "[WARN] run -> loop -> startMultiPeriodMarket -> onFinished -> Month fully processed. This Loop will be terminated."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> startProcessFramework -> onFinished -> Month fully processed. This Loop will be terminated."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -762,7 +767,7 @@ return; } case "End of the month reached.": { - logger.write(MODULE_NAME, "[WARN] run -> loop -> startMultiPeriodMarket -> onFinished -> End of the month reached. This Loop will be terminated."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> startProcessFramework -> onFinished -> End of the month reached. This Loop will be terminated."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -772,7 +777,7 @@ return; } default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> Unhandled custom response received. -> err.message = " + err.message); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Unhandled custom response received. -> err.message = " + err.message); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -784,8 +789,8 @@ } } default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> Unhandled err.result received. -> err.result = " + err.result); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> Unhandled err.result received. -> err.message = " + err.message); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Unhandled err.result received. -> err.message = " + err.message); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -798,7 +803,7 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> onFinished -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> err = " + err.message); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -809,7 +814,7 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startMultiPeriodMarket -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> err = " + err.message); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 9f2ef1466..194497a1c 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -26,7 +26,7 @@ return thisObject; - function initialize(pStatusDependencies, pDataDependencies, pMonth, pYear, callBackFunction) { + function initialize(pStatusDependencies, pDataDependencies, callBackFunction) { try { @@ -63,7 +63,7 @@ if (err.result === global.DEFAULT_OK_RESPONSE.result) { usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE); - usertBot.initialize(dataDependencies, pMonth, pYear, callBackFunction); + usertBot.initialize(dataDependencies, callBackFunction); } else { logger.write(MODULE_NAME, "[ERROR] initialize -> onStorageInizialized -> err = " + err.message); diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index a331941b8..485a087e0 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -25,7 +25,7 @@ return thisObject; - function initialize(pStatusDependencies, pDataDependencies, pMonth, pYear, callBackFunction) { + function initialize(pStatusDependencies, pDataDependencies, callBackFunction) { try { @@ -56,7 +56,7 @@ } usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE); - usertBot.initialize(dataDependencies, pMonth, pYear, callBackFunction); + usertBot.initialize(dataDependencies, callBackFunction); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); From f08383d65b4df330c22458ed83c03add129bcace Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Mar 2019 09:15:44 +0100 Subject: [PATCH 14/18] Sending current date to the bot --- MultiPeriodDaily.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 194497a1c..46e72f39b 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -463,7 +463,7 @@ const outputPeriod = global.dailyFilesPeriods[n][0]; const timePeriod = global.dailyFilesPeriods[n][1]; - usertBot.start(dataFiles, outputPeriod, timePeriod, onBotFinished); + usertBot.start(dataFiles, outputPeriod, timePeriod, currentDate, onBotFinished); function onBotFinished(err) { From 9197efc3ad9acbeded96b2c5eb758d8e668b6ce6 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Mar 2019 09:28:53 +0100 Subject: [PATCH 15/18] Better naming --- MultiPeriodDaily.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 46e72f39b..01108f4df 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -89,8 +89,8 @@ let contextVariables = { lastFile: undefined, // Datetime of the last file files sucessfully produced by this process. - dateBeginingOfMarket: undefined, // Datetime of the first trade file in the whole market history. - dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process. + dateBeginingOfMarket: undefined, // Datetime of the first trade file in the whole market history. + dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process. }; let previousDay; // Holds the date of the previous day relative to the processing date. From b6d7b6b220ab70a153bbe449a3f563e21c27f39a Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Mar 2019 09:45:58 +0100 Subject: [PATCH 16/18] Missing parts added --- MultiPeriodDaily.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 01108f4df..7fd39a18f 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -2,12 +2,11 @@ const FULL_LOG = true; const LOG_FILE_CONTENT = false; - const MODULE_NAME = "Multi Period Daily"; - const EXCHANGE_NAME = "Poloniex"; - const commons = COMMONS.newCommons(bot, logger, UTILITIES); + const GMT_SECONDS = ':00.000 GMT+0000'; + const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; thisObject = { initialize: initialize, @@ -243,6 +242,7 @@ try { let n; + currentDate = new Date(contextVariables.lastFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. advanceTime(); From bf8c8fbd2b04263b0752ee87b8aa46568111ef35 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Mar 2019 10:23:59 +0100 Subject: [PATCH 17/18] Loading Dependencies and merging files --- MultiPeriodDaily.js | 56 ++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 7fd39a18f..b3f46f47a 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -93,7 +93,7 @@ }; let previousDay; // Holds the date of the previous day relative to the processing date. - let currentDate; // Holds the processing date. + let currentDay; // Holds the processing date. getContextVariables(); @@ -142,8 +142,9 @@ } contextVariables.dateBeginingOfMarket = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS); + contextVariables.dateBeginingOfMarket = new Date("2018-01-01"); // Remove this temporaty patch. - /* Second, we get the report from Olivia, to know when the marted ends. */ + /* Second, we get the report from Olivia, to know when the martet ends. */ reportKey = "AAMasters" + "-" + "AAOlivia" + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } @@ -242,7 +243,9 @@ try { let n; - currentDate = new Date(contextVariables.lastFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. + let botNeverRan = true; + + currentDay = new Date(contextVariables.lastFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. advanceTime(); @@ -254,15 +257,15 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> Entering function."); } - currentDate = new Date(currentDate.valueOf() + ONE_DAY_IN_MILISECONDS); - previousDay = new Date(currentDate.valueOf() - ONE_DAY_IN_MILISECONDS); + currentDay = new Date(currentDay.valueOf() + ONE_DAY_IN_MILISECONDS); + previousDay = new Date(currentDay.valueOf() - ONE_DAY_IN_MILISECONDS); - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> currentDate = " + currentDate); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> currentDay = " + currentDay); } if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> previousDay = " + previousDay); } /* Validation that we are not going past the head of the market. */ - if (currentDate.valueOf() > contextVariables.dateEndOfMarket.valueOf()) { + if (currentDay.valueOf() > contextVariables.dateEndOfMarket.valueOf()) { const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> " + logText); } @@ -309,9 +312,9 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Entering function."); } - const outputPeriod = global.dailyFilesPeriods[n][0]; - const timePeriod = global.dailyFilesPeriods[n][1]; - + const outputPeriod = global.dailyFilePeriods[n][0]; + const timePeriod = global.dailyFilePeriods[n][1]; + let dependencyIndex = 0; dataFiles = []; @@ -350,6 +353,22 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); } if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); } + if (err.message === "File does not exist." && botNeverRan === true) { + + /* + Sometimes one of the dependencies of an indicator for some reasons are not calculated from the begining of the market. + When that happens we can not find those files. What we do in this situation is to move the time fordward until we can find + all the dependencies and the first run of the bot is successful. + + After that, we will not accept more missing files on any of the dependencies, and if any is missing we will abort the processing. + */ + logger.write(MODULE_NAME, "[WARN] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found."); + + advanceTime(); + return; + } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.message); @@ -406,7 +425,7 @@ currentFile = JSON.parse(text); - let datafile = previousFile.concat(currentFile); + let dataFile = previousFile.concat(currentFile); dataFiles.push(dataFile); dependencyControlLoop(); @@ -460,10 +479,10 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> Entering function."); } - const outputPeriod = global.dailyFilesPeriods[n][0]; - const timePeriod = global.dailyFilesPeriods[n][1]; + const outputPeriod = global.dailyFilePeriods[n][0]; + const timePeriod = global.dailyFilePeriods[n][1]; - usertBot.start(dataFiles, outputPeriod, timePeriod, currentDate, onBotFinished); + usertBot.start(dataFiles, outputPeriod, timePeriod, currentDay, onBotFinished); function onBotFinished(err) { @@ -477,6 +496,7 @@ return; } + botNeverRan = false; periodsControlLoop(); } catch (err) { @@ -505,7 +525,7 @@ n++; - if (n < global.dailyFilesPeriods.length) { + if (n < global.dailyFilePeriods.length) { periodsLoopBody(); @@ -527,7 +547,7 @@ return; } - writeStatusReport(currentDate, advanceTime); + writeStatusReport(currentDay, advanceTime); } catch (err) { logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> controlLoop -> onWritten -> err = " + err.message); @@ -562,7 +582,7 @@ let folderName = bot.products[productIndex].codeName; - writeDataRange(contextVariables.dateBeginingOfMarket, currentDate, folderName, controlLoop); + writeDataRange(contextVariables.dateBeginingOfMarket, currentDay, folderName, controlLoop); } function controlLoop() { @@ -637,7 +657,7 @@ let reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; let thisReport = statusDependencies.statusReports.get(reportKey); - thisReport.file.lastExecution = bot.currentDatetime; + thisReport.file.lastExecution = bot.currentDaytime; thisReport.file.lastFile = lastFileDate; thisReport.save(callBack); From 664b688820d100825127133a9de15ec749400d0c Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Mar 2019 15:47:08 +0100 Subject: [PATCH 18/18] TimePeriod corrected --- IndicatorBotProcessMainLoop.js | 2 +- MultiPeriodDaily.js | 40 +++++++++++++++++++++------------- MultiPeriodMarket.js | 14 ++++++------ StatusDependencies.js | 3 +++ 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 58f34ccce..45e6cf024 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -601,7 +601,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeProcessFramework -> Entering function."); } - processFramework.initialize(statusDependencies, dataDependencies, onInizialized, dataDependencies); + processFramework.initialize(processConfig, statusDependencies, dataDependencies, onInizialized, dataDependencies); function onInizialized(err) { diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index b3f46f47a..c9c8fe1cf 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -23,9 +23,11 @@ let usertBot; let currentBotStorage = BLOB_STORAGE.newBlobStorage(bot, logger); + let pProcessConfig; + return thisObject; - function initialize(pStatusDependencies, pDataDependencies, callBackFunction) { + function initialize(pProcessConfig, pStatusDependencies, pDataDependencies, callBackFunction) { try { @@ -36,6 +38,7 @@ statusDependencies = pStatusDependencies; dataDependencies = pDataDependencies; + processConfig = pProcessConfig; for (let i = 0; i < dataDependencies.config.length; i++) { @@ -107,9 +110,14 @@ let reportKey; let statusReport; - /* We look first for Charly in order to get when the market starts. */ + /* + We look first for the bot who knows the begining of the marke in order to get when the market starts. + */ - reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Poloniex-Historic-Trades" + "-" + "dataSet.V1"; + let botWhoKnowsTheBeginingOfTheMarket = statusDependencies.config[processConfig.statusDependenciesIndexBegingOfMarket]; + let botWhoKnowsTheEndOfTheMarket = statusDependencies.config[processConfig.statusDependenciesIndexEndOfMarket]; + + reportKey = botWhoKnowsTheBeginingOfTheMarket.devTeam + "-" + botWhoKnowsTheBeginingOfTheMarket.bot + "-" + botWhoKnowsTheBeginingOfTheMarket.process + "-" + "dataSet.V1"; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } statusReport = statusDependencies.statusReports.get(reportKey); @@ -142,15 +150,17 @@ } contextVariables.dateBeginingOfMarket = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS); - contextVariables.dateBeginingOfMarket = new Date("2018-01-01"); // Remove this temporaty patch. + //contextVariables.dateBeginingOfMarket = new Date("2018-01-01"); // TODO: Remove this temporaty patch. - /* Second, we get the report from Olivia, to know when the martet ends. */ + /* + Here we get the status report from the bot who knows which is the end of the market. + */ - reportKey = "AAMasters" + "-" + "AAOlivia" + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; + reportKey = botWhoKnowsTheEndOfTheMarket.devTeam + "-" + botWhoKnowsTheEndOfTheMarket.bot + "-" + botWhoKnowsTheEndOfTheMarket.process + "-" + "dataSet.V1"; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } statusReport = statusDependencies.statusReports.get(reportKey); - + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); callBackFunction(global.DEFAULT_RETRY_RESPONSE); @@ -312,8 +322,8 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Entering function."); } - const outputPeriod = global.dailyFilePeriods[n][0]; - const timePeriod = global.dailyFilePeriods[n][1]; + const timePeriod = global.dailyFilePeriods[n][0]; + const outputPeriodLabel = global.dailyFilePeriods[n][1]; let dependencyIndex = 0; dataFiles = []; @@ -341,7 +351,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2); - let filePath = dependency.product + '/' + "Multi-Period-Daily" + "/" + timePeriod + "/" + dateForPath; + let filePath = dependency.product + '/' + "Multi-Period-Daily" + "/" + outputPeriodLabel + "/" + dateForPath; let fileName = market.assetA + '_' + market.assetB + ".json"; storage.getTextFile(filePath, fileName, onFileReceived, true); @@ -401,8 +411,8 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } - let dateForPath = currentDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2); - let filePath = dependency.product + '/' + "Multi-Period-Daily" + "/" + timePeriod + "/" + dateForPath; + let dateForPath = currentDay.getUTCFullYear() + '/' + utilities.pad(currentDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(currentDay.getUTCDate(), 2); + let filePath = dependency.product + '/' + "Multi-Period-Daily" + "/" + outputPeriodLabel + "/" + dateForPath; let fileName = market.assetA + '_' + market.assetB + ".json"; storage.getTextFile(filePath, fileName, onFileReceived, true); @@ -479,10 +489,10 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> Entering function."); } - const outputPeriod = global.dailyFilePeriods[n][0]; - const timePeriod = global.dailyFilePeriods[n][1]; + const timePeriod = global.dailyFilePeriods[n][0]; + const outputPeriodLabel = global.dailyFilePeriods[n][1]; - usertBot.start(dataFiles, outputPeriod, timePeriod, currentDay, onBotFinished); + usertBot.start(dataFiles, timePeriod, outputPeriodLabel, currentDay, onBotFinished); function onBotFinished(err) { diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index 485a087e0..f0e3ffccb 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -25,7 +25,7 @@ return thisObject; - function initialize(pStatusDependencies, pDataDependencies, callBackFunction) { + function initialize(pProcessConfig, pStatusDependencies, pDataDependencies, callBackFunction) { try { @@ -113,8 +113,8 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Entering function."); } - const outputPeriod = global.marketFilesPeriods[n][0]; - const timePeriod = global.marketFilesPeriods[n][1]; + const timePeriod = global.marketFilesPeriods[n][0]; + const outputPeriodLabel = global.marketFilesPeriods[n][1]; let dependencyIndex = 0; dataFiles = []; @@ -140,7 +140,7 @@ let fileName = market.assetA + '_' + market.assetB + ".json"; - let filePath = dependency.product + "/" + "Multi-Period-Market" + "/" + timePeriod; + let filePath = dependency.product + "/" + "Multi-Period-Market" + "/" + outputPeriodLabel; storage.getTextFile(filePath, fileName, onFileReceived, true); @@ -215,10 +215,10 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> Entering function."); } - const outputPeriod = global.marketFilesPeriods[n][0]; - const timePeriod = global.marketFilesPeriods[n][1]; + const timePeriod = global.marketFilesPeriods[n][0]; + const outputPeriodLabel = global.marketFilesPeriods[n][1]; - usertBot.start(dataFiles, outputPeriod, timePeriod, onBotFinished); + usertBot.start(dataFiles, timePeriod, outputPeriodLabel, onBotFinished); function onBotFinished(err) { diff --git a/StatusDependencies.js b/StatusDependencies.js index 635efdf37..2ce421ef8 100644 --- a/StatusDependencies.js +++ b/StatusDependencies.js @@ -6,6 +6,7 @@ const MODULE_NAME = "Status Dependencies"; let thisObject = { + config: undefined, statusReports: new Map(), initialize: initialize, keys: [] @@ -22,6 +23,8 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } + thisObject.config = pStatusDependenciesConfig; + /* For each dependency declared at the bot config, we will initialize the status report, and load it as part of this initialization process.