From df3139e4ce3851568049ccb75a694f1a362d449b Mon Sep 17 00:00:00 2001 From: Matias Benitez Date: Thu, 7 Jun 2018 15:55:18 -0300 Subject: [PATCH] Fixing Market Files --- Multi-Period-Daily/User.bot.js | 14 +- Multi-Period-Market/User.bot.js | 1197 ++++++++++++++++--------------- this.bot.config.json | 58 +- 3 files changed, 675 insertions(+), 594 deletions(-) diff --git a/Multi-Period-Daily/User.bot.js b/Multi-Period-Daily/User.bot.js index c2a04af37..fc72992dd 100644 --- a/Multi-Period-Daily/User.bot.js +++ b/Multi-Period-Daily/User.bot.js @@ -104,7 +104,7 @@ LRCIndicator.prototype.calculate = function (price) { } exports.newUserBot = function newUserBot(BOT, COMMONS, UTILITIES, DEBUG_MODULE, BLOB_STORAGE) { - const FULL_LOG = false; + const FULL_LOG = true; const LOG_FILE_CONTENT = false; const USE_PARTIAL_LAST_CANDLE = true; // When running live the last candle generated is a partial candle. @@ -289,7 +289,7 @@ exports.newUserBot = function newUserBot(BOT, COMMONS, UTILITIES, DEBUG_MODULE, if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> advanceTime -> Entering function."); } - executionTime = new Date(lastCandles[10] + ONE_MIN_IN_MILISECONDS); + executionTime = new Date(lastCandles[2] + (ONE_MIN_IN_MILISECONDS*30)); if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> advanceTime -> New processing time @ " + executionTime.toISOString()); } @@ -324,7 +324,7 @@ exports.newUserBot = function newUserBot(BOT, COMMONS, UTILITIES, DEBUG_MODULE, isTimeToRun(); function isTimeToRun() { - let nextExecution = lastCandles[n] + outputPeriod; + let nextExecution = lastCandles[2] + outputPeriod; if (executionTime.valueOf() >= nextExecution) { getLRCPoints(); } else { @@ -345,7 +345,7 @@ exports.newUserBot = function newUserBot(BOT, COMMONS, UTILITIES, DEBUG_MODULE, let candleArray = []; let queryDate = new Date(executionTime); - let candleFile = getDailyFile(queryDate, onDailyFileReceived); + let candleFile = getDailyFile(queryDate, onDailyFileReceived); //TODO Implement remote file cache function onDailyFileReceived(err, candleFile) { if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> onDailyFileReceived."); } @@ -597,7 +597,7 @@ exports.newUserBot = function newUserBot(BOT, COMMONS, UTILITIES, DEBUG_MODULE, } // We keep a record of the last candle used for the time period - lastCandles[n] = lrcPoint[0]; + lastCandles[2] = lrcPoint[0]; controlLoop(); } @@ -626,7 +626,7 @@ exports.newUserBot = function newUserBot(BOT, COMMONS, UTILITIES, DEBUG_MODULE, } // We keep a record of the last candle used for the time period - lastCandles[n] = lrcPoint[0]; + lastCandles[2] = lrcPoint[0]; controlLoop(); } @@ -650,7 +650,7 @@ exports.newUserBot = function newUserBot(BOT, COMMONS, UTILITIES, DEBUG_MODULE, loopBody(); } else { - let lastOneMinuteCandle = lastCandles[10]; + let lastOneMinuteCandle = lastCandles[2]; writeDataRange(lastOneMinuteCandle, onWritten); function onWritten(err) { diff --git a/Multi-Period-Market/User.bot.js b/Multi-Period-Market/User.bot.js index 3cb5ecb23..88e606912 100644 --- a/Multi-Period-Market/User.bot.js +++ b/Multi-Period-Market/User.bot.js @@ -1,589 +1,4 @@ -exports.newUserBot = function newUserBot(BOT, COMMONS, UTILITIES, DEBUG_MODULE, BLOB_STORAGE) { - const FULL_LOG = true; - const LOG_FILE_CONTENT = false; - const USE_PARTIAL_LAST_CANDLE = true; // When running live the last candle generated is a partial candle. - - let bot = BOT; - - const ONE_MIN_IN_MILISECONDS = 60 * 1000; - - const MODULE_NAME = "User Bot"; - - const EXCHANGE_NAME = "Poloniex"; - - const logger = DEBUG_MODULE.newDebugLog(); - logger.fileName = MODULE_NAME; - logger.bot = bot; - - thisObject = { - initialize: initialize, - start: start - }; - - let gaussStorage = BLOB_STORAGE.newBlobStorage(bot); - - let utilities = UTILITIES.newUtilities(bot); - - let statusDependencies; - - return thisObject; - - function initialize(pStatusDependencies, pMonth, pYear, callBackFunction) { - - try { - - logger.fileName = MODULE_NAME; - - if (FULL_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } - - statusDependencies = pStatusDependencies; - - gaussStorage.initialize({ bot: "AAGauss", devTeam: "AAMasters" }, onGaussInizialized); - - function onGaussInizialized(err) { - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - if (FULL_LOG === true) { logger.write("[INFO] initialize -> onGaussInizialized -> Initialization Succeed."); } - callBackFunction(global.DEFAULT_OK_RESPONSE); - - } else { - logger.write("[ERROR] initialize -> onGaussInizialized -> err = " + err.message); - callBackFunction(err); - } - } - - } catch (err) { - logger.write("[ERROR] initialize -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - /* - TODO Create Description - */ - - function start(callBackFunction) { - - try { - - if (FULL_LOG === true) { logger.write("[INFO] start -> Entering function."); } - - let market = global.MARKET; - let reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process; - let executionTime; - let maxCandleTime; - let lastCandles; - - getContextVariables(); - - function getContextVariables() { - - try { - - if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> Entering function."); } - - let reportKey = "AAMasters" + "-" + "AAOlivia" + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; - if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> reportKey = " + reportKey); } - - if (statusDependencies.statusReports.get(reportKey).status === "Status Report is corrupt.") { - logger.write("[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process."); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - let thisReport = statusDependencies.statusReports.get(reportKey).file; - - if (thisReport.lastFile === undefined) { - logger.write("[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); - - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Dependency not ready." - } - logger.write("[WARN] start -> getContextVariables -> customOK = " + customOK.message); - callBackFunction(customOK); - return; - } - - maxCandleTime = new Date(Date.UTC(thisReport.lastFile.year, thisReport.lastFile.month - 1, thisReport.lastFile.days, thisReport.lastFile.hours, thisReport.lastFile.minutes)); - - reportKey = "AAMasters" + "-" + "AAGauss" + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; - if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> reportKey = " + reportKey); } - - if (statusDependencies.statusReports.get(reportKey).status === "Status Report is corrupt.") { - logger.write("[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process."); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - thisReport = statusDependencies.statusReports.get(reportKey).file; - if (thisReport.lastCandles !== undefined) { - logger.write("[INFO] start -> getContextVariables -> thisReport.lastCandles: " + JSON.stringify(thisReport.lastCandles)); - lastCandles = thisReport.lastCandles; - - buildLRCPoints(); - - } else { - - if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> First time running the bot."); } - - lastCandles = []; - - for (n = 0; n < global.dailyFilePeriods.length; n++) { - lastCandles.push(0); - } - - lastCandles[10] = new Date(Date.UTC(2018, 3).valueOf()); - - if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> Starting at: " + new Date(lastCandles[10]).toISOString()); } - - buildLRCPoints(); - - } - - } catch (err) { - logger.write("[ERROR] start -> getContextVariables -> err = " + err.message); - if (err.message === "Cannot read property 'file' of undefined") { - logger.write("[HINT] start -> getContextVariables -> Check the bot configuration to see if all of its statusDependencies declarations are correct. "); - logger.write("[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys)); - } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function buildLRCPoints() { - - if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> Entering function."); } - - advanceTime(); - - function advanceTime() { - - if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> advanceTime -> Entering function."); } - - executionTime = new Date(lastCandles[10].valueOf() + ONE_MIN_IN_MILISECONDS); - - if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> advanceTime -> New processing time @ " + executionTime.toISOString()); } - - /* Validation that we are not going past the head of the market. */ - - if (executionTime.valueOf() > maxCandleTime.valueOf()) { - - if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> advanceTime -> Head of the market found @ " + maxCandleTime.toISOString()); } - - callBackFunction(global.DEFAULT_OK_RESPONSE); // Here is where we finish processing and wait for the platform to run this module again. - return; - } - - periodsLoop(); - - } - } - - function periodsLoop() { - - let n = 0 // loop Variable representing each possible period as defined at the periods array. - - loopBody(); - - function loopBody() { - - if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCChannels -> periodsLoop -> loopBody -> Entering function. Time: " + executionTime.toISOString() + ". Loop: "+n); } - - const outputPeriod = global.dailyFilePeriods[n][0]; - const folderName = global.dailyFilePeriods[n][1]; - - isTimeToRun(); - - function isTimeToRun() { - let nextExecution = lastCandles[n].valueOf() + outputPeriod; - if (executionTime.valueOf() >= nextExecution) { - getLRCPoints(); - } else { - - if (FULL_LOG === true) logger.write("[INFO] start -> periodsLoop -> loopBody -> It's not time to run this period. Next Execution: " + new Date(nextExecution).toISOString()); - - controlLoop(); - } - } - - function getLRCPoints() { - - if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> Entering function."); } - - const maxLRCDepth = 63; - const maxBackwardsCount = 60; - - let backwardsCount = 0; - let candleArray = []; - - let queryDate = new Date(executionTime); - let candleFile = getDailyFile(queryDate, onDailyFileReceived); - - function onDailyFileReceived(err, candleFile) { - if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> onDailyFileReceived."); } - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - for (let i = 0; i < candleFile.length; i++) { - let candle = { - open: undefined, - close: undefined, - min: 10000000000000, - max: 0, - begin: undefined, - end: undefined - }; - - candle.min = candleFile[i][0]; - candle.max = candleFile[i][1]; - - candle.open = candleFile[i][2]; - candle.close = candleFile[i][3]; - - candle.begin = candleFile[i][4]; - candle.end = candleFile[i][5]; - - if (LOG_FILE_CONTENT === true) { logger.write("[INFO] Candle Date: " + new Date(candle.begin).toISOString() + ". Process Date: " + executionTime.toISOString()); } - - if (candleArray.length < maxLRCDepth && candle.begin <= executionTime.valueOf()) { - candleArray.push(candleFile[i]); - } - } - - if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> Candle Array Length: " + candleArray.length); } - - if (candleArray.length >= maxLRCDepth) { - if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> All candles available proceed with LRC calculations."); } - - let lrcChannel = performLRCCalculations(candleArray); - saveChannel(lrcChannel); - - } else if (backwardsCount <= maxBackwardsCount) { - if (FULL_LOG === true) { logger.write("[INFO] start -> getChannelTilt -> Getting file for day before."); } - - queryDate.setDate(queryDate.getDate() - 1); - getDailyFile(queryDate, onDailyFileReceived); - backwardsCount++; - } else { - logger.write("[ERROR] start -> getChannelTilt -> Not enough history to calculate LRC."); - callBack(global.DEFAULT_RETRY_RESPONSE); - } - } - } - - function getDailyFile(dateTime, onDailyFileReceived) { - try { - if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> getDailyFile -> Entering function."); } - - let datePath = dateTime.getUTCFullYear() + "/" + utilities.pad(dateTime.getUTCMonth() + 1, 2) + "/" + utilities.pad(dateTime.getUTCDate(), 2); - let filePath = "AAMasters/AAOlivia.1.0/AACloud.1.1/Poloniex/dataSet.V1/Output/Candles/Multi-Period-Daily/" + folderName + "/" + datePath; - let fileName = market.assetA + '_' + market.assetB + ".json" - - gaussStorage.getTextFile(filePath, fileName, onFileReceived); - - function onFileReceived(err, text) { - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> getDailyFile -> onFileReceived > Entering Function."); } - - let candleFile = JSON.parse(text); - onDailyFileReceived(global.DEFAULT_OK_RESPONSE, candleFile); - } else { - logger.write("[ERROR] start -> getLRCPoints -> getDailyFile -> onFileReceived -> Failed to get the file. Will abort the process and request a retry."); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - } - } catch (err) { - logger.write("[ERROR] start -> getLRCPoints -> getDailyFile -> err = " + err.message); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - } - } - - function performLRCCalculations(candleArray) { - - if (FULL_LOG === true) { logger.write("[INFO] start -> performLRCCalculations -> Entering function."); } - - /* - * It's needed to order since it's possible that we need to get another file and it will put an older candle at the end of the array. - */ - candleArray.sort(function (a, b) { - return a[4] - b[4]; - }); - - if (USE_PARTIAL_LAST_CANDLE === false) candleArray = candleArray.slice(0, candleArray.length - 1); - - let lrcPoints = []; - lrcPoints.push(0); //firstCandleBeginTime 0 - lrcPoints.push(0); //firstCandleEndTime 1 - lrcPoints.push(0); //lastCandleBeginTime 2 - lrcPoints.push(0); //lastCandleEndTime 3 - lrcPoints.push(0); //minimumChannelValue 4 - lrcPoints.push(0); //middleChannelValue 5 - lrcPoints.push(0); //maximumChannelValue 6 - - if (LOG_FILE_CONTENT === true) - lrcPoints.push(candleArray); //Entire candle array 7 - - calculateLRC(candleArray, lrcPoints); - - let lastCandle = candleArray[0]; - let firstCandle = candleArray[candleArray.length - 1]; - lrcPoints[0] = firstCandle[4]; - lrcPoints[1] = firstCandle[5]; - lrcPoints[2] = lastCandle[4]; - lrcPoints[3] = lastCandle[5]; - - if (FULL_LOG === true) { - logger.write("[INFO] start -> getChannelTilt -> performLRCCalculations -> LRC Points calculation results: " + JSON.stringify(lrcPoints)); } - - return lrcPoints; - } - - function calculateLRC(candlesArray, lrcPoints) { - - if (FULL_LOG === true) { logger.write("[INFO] start -> getChannelTilt -> calculateLRC -> Entering function."); } - - let lrcMinIndicator = new LRCIndicator(15); - let lrcMidIndicator = new LRCIndicator(30); - let lrcMaxIndicator = new LRCIndicator(60); - - for (let i = 0; i < candlesArray.length; i++) { - let tempCandle = candlesArray[i]; - let averagePrice = (tempCandle[0] + tempCandle[1] + tempCandle[2] + tempCandle[3]) / 4; // TODO Check which price should be take to get the LRC - - lrcMinIndicator.update(averagePrice); - lrcMidIndicator.update(averagePrice); - lrcMaxIndicator.update(averagePrice); - } - - /* - * Only if there is enough history the result will be calculated - */ - if (lrcMinIndicator.result != false && lrcMidIndicator.result != false && lrcMaxIndicator.result != false) { - lrcPoints[4] = lrcMinIndicator.result; - lrcPoints[5] = lrcMidIndicator.result; - lrcPoints[6] = lrcMaxIndicator.result; - - return lrcPoints; - } else { - logger.write("[ERROR] start -> getChannelTilt -> calculateLRC -> There is not enough history to calculate the LRC."); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function saveChannel(lrcChannel) { - - try { - - if (FULL_LOG === true) { logger.write("[INFO] start -> saveChannel -> Entering function."); } - let fileContent; - let datePath = executionTime.getUTCFullYear() + "/" + utilities.pad(executionTime.getUTCMonth() + 1, 2) + "/" + utilities.pad(executionTime.getUTCDate(), 2); - let filePath = bot.filePathRoot + "/Output/LRC-Points/Multi-Period-Daily/" + folderName + "/" + datePath; - let fileName = market.assetA + '_' + market.assetB + ".json" - - if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCChannels -> periodsLoop -> loopBody -> getCurrentContent -> fileName = " + fileName); } - if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCChannels -> periodsLoop -> loopBody -> getCurrentContent -> filePath = " + filePath); } - - gaussStorage.getTextFile(filePath, fileName, onFileRetrieved); - - function onFileRetrieved(err, text) { - - if (FULL_LOG === true) { logger.write("[INFO] start -> onFileRetrieved -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - if (err.message === "File does not exist.") { - if (FULL_LOG === true) { logger.write("[INFO] start -> onFileRetrieved -> Daily file does not exist, creating folders."); } - - createFolders(filePath); - return; - } else { - logger.write("[ERROR] start -> onFileRetrieved -> err = " + err.message); - callBackFunction(err); - return; - } - } - - if (FULL_LOG === true) { logger.write("[INFO] start -> onFileRetrieved -> Daily file exist, appending content."); } - - fileContent = JSON.parse(text); - appendToFile(fileContent); - } - - - function createFolders(filePath) { - - try { - - if (FULL_LOG === true) { logger.write("[INFO] start -> createFolders -> Entering function."); } - - utilities.createFolderIfNeeded(filePath, gaussStorage, onFolderACreated); - - function onFolderACreated(err) { - - if (FULL_LOG === true) { logger.write("[INFO] start -> createFolders -> onFolderACreated -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write("[ERROR] start -> createFolders -> onFolderACreated -> err = " + err.message); - callBackFunction(err); - return; - } - - utilities.createFolderIfNeeded(reportFilePath, gaussStorage, onFolderCreated); - } - - function onFolderCreated(err) { - - if (FULL_LOG === true) { logger.write("[INFO] start -> createFolders -> onFolderCreated -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write("[ERROR] start -> createFolders -> onFolderCreated -> err = " + err.message); - callBackFunction(err); - return; - } - createFile(); - } - - } catch (err) { - logger.write("[ERROR] start -> createFolders -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function createFile() { - - if (FULL_LOG === true) { logger.write("[INFO] start -> saveChannel -> createFile -> Entering function."); } - - let existingContent = []; - existingContent.push(lrcChannel); - - gaussStorage.createTextFile(filePath, fileName, JSON.stringify(existingContent), onFileCreated); - - function onFileCreated(err) { - - if (FULL_LOG === true) { logger.write("[INFO] start -> saveChannel -> onFileCreated -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write("[ERROR] start -> saveChannel -> onFileCreated -> err = " + err.message); - callBackFunction(err); - return; - } - - if (LOG_FILE_CONTENT === true) { - logger.write("[INFO] start -> saveChannel -> onFileCreated -> Content written = " + JSON.stringify(existingContent)); - } - - // We keep a record of the last candle used for the time period - lastCandles[n] = lrcChannel[0]; - - controlLoop(); - } - } - - function appendToFile(fileContent) { - - if (FULL_LOG === true) { logger.write("[INFO] start -> saveChannel -> appendToFile -> Entering function."); } - - fileContent.push(lrcChannel); - - gaussStorage.createTextFile(filePath, fileName, JSON.stringify(fileContent), onExistingFileUpdated); - - function onExistingFileUpdated(err) { - - if (FULL_LOG === true) { logger.write("[INFO] start -> saveChannel -> onExistingFileUpdated -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write("[ERROR] start -> saveChannel -> onExistingFileUpdated -> err = " + err.message); - callBackFunction(err); - return; - } - - if (LOG_FILE_CONTENT === true) { - logger.write("[INFO] start -> saveChannel -> onExistingFileUpdated -> Content written = " + JSON.stringify(fileContent)); - } - - // We keep a record of the last candle used for the time period - lastCandles[n] = lrcChannel[0]; - - controlLoop(); - } - } - } catch (err) { - logger.write("[ERROR] start -> saveChannel -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - } - - function controlLoop() { - - if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCChannels -> periodsLoop -> controlLoop -> Entering function."); } - - n++; - - if (n < global.dailyFilePeriods.length) { - - loopBody(); - - } else { - - writeStatusReport(); - - } - } - - } - - function writeStatusReport() { - - try { - - if (FULL_LOG === true) { logger.write("[INFO] start -> writeStatusReport -> Entering function."); } - - let key = bot.devTeam + "-" + bot.codeName + "-" + bot.process + "-" + bot.dataSetVersion; - - let statusReport = statusDependencies.statusReports.get(key); - - statusReport.file.lastExecution = bot.processDatetime; - statusReport.file.lastCandles = lastCandles; - statusReport.save(onSaved); - - function onSaved(err) { - - if (FULL_LOG === true) { logger.write("[INFO] start -> writeStatusReport -> onSaved -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write("[ERROR] start -> writeStatusReport -> onSaved -> err = " + err.message); - callBackFunction(err); - return; - } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - } - - } catch (err) { - logger.write("[ERROR] start -> writeStatusReport -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - - } catch (err) { - logger.write("[ERROR] start -> err = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } -}; - -/* - * Linear regression curve. SRC: Gekko - */ - -//var log = require('../../core/log'); - -var LRCIndicator = function (settings) { +var LRCIndicator = function (settings) { this.input = 'price'; this.depth = settings; this.result = false; @@ -687,3 +102,613 @@ LRCIndicator.prototype.calculate = function (price) { // y = a * x + b this.result = ((this.depth - 1) * reg[0]) + reg[1]; } + +exports.newUserBot = function newUserBot(BOT, COMMONS, UTILITIES, DEBUG_MODULE, BLOB_STORAGE) { + const FULL_LOG = true; + const LOG_FILE_CONTENT = false; + const USE_PARTIAL_LAST_CANDLE = true; // When running live the last candle generated is a partial candle. + + let bot = BOT; + + const ONE_MIN_IN_MILISECONDS = 60 * 1000; + + const MODULE_NAME = "User Bot"; + + const EXCHANGE_NAME = "Poloniex"; + + const logger = DEBUG_MODULE.newDebugLog(); + logger.fileName = MODULE_NAME; + logger.bot = bot; + + thisObject = { + initialize: initialize, + start: start + }; + + let gaussStorage = BLOB_STORAGE.newBlobStorage(bot); + let oliviaStorage = BLOB_STORAGE.newBlobStorage(bot); + + let utilities = UTILITIES.newCloudUtilities(bot); + + let statusDependencies; + + return thisObject; + + function initialize(pStatusDependencies, pMonth, pYear, callBackFunction) { + + try { + + logger.fileName = MODULE_NAME; + + if (FULL_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + + statusDependencies = pStatusDependencies; + + gaussStorage.initialize(bot.devTeam, onGaussInizialized); + + function onGaussInizialized(err) { + + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + + if (FULL_LOG === true) { logger.write("[INFO] initialize -> onGaussInizialized -> Initialization Succeed."); } + oliviaStorage.initialize("AAMasters", onOliviaInizialized); + + } else { + logger.write("[ERROR] initialize -> onGaussInizialized -> err = " + err.message); + callBackFunction(err); + } + } + + function onOliviaInizialized(err) { + + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + + if (FULL_LOG === true) { logger.write("[INFO] initialize -> onOliviaInizialized -> Initialization Succeed."); } + callBackFunction(global.DEFAULT_OK_RESPONSE); + + } else { + logger.write("[ERROR] initialize -> onOliviaInizialized -> err = " + err.message); + callBackFunction(err); + } + } + + } catch (err) { + logger.write("[ERROR] initialize -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + /* + This process will do the following: + For every period on the market: + a. Get olivia files for the current execution time + i. If the file exist, validate we can get 63 candles + 1) 63 candles exist, proceed + 2) 63 candles doesn't exist, call for previous execution day until have 63 + ii. File doesn't exist, wait until get the files + b. Once we have the 63 candles, calculate LRC + c. Save LRC points + */ + + function start(callBackFunction) { + + try { + + if (FULL_LOG === true) { logger.write("[INFO] start -> Entering function."); } + + let market = global.MARKET; + let reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process; + let executionTime; + let maxCandleTime; + let lastCandles; + let marketFileCache = {}; + + getContextVariables(); + + function getContextVariables() { + + try { + + if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> Entering function."); } + + let reportKey = "AAMasters" + "-" + "AAOlivia" + "-" + "Multi-Period-Market" + "-" + "dataSet.V1"; + if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> reportKey = " + reportKey); } + + if (statusDependencies.statusReports.get(reportKey).status === "Status Report is corrupt.") { + logger.write("[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + + let thisReport = statusDependencies.statusReports.get(reportKey).file; + + if (thisReport.lastFile === undefined) { + logger.write("[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); + + let customOK = { + result: global.CUSTOM_OK_RESPONSE.result, + message: "Dependency not ready." + } + logger.write("[WARN] start -> getContextVariables -> customOK = " + customOK.message); + callBackFunction(customOK); + return; + } + + maxCandleTime = new Date(thisReport.lastFile); + + reportKey = "AAVikings" + "-" + "AAGauss" + "-" + "Multi-Period-Market" + "-" + "dataSet.V1"; + if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> reportKey = " + reportKey); } + + if (statusDependencies.statusReports.get(reportKey).status === "Status Report is corrupt.") { + logger.write("[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + + thisReport = statusDependencies.statusReports.get(reportKey).file; + if (thisReport.lastCandles !== undefined) { + if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> thisReport.lastCandles: " + JSON.stringify(thisReport.lastCandles)) }; + + lastCandles = thisReport.lastCandles; + buildLRCPoints(); + + } else { + + if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> First time running the bot."); } + + lastCandles = []; + + for (n = 0; n < global.marketFilesPeriods.length; n++) { + lastCandles.push(0); + } + + lastCandles[7] = new Date(Date.UTC(2018, 3)).valueOf(); + + if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> Starting at: " + new Date(lastCandles[7]).toISOString()); } + + buildLRCPoints(); + + } + + } catch (err) { + logger.write("[ERROR] start -> getContextVariables -> err = " + err.message); + if (err.message === "Cannot read property 'file' of undefined") { + logger.write("[HINT] start -> getContextVariables -> Check the bot configuration to see if all of its statusDependencies declarations are correct. "); + logger.write("[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys)); + } + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function buildLRCPoints() { + + if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> Entering function."); } + + advanceTime(); + + function advanceTime() { + + if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> advanceTime -> Entering function."); } + + executionTime = new Date(lastCandles[7] + global.marketFilesPeriods[7][0]); + + if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> advanceTime -> New processing time @ " + executionTime.toISOString()); } + + /* Validation that we are not going past the head of the market. */ + + if (executionTime.valueOf() > maxCandleTime.valueOf()) { + + if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> advanceTime -> Head of the market found @ " + maxCandleTime.toISOString()); } + + callBackFunction(global.DEFAULT_OK_RESPONSE); // Here is where we finish processing and wait for the platform to run this module again. + return; + } + + periodsLoop(); + + } + } + + function periodsLoop() { + + let n = 0 // loop Variable representing each possible period as defined at the periods array. + + loopBody(); + + function loopBody() { + + if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> periodsLoop -> loopBody -> Entering function. Time: " + executionTime.toISOString() + ". Loop: "+n); } + + const outputPeriod = global.marketFilesPeriods[n][0]; + const folderName = global.marketFilesPeriods[n][1]; + + isTimeToRun(); + + function isTimeToRun() { + let nextExecution = lastCandles[n] + outputPeriod; + if (executionTime.valueOf() >= nextExecution) { + getLRCPoints(); + } else { + if (FULL_LOG === true) logger.write("[INFO] start -> periodsLoop -> loopBody -> It's not time to run this period. Next Execution: " + new Date(nextExecution).toISOString()); + + controlLoop(); + } + } + + function getLRCPoints() { + + if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> Entering function."); } + + const maxLRCDepth = 63; + const maxBackwardsCount = 60; + + let backwardsCount = 0; + let candleArray = []; + + let candleFile = getMarketFile(onMarketFileReceived); //TODO Implement remote file cache + + function onMarketFileReceived(err, candleFile) { + if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> onMarketFileReceived."); } + + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + for (let i = 0; i < candleFile.length; i++) { + let candle = { + open: undefined, + close: undefined, + min: 10000000000000, + max: 0, + begin: undefined, + end: undefined + }; + + candle.min = candleFile[i][0]; + candle.max = candleFile[i][1]; + + candle.open = candleFile[i][2]; + candle.close = candleFile[i][3]; + + candle.begin = candleFile[i][4]; + candle.end = candleFile[i][5]; + + if (LOG_FILE_CONTENT === true) { logger.write("[INFO] Candle Date: " + new Date(candle.begin).toISOString() + ". Process Date: " + executionTime.toISOString()); } + + let lastCandle = executionTime.valueOf() - (outputPeriod * maxLRCDepth); + + if (candleArray.length < maxLRCDepth && candle.begin > lastCandle && candle.begin <= executionTime.valueOf()) { + candleArray.push(candle); + } + } + + if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> Candle Array Length: " + candleArray.length); } + + if (candleArray.length >= maxLRCDepth) { + if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> All candles available proceed with LRC calculations."); } + + let lrcPoint = performLRCCalculations(candleArray); + savePoint(lrcPoint); + + }else { + logger.write("[ERROR] start -> getLRCPoints -> Not enough history to calculate LRC, advancing time."); + + // We keep a record of the last candle used for the time period + lastCandles[n] = lrcPoint[0]; + + controlLoop(); + } + } + } + + function getMarketFile(onMarketFileReceived) { + try { + if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> getMarketFile -> Entering function."); } + + /** + * Let's first check if we can use our own process cache in order to reduce network requests. + * Otherwise, continue with the one from the servers. + * + */ + let localDate = new Date(); + + let now = new Date(Date.UTC( + localDate.getUTCFullYear(), + localDate.getUTCMonth(), + localDate.getUTCDate(), + localDate.getUTCHours(), + localDate.getUTCMinutes(), + localDate.getUTCSeconds(), + localDate.getUTCMilliseconds())); + + if (marketFileCache[n] !== undefined && executionTime < (now.valueOf() - outputPeriod)){ + onMarketFileReceived(global.DEFAULT_OK_RESPONSE, marketFileCache[n]); + return; + } + + let filePath = "AAMasters/AAOlivia.1.0/AACloud.1.1/Poloniex/dataSet.V1/Output/Candles/Multi-Period-Market/" + folderName; + let fileName = market.assetA + '_' + market.assetB + ".json" + + oliviaStorage.getTextFile(filePath, fileName, onFileReceived); + + function onFileReceived(err, text) { + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> getMarketFile -> onFileReceived > Entering Function."); } + + let candleFile = JSON.parse(text); + marketFileCache[n] = candleFile; // We keep a copy of the results for local use + onMarketFileReceived(global.DEFAULT_OK_RESPONSE, candleFile); + } else { + logger.write("[ERROR] start -> getLRCPoints -> getMarketFile -> onFileReceived -> Failed to get the file. Will abort the process and request a retry."); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + } + } catch (err) { + logger.write("[ERROR] start -> getLRCPoints -> getMarketFile -> err = " + err.message); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + } + } + + function performLRCCalculations(candleArray) { + + if (FULL_LOG === true) { logger.write("[INFO] start -> performLRCCalculations -> Entering function."); } + + /* + * It's needed to order since it's possible that we need to get another file and it will put an older candle at the end of the array. + */ + candleArray.sort(function (a, b) { + return a.begin - b.begin; + }); + + if (USE_PARTIAL_LAST_CANDLE === false) candleArray = candleArray.slice(0, candleArray.length - 1); + + let lrcPoints = []; + lrcPoints.push(0); //firstCandleBeginTime 0 + lrcPoints.push(0); //firstCandleEndTime 1 + lrcPoints.push(0); //minimumPointValue 2 + lrcPoints.push(0); //middlePointValue 3 + lrcPoints.push(0); //maximumPointValue 4 + + if (LOG_FILE_CONTENT === true) + lrcPoints.push(candleArray); //Entire candle array 5 + + calculateLRC(candleArray, lrcPoints); + + let lastCandle = candleArray[0]; + let firstCandle = candleArray[candleArray.length - 1]; + lrcPoints[0] = firstCandle.begin; + lrcPoints[1] = firstCandle.end; + + if (FULL_LOG === true) { + logger.write("[INFO] start -> getLRCPoints -> performLRCCalculations -> LRC Points calculation results: " + JSON.stringify(lrcPoints)); } + + return lrcPoints; + } + + function calculateLRC(candlesArray, lrcPoints) { + + if (FULL_LOG === true) { logger.write("[INFO] start -> getLRCPoints -> calculateLRC -> Entering function."); } + + let lrcMinIndicator = new LRCIndicator(15); + let lrcMidIndicator = new LRCIndicator(30); + let lrcMaxIndicator = new LRCIndicator(60); + + for (let i = 0; i < candlesArray.length; i++) { + let tempCandle = candlesArray[i]; + let averagePrice = (tempCandle.min + tempCandle.max + tempCandle.open + tempCandle.close) / 4; // TODO Check which price should be take to get the LRC + + lrcMinIndicator.update(averagePrice); + lrcMidIndicator.update(averagePrice); + lrcMaxIndicator.update(averagePrice); + } + + /* + * Only if there is enough history the result will be calculated + */ + if (lrcMinIndicator.result != false && lrcMidIndicator.result != false && lrcMaxIndicator.result != false) { + lrcPoints[2] = lrcMinIndicator.result; + lrcPoints[3] = lrcMidIndicator.result; + lrcPoints[4] = lrcMaxIndicator.result; + + return lrcPoints; + } else { + logger.write("[ERROR] start -> getLRCPoints -> calculateLRC -> There is not enough history to calculate the LRC."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function savePoint(lrcPoint) { + + try { + + if (FULL_LOG === true) { logger.write("[INFO] start -> savePoint -> Entering function."); } + + let filePath = bot.filePathRoot + "/Output/LRC-Points/Multi-Period-Market/" + folderName; + let fileName = market.assetA + '_' + market.assetB + ".json" + + if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> periodsLoop -> loopBody -> getCurrentContent -> fileName = " + fileName); } + if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> periodsLoop -> loopBody -> getCurrentContent -> filePath = " + filePath); } + + gaussStorage.getTextFile(filePath, fileName, onFileRetrieved); + + function onFileRetrieved(err, text) { + + if (FULL_LOG === true) { logger.write("[INFO] start -> onFileRetrieved -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + if (err.message === "File does not exist.") { + if (FULL_LOG === true) { logger.write("[INFO] start -> onFileRetrieved -> Market file does not exist, creating folders."); } + + createFolders(filePath); + return; + } else { + logger.write("[ERROR] start -> onFileRetrieved -> err = " + err.message); + callBackFunction(err); + return; + } + } + + if (FULL_LOG === true) { logger.write("[INFO] start -> onFileRetrieved -> Market file exist, appending content."); } + + let fileContent = JSON.parse(text); + appendToFile(fileContent); + } + + function createFolders(filePath) { + + try { + + if (FULL_LOG === true) { logger.write("[INFO] start -> createFolders -> Entering function."); } + + utilities.createFolderIfNeeded(filePath, gaussStorage, onFolderACreated); + + function onFolderACreated(err) { + + if (FULL_LOG === true) { logger.write("[INFO] start -> createFolders -> onFolderACreated -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write("[ERROR] start -> createFolders -> onFolderACreated -> err = " + err.message); + callBackFunction(err); + return; + } + + utilities.createFolderIfNeeded(reportFilePath, gaussStorage, onFolderCreated); + } + + function onFolderCreated(err) { + + if (FULL_LOG === true) { logger.write("[INFO] start -> createFolders -> onFolderCreated -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write("[ERROR] start -> createFolders -> onFolderCreated -> err = " + err.message); + callBackFunction(err); + return; + } + createFile(); + } + + } catch (err) { + logger.write("[ERROR] start -> createFolders -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function createFile() { + + if (FULL_LOG === true) { logger.write("[INFO] start -> savePoint -> createFile -> Entering function."); } + + let existingContent = []; + existingContent.push(lrcPoint); + + gaussStorage.createTextFile(filePath, fileName, JSON.stringify(existingContent), onFileCreated); + + function onFileCreated(err) { + + if (FULL_LOG === true) { logger.write("[INFO] start -> savePoint -> onFileCreated -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write("[ERROR] start -> savePoint -> onFileCreated -> err = " + err.message); + callBackFunction(err); + return; + } + + if (LOG_FILE_CONTENT === true) { + logger.write("[INFO] start -> savePoint -> onFileCreated -> Content written = " + JSON.stringify(existingContent)); + } + + // We keep a record of the last candle used for the time period + lastCandles[n] = lrcPoint[0]; + + controlLoop(); + } + } + + function appendToFile(fileContent) { + + if (FULL_LOG === true) { logger.write("[INFO] start -> savePoint -> appendToFile -> Entering function."); } + + fileContent.push(lrcPoint); + + gaussStorage.createTextFile(filePath, fileName, JSON.stringify(fileContent), onExistingFileUpdated); + + function onExistingFileUpdated(err) { + + if (FULL_LOG === true) { logger.write("[INFO] start -> savePoint -> onExistingFileUpdated -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write("[ERROR] start -> savePoint -> onExistingFileUpdated -> err = " + err.message); + callBackFunction(err); + return; + } + + if (LOG_FILE_CONTENT === true) { + logger.write("[INFO] start -> savePoint -> onExistingFileUpdated -> Content written = " + JSON.stringify(fileContent)); + } + + // We keep a record of the last candle used for the time period + lastCandles[n] = lrcPoint[0]; + + controlLoop(); + } + } + } catch (err) { + logger.write("[ERROR] start -> savePoint -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + } + + function controlLoop() { + + if (FULL_LOG === true) { logger.write("[INFO] start -> buildLRCPoints -> periodsLoop -> controlLoop -> Entering function."); } + + n++; + + if (n < global.marketFilesPeriods.length) { + loopBody(); + } else { + writeStatusReport(); + } + } + } + + function writeStatusReport() { + + try { + + if (FULL_LOG === true) { logger.write("[INFO] start -> writeStatusReport -> Entering function."); } + + let key = bot.devTeam + "-" + bot.codeName + "-" + bot.process + "-" + bot.dataSetVersion; + + let statusReport = statusDependencies.statusReports.get(key); + + statusReport.file.lastExecution = bot.processDatetime; + statusReport.file.lastCandles = lastCandles; + statusReport.save(onSaved); + + function onSaved(err) { + + if (FULL_LOG === true) { logger.write("[INFO] start -> writeStatusReport -> onSaved -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write("[ERROR] start -> writeStatusReport -> onSaved -> err = " + err.message); + callBackFunction(err); + return; + } + + callBackFunction(global.DEFAULT_OK_RESPONSE); + } + + } catch (err) { + logger.write("[ERROR] start -> writeStatusReport -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + logger.write("[ERROR] start -> err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + +}; diff --git a/this.bot.config.json b/this.bot.config.json index f53d75989..6994d3c35 100644 --- a/this.bot.config.json +++ b/this.bot.config.json @@ -10,6 +10,55 @@ "devTeam": "AAVikings", "dataSetVersion": "dataSet.V1", "processes": [ + { + "name": "Multi-Period-Market", + "description": "Produces files with the LRC Points for the entire market.", + "startMode": { + "allMonths": { + "run": "false", + "minYear": "", + "maxYear": "" + }, + "oneMonth": { + "run": "false", + "year": "", + "month": "" + }, + "noTime": { + "run": "true" + }, + "fixedInterval": { + "run": "false", + "interval": 0 + } + }, + "normalWaitTime": 50, + "retryWaitTime": 10000, + "sleepWaitTime": 3600000, + "comaWaitTime": 86400000, + "statusDependencies": [ + { + "devTeam": "AAVikings", + "bot": "AAGauss", + "botVersion": { + "major": 1, + "minor": 0 + }, + "process": "Multi-Period-Market", + "dataSetVersion": "dataSet.V1" + }, + { + "devTeam": "AAMasters", + "bot": "AAOlivia", + "botVersion": { + "major": 1, + "minor": 0 + }, + "process": "Multi-Period-Market", + "dataSetVersion": "dataSet.V1" + } + ] + }, { "name": "Multi-Period-Daily", "description": "Produces files with the LRC Points on a daily format.", @@ -67,10 +116,17 @@ "description": "Two sets of files per period that contains LRC points.", "storageAccount": "aagauss", "dataSets": [ + { + "codeName": "Multi-Period-Market", + "type": "Market Files", + "validPeriods": [ "24-hs", "12-hs", "08-hs", "06-hs", "04-hs", "03-hs", "02-hs", "01-hs" ], + "filePath": "AAVikings/AAGauss.1.0/AACloud.1.1/@Exchange/dataSet.V1/Output/LRC-Points/Multi-Period-Market/@Period", + "fileName": "@AssetA_@AssetB.json" + }, { "codeName": "Multi-Period-Daily", "type": "Daily Files", - "validPeriods": [ "45-min", "40-min", "30-min", "20-min", "15-min", "10-min", "05-min", "04-min", "03-min", "02-min", "01-min" ], + "validPeriods": [ "30-min" ], "filePath": "AAVikings/AAGauss.1.0/AACloud.1.1/@Exchange/dataSet.V1/Output/LRC-Points/Multi-Period-Daily/@Period/@Year/@Month/@Day", "fileName": "@AssetA_@AssetB.json", "dataRange": {