From 04fa80cc412fc49e257ec0d38d99dabed0f74cdf Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 3 Feb 2020 17:58:56 +0100 Subject: [PATCH] Concept of Multiple Instances of a Process Removed --- Historic-Trades/User.Bot.js | 11 +- Hole-Fixing/User.Bot.js | 1447 ----------------------------------- Live-Trades/User.Bot.js | 377 --------- 3 files changed, 5 insertions(+), 1830 deletions(-) delete mode 100644 Hole-Fixing/User.Bot.js delete mode 100644 Live-Trades/User.Bot.js diff --git a/Historic-Trades/User.Bot.js b/Historic-Trades/User.Bot.js index a8ba2f333..1557d27cf 100644 --- a/Historic-Trades/User.Bot.js +++ b/Historic-Trades/User.Bot.js @@ -30,7 +30,7 @@ exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, fileSt return thisObject; - function initialize(pStatusDependencies, pMonth, pYear, callBackFunction) { + function initialize(pStatusDependencies, callBackFunction) { try { logger.fileName = MODULE_NAME; @@ -97,7 +97,7 @@ exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, fileSt verbose: false, options: options } - console.log(JSON.stringify(exchangeConstructorParams)) + const exchange = new exchangeClass(exchangeConstructorParams) begin() @@ -199,7 +199,7 @@ exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, fileSt let processingDate = new Date(since) processingDate = processingDate.getUTCFullYear() + '-' + utilities.pad(processingDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(processingDate.getUTCDate(), 2); if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getTrades -> Fetching Trades @ " + processingDate + "-> exchange = " + bot.exchange + " -> symbol = " + symbol + " -> since = " + since + " -> limit = " + limit ) } - console.log("Charly -> " + MODULE_NAME + " -> start -> getTrades -> Fetching Trades from " + bot.exchange + " " + symbol + " @ " + processingDate + " collecting " + allTrades.length + " records trades so far.") + //console.log("Charly -> " + MODULE_NAME + " -> start -> getTrades -> Fetching Trades from " + bot.exchange + " " + symbol + " @ " + processingDate + " collecting " + allTrades.length + " records trades so far.") bot.processHeartBeat("Fetching " + allTrades.length.toFixed(0) + " trades from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well if (fetchType === "by Id") { @@ -295,7 +295,7 @@ exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, fileSt let processingDate = new Date(trade.timestamp) processingDate = processingDate.getUTCFullYear() + '-' + utilities.pad(processingDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(processingDate.getUTCDate(), 2); if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> saveTrades -> Saving Trades @ " + processingDate + " -> i = " + i + " -> total = " + allTrades.length) } - console.log("Charly -> " + MODULE_NAME + " -> start -> saveTrades -> Saving Trades from " + bot.exchange + " " + symbol + " @ " + processingDate) + //console.log("Charly -> " + MODULE_NAME + " -> start -> saveTrades -> Saving Trades from " + bot.exchange + " " + symbol + " @ " + processingDate) bot.processHeartBeat("Saving " + i.toFixed(0) + " trades from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well } @@ -425,8 +425,7 @@ exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, fileSt days: beginingOfMarket.getUTCDate(), hours: beginingOfMarket.getUTCHours(), minutes: beginingOfMarket.getUTCMinutes() - }, - completeHistory: true + } }; if (fetchType === "by Id") { diff --git a/Hole-Fixing/User.Bot.js b/Hole-Fixing/User.Bot.js deleted file mode 100644 index 60ff4709a..000000000 --- a/Hole-Fixing/User.Bot.js +++ /dev/null @@ -1,1447 +0,0 @@ -exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, fileStorage, STATUS_REPORT, EXCHANGE_API) { - - const FULL_LOG = true; - const LOG_FILE_CONTENT = false; - const GMT_SECONDS = ':00.000 GMT+0000'; - const GMT_MILI_SECONDS = '.000 GMT+0000'; - const MODULE_NAME = "User Bot Hole Fixing"; - const TRADES_FOLDER_NAME = "Trades"; - - thisObject = { - initialize: initialize, - start: start - }; - - let utilities = UTILITIES.newCloudUtilities(bot, logger); - - let year; - let month; - - let statusDependencies; - - return thisObject; - - function initialize(pStatusDependencies, pMonth, pYear, callBackFunction) { - try { - - year = pYear; - month = pMonth; - month = utilities.pad(month, 2); // Adding a left zero when needed. - statusDependencies = pStatusDependencies; - - logger.fileName = MODULE_NAME + "-" + year + "-" + month; - logger.initialize(); - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> pYear = " + year); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> pMonth = " + month); } - - /* The very first validation is about if we are not too far in the future. In those cases we will not proceed and expect this instance to be restarted later. */ - - let processDate = new Date(year + "-" + month + "-1 00:00:00.000 GMT+0000"); - let today = new Date(); - let tomorrow = new Date(today.valueOf() + 1000 * 60 * 60 * 24); - - if (processDate.valueOf() > tomorrow.valueOf()) { // This means that it should start more than a day from current time. - logger.write(MODULE_NAME, "[WARN] initialize -> Too far in the future."); - - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Too far in the future." - } - logger.write(MODULE_NAME, "[WARN] initialize -> customOK = " + customOK.message); - callBackFunction(customOK); - return; - } - - if (processDate.valueOf() > today.valueOf()) { // This means that is should start in less than a day from current time. - logger.write(MODULE_NAME, "[WARN] initialize -> Too far in the future."); - - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Not needed now, but soon." - } - logger.write(MODULE_NAME, "[WARN] initialize -> customOK = " + customOK.message); - callBackFunction(customOK); - return; - } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - /* - - This process is going to do the following: - - It will try to find "holes" on the trades history and fix them by retrieving the missing trades from the exchange. - - What are holes? - - Holes are missing trades in the history of the market and the reason there can be holes are the following: - - 1) The "Lives Trades" process can be stopped and later started again. The process will only retrieve live trades, leaving a "hole" between it was stopped and restarted. - 2) The "Historic Trades" process might have received a "hole" of transactions directly from the exchange. - 3) The "Historic Trades" process failed to write some files and now they are missing, together with the trades in them. - 4) The "Historic Trades" process wrote some files to the storage with corrupted content. - 5) The "Historic Trades" process wrote files with trades not properly ordered by Trade Id (sometimes the exchange sent this wrong). - - How does the process work? - - Transactions have a unique id generated by the exchange. It is a secuencial number. This process scans the transaction history looking for missing ids. When it finds - some group of missing ids, it request the exchange the missing records and it writes the necesary files to fix the hole. But some things might happen: - - 1) The exchange responds, but does not provides the missing records. In this case the process will flag this hole for retry later, up to 3 times. After that it will flag - the hole as "permanent". - 2) The exchange call timeouts: the process will continue retrying later. - 3) The exchange returns an unexpected error: The process will retry up to 3 times later under this condition. - 4) The exchange returns neighboring transactions but not the missing ones. The process will retry 3 more times and mark the hole as permanent if the exchange refuses to send the missing trades. - - Since the process is run in an infinite loop but can be restarted any time, the status of its running is recorded in the Status Report file. - - What is the lastFile pointer? - - It is the Datetime since the begining of the hitory of a market that is considered without holes (or only with permanent ones, unfixables). In other words, the process - starts from the begining of the history of a market, and only when its history is signaled as "Complete" by the "Historic Trades" process. From there, the process is advancing - in time, validating the history and fixing holes, and moving the lastFile pointer forward as it gets sure that all the previous trades files are with no holes or unfixable ones. - This pointer will later be used by Indicators bots that depends of these trades, and it will be considered the head of the market. - - Manually modifying the Status Report: - - If for any reason you need to modify manually the status report consider this: - - lastFile: put here the date of the previous file to the one that contains the last verified trade. - lastTrade: point here to a trade that you want to be considered as the last trade verfied by the process. - - */ - - function start(callBackFunction) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); } - - let processDate = new Date(year + "-" + month + "-1 00:00:00.000 GMT+0000"); - let lastMinuteOfMonth = new Date(year + "-" + month + "-1 00:00:00.000 GMT+0000"); - - lastMinuteOfMonth.setUTCMonth(lastMinuteOfMonth.getUTCMonth() + 1); // First we go 1 month into the future. - lastMinuteOfMonth.setUTCSeconds(lastMinuteOfMonth.getUTCSeconds() - 30); // Then we go back 30 seconds, or to the last minute of the original month. - - let nextIntervalExecution = false; // This tell weather the Interval module will be executed again or not. By default it will not unless some hole have been found in the current execution. - - let currentDate; // This will hold the current datetime of each execution. - - let market = bot.market; - - let dateForPath; - let filePath; - let exchangeCallTime; - - const MAX_EXCHANGE_CALL_RETRIES = 3; - const MAX_HOLE_FIXING_RETRIES = 3; - const FIRST_TRADE_RECORD_ID = -1; - const UNKNOWN_TRADE_RECORD_ID = -2; - const MAX_EXCHANGE_CALLS_FOR_GETTING_TRADES = 10 - - let exchangeCallRetries = 0; - - let tradesAtFileWhereHoleBegins = []; // File content of the file where a hole starts. - let tradesAtFileWhereHoleEnds = []; // File content of the file where we realized we had a hole. - - let lastGoodTradeId; // This points to the last Trade Id that is ok. - let lastGoodTradeDatetime; // This points to the last Trade datetime that is ok. - - /* The next 3 variables hold the information read from varios Status Reports. */ - - let datetimeLastFileLiveTrade; // Datetime of the last complete trades file written by the Live Trades process. - let datetimeLastFileHistoricTrade; // Datatime of the last trades file written by the Historic Trades process. - let datetimeLastFileWithoutHoles; // Datetime of the last file certified by the Hole Fixing process as without permanent holes. - - /* The next 4 variables hold the results of the search of the next hole. */ - - let holeInitialId; // This is the Id just before the hole. - let holeInitialDatetime; // This is the Datetime just before the hole. - - let holeFinalId; // This is the Id just after the hole. - let holeFinalDatetime; // This is the Datetime just after the hole. - - let holeFixingStatusReport; // Current hole Fixing Status Report. - - let exchangeCalls = 0 // Keep track of how many times we call the exchange - let consolidatedResponse = [] // Here we will accumulate all the trades received from the exchange during different calls. - let lastReceivedTradeId // Used to stop the recursive call to the exchange. - - getContextVariables(); - - function getContextVariables() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Entering function."); } - - let thisReport; - let reportKey; - - reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Live-Trades" + "-" + "dataSet.V1"; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } - - if (statusDependencies.statusReports.get(reportKey).status === "Status Report is corrupt.") { - logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - thisReport = statusDependencies.statusReports.get(reportKey).file; - - if (thisReport.lastFile === undefined) { - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); - - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Dependency does not exist." - } - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); - callBackFunction(customOK); - return; - } - - datetimeLastFileLiveTrade = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS); - - reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Historic-Trades" + "-" + "dataSet.V1"; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } - - if (statusDependencies.statusReports.get(reportKey).status === "Status Report is corrupt.") { - logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - thisReport = statusDependencies.statusReports.get(reportKey).file; - - if (thisReport.lastFile === undefined) { - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); - logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet."); - - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Dependency does not exist." - } - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); - callBackFunction(customOK); - return; - } - - if (thisReport.completeHistory === true) { // We get from the file to know if this markets history is complete or not. - - datetimeLastFileHistoricTrade = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS); - - /* Before processing this month we need to check if it is not too far in the past.*/ - - if ( - processDate.getUTCFullYear() < datetimeLastFileHistoricTrade.getUTCFullYear() - || - (processDate.getUTCFullYear() === datetimeLastFileHistoricTrade.getUTCFullYear() && processDate.getUTCMonth() < datetimeLastFileHistoricTrade.getUTCMonth()) - ) { - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> The current year / month is before the start of the market history for market."); - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Month before it is needed." - } - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); - callBackFunction(customOK); - return; - } - - } else { - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Trade History is not complete."); - - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Dependency not ready." - } - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); - callBackFunction(customOK); - return; - } - - reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Hole-Fixing" + "-" + "dataSet.V1" + "-" + year + "-" + month; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } - - if (statusDependencies.statusReports.get(reportKey).status === "Status Report is corrupt.") { - logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process."); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - holeFixingStatusReport = statusDependencies.statusReports.get(reportKey).file; - - if (holeFixingStatusReport.lastFile === undefined) { - - /* - - The file does not exist, so this means this is the first time we are running this process. - Then, we might me under two different possible situations: - - 1. We are at the first month of the market. In this case, the starting date is not the begining of the month but the first date of - the history of the market. - - 2. We are at some other month different than the first one. In this case, the starting date is the first minute of the market. - - */ - - if (processDate.valueOf() < datetimeLastFileHistoricTrade.valueOf()) { - - datetimeLastFileWithoutHoles = new Date(datetimeLastFileHistoricTrade.valueOf() - 60 * 1000); // One minute less that the begining of market history. - - lastGoodTradeId = FIRST_TRADE_RECORD_ID; - lastGoodTradeDatetime = new Date(datetimeLastFileHistoricTrade.valueOf()); - - } else { - - datetimeLastFileWithoutHoles = new Date(processDate.valueOf() - 60 * 1000); // One minute less that the begining of the month. - - lastGoodTradeId = UNKNOWN_TRADE_RECORD_ID; - lastGoodTradeDatetime = new Date(processDate.valueOf()); - - } - - findNextHole(); - - } else { - - if (holeFixingStatusReport.monthChecked === true) { - - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> The current year / month was already fully checked for market."); - - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "Month fully processed." - } - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); - callBackFunction(customOK); - return; - - } else { - - /* We get from the file the datetime of the last file without holes. */ - - datetimeLastFileWithoutHoles = new Date(holeFixingStatusReport.lastFile.year + "-" + holeFixingStatusReport.lastFile.month + "-" + holeFixingStatusReport.lastFile.days + " " + holeFixingStatusReport.lastFile.hours + ":" + holeFixingStatusReport.lastFile.minutes + GMT_SECONDS); - - lastGoodTradeId = holeFixingStatusReport.lastTrade.id; - lastGoodTradeDatetime = new Date(holeFixingStatusReport.lastTrade.year + "-" + holeFixingStatusReport.lastTrade.month + "-" + holeFixingStatusReport.lastTrade.days + " " + holeFixingStatusReport.lastTrade.hours + ":" + holeFixingStatusReport.lastTrade.minutes + ":" + holeFixingStatusReport.lastTrade.seconds + GMT_MILI_SECONDS); - - findNextHole(); - } - } - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> err = " + err.stack); - if (err.message === "Cannot read property 'file' of undefined") { - logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Check the bot configuration to see if all of its statusDependencies declarations are correct. "); - logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys)); - } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function findNextHole() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> Entering function."); } - - /* - - To find the next hole, we will read first the current file (which we know if fully ok) and get from it the current Trade Id. - After that, we will continue reading the next files, checking all the ids, until we find a hole. - - */ - - let filePath; - let fileName = '' + market.baseAsset + '_' + market.quotedAsset + '.json'; - let date; // This is pointing to each Trades File - - let fileCheckedCounter = 0; - - date = new Date(datetimeLastFileWithoutHoles.valueOf()); - - readNextFile(); - - function readNextFile() { - - try { - - date = new Date(date.valueOf() + 60 * 1000); - - if (date.valueOf() > datetimeLastFileLiveTrade.valueOf()) { - - /* This mean we reached the forward end of the market */ - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> readNextFile -> Head of the market reached at date = " + date.toUTCString()); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> readNextFile -> datetimeLastFileLiveTrade = " + datetimeLastFileLiveTrade.toUTCString()); } - - writeStatusReport(lastGoodTradeDatetime, lastGoodTradeId, false, true, onStatusReportWritten); - - function onStatusReportWritten(err) { - - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> readNextFile -> onStatusReportWritten -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> readNextFile -> onStatusReportWritten -> err = " + err.stack); - callBackFunction(err); - return; - } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> readNextFile -> onStatusReportWritten -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - return; - } - - /* Lets check if we have reached the end of the month. */ - - if (date.valueOf() > lastMinuteOfMonth.valueOf()) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> readNextFile -> End of the month reached at date = " + date.toUTCString()); } - - writeStatusReport(lastGoodTradeDatetime, lastGoodTradeId, true, false, onStatusReportWritten); - - function onStatusReportWritten(err) { - - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> readNextFile -> onStatusReportWritten -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> readNextFile -> onStatusReportWritten -> err = " + err.stack); - callBackFunction(err); - return; - } - - let customOK = { - result: global.CUSTOM_OK_RESPONSE.result, - message: "End of the month reached." - } - logger.write(MODULE_NAME, "[WARN] start -> findNextHole -> readNextFile -> onStatusReportWritten -> customOK = " + customOK.message); - callBackFunction(customOK); - - return; - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> readNextFile -> onStatusReportWritten -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - return; - } - - dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2); - - filePath = bot.filePathRoot + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath; - filePath += '/' + fileName - - fileStorage.getTextFile(filePath, onNextFileReceived, true); - - logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> readNextFile -> reading file at dateForPath = " + dateForPath); - - function onNextFileReceived(err, text) { - - try { - - if (err.result === global.DEFAULT_FAIL_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> readNextFile -> onNextFileReceived -> err = " + err.stack); - callBackFunction(err); - return; - } - - if ( - (err.result === global.CUSTOM_FAIL_RESPONSE.result && - (err.message === 'Folder does not exist.' || err.message === 'File does not exist.')) || - err.code === 'The specified key does not exist.' - ) { - logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> readNextFile -> onNextFileReceived -> err = " + err.stack); - - /* - FILE IS MISSING SITUATION: - - The file does not exist, so this means there is a hole. To be precise we dont know if there are missing trades at the - missing files, but this process responsibility is to fix the data set and create all missing files even if there are - no trades on them. - - */ - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> readNextFile -> onNextFileReceived -> onNextFileReceived -> Hole by missing file detected. Date = " + date.toUTCString()); } - - holeInitialId = lastGoodTradeId; - holeInitialDatetime = new Date(lastGoodTradeDatetime.valueOf()); // Field #5 contains the seconds. - - findEndOfHole(); - return; - } - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - try { - - /* Sometimes files get corruped when written to disk. We make this test to see if this is the case. */ - - let tradesTest = JSON.parse(text); - - } catch (err) { - - /* - FILE IS CORRUPT SITUATION: - - If the file is corrupt, then we are in a similar situation as if it does not exist. - */ - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> readNextFile -> onNextFileReceived -> onNextFileReceived -> Hole by corrupt file detected. Date = " + date.toUTCString()); } - - holeInitialId = lastGoodTradeId; - holeInitialDatetime = new Date(lastGoodTradeDatetime.valueOf()); // Field #5 contains the seconds. - - findEndOfHole(); - return; - } - - /* - MAIN EXECUTION CONTINUES HERE: - - The file exists and it is not corruped, then we need to examine it to see if it has holes. - - */ - - checkHolesInFile(text); - return; - } - - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> readNextFile -> onNextFileReceived -> onNextFileReceived -> Unhandled response received. err = " + err.stack); - callBackFunction(err); - return; - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> readNextFile -> onNextFileReceived -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> readNextFile -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - function checkHolesInFile(text) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> checkHolesInFile -> Entering function."); } - - let trades = JSON.parse(text); - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> checkHolesInFile -> Checking File '" + fileName + "' @ " + filePath + " - " + trades.length + " records in it."); } - - /* The next block is needed ONLY at the begining of the market. */ - - if (lastGoodTradeId === FIRST_TRADE_RECORD_ID || lastGoodTradeId === UNKNOWN_TRADE_RECORD_ID) { - - /* - Here we dont know the lastGoodTradeId, so we might take it directly from the file. There will be a problem though if the market starts - with some empty files. In those cases, we need to jump to the next file until we find one with some records. - */ - - if (trades.length > 0) { - - lastGoodTradeId = trades[0][0] - 1; - - } else { - - readNextFile(); - return; - } - } - - for (let i = 0; i < trades.length; i++) { // We go through every trade at the file. - - let firstTradeIdAtCurrentFile = trades[i][0]; - - if (lastGoodTradeId + 1 > firstTradeIdAtCurrentFile) { - - /* - This happens when the process resumes execution, reads the first file and the first trades have lowers ids that the ones the process already - checked during the last execution. - - It also happens when we find a non valid id, as the one used in an empty record signaling that the file is incomplete. (zero). - */ - continue; // we simply jump to the next trade. - } - - if (lastGoodTradeId + 1 < firstTradeIdAtCurrentFile) { - - /* - - WE FOUND A HOLE! - - We should usually try to fix the hole, but there is an exception. If we already tried this 3 times already, we must declare the problem - unsolvable and move forward. - - */ - - tradesAtFileWhereHoleEnds = trades; - - let lastRecordedTradeId = 0; - let lastRecordedCounter = 0; - - if (holeFixingStatusReport.lastTrade !== undefined) { // The hole could have benn found before the monthly report was created. - - lastRecordedTradeId = holeFixingStatusReport.lastTrade.id; - lastRecordedCounter = holeFixingStatusReport.lastTrade.counter; - } - - if (lastGoodTradeId === lastRecordedTradeId && lastRecordedCounter >= MAX_HOLE_FIXING_RETRIES) { - - /* This is part of the mechanism to avoid getting stuck for ever at a hole which can not be fixed. By remmebering at the status - report this id, and counting the amount of time we tried to fix it, we can later check here if we reach the limit and just assume - this hole is not possible to fix. */ - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> checkHolesInFile -> Hole by non consecutive ID detected. MAX_HOLE_FIXING_RETRIES reched, giving up with this validation. Date = " + date.toUTCString()); } - - /* We advance anyway to the next Id since there is no other solution. */ - - lastGoodTradeId = firstTradeIdAtCurrentFile; - lastGoodTradeDatetime = new Date(date.valueOf() + trades[i][5] * 1000); - - } else { - - /* - - MAIN EXECUTION CONTINUES HERE: - - Here we have a hole that needs to be fixed !!! - - */ - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> checkHolesInFile -> Hole by non consecutive ID detected. Date = " + date.toUTCString()); } - - holeInitialId = lastGoodTradeId; - holeInitialDatetime = new Date(lastGoodTradeDatetime.valueOf()); - - holeFinalId = firstTradeIdAtCurrentFile; - holeFinalDatetime = new Date(date.valueOf() + trades[i][5] * 1000); // Field #5 contains the seconds. - - getTheTrades(); - return; - } - - } else { - - /* - Two last IDs compared are consecutive, which means there is no hole. - We remember the last Trade Id and Datetime that passed the check. - */ - - lastGoodTradeId = firstTradeIdAtCurrentFile; - lastGoodTradeDatetime = new Date(date.valueOf() + trades[i][5] * 1000); - } - } - - /* We went through all the trades in the file and there seems not to be any problems here. */ - - fileCheckedCounter++; - - /* Every hour checked we write a Status Report so that if the process is terminated, it can resume later from there. */ - - if (fileCheckedCounter === 60) { - - writeStatusReport(lastGoodTradeDatetime, lastGoodTradeId, false, false, onStatusReportWritten); - - function onStatusReportWritten(err) { - - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> checkHolesInFile -> onStatusReportWritten -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> checkHolesInFile -> onStatusReportWritten -> err = " + err.stack); - callBackFunction(err); - return; - } - - fileCheckedCounter = 0; - - return; - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> checkHolesInFile -> onStatusReportWritten -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - } - - /* - We need to remember the trades at the file where the hole starts, in order to later being able to complete this file. At this moment - we dont know if this file will be the one holding the last good id. So we store its trades at this variable so that if it is indeed, - we can later use those trades to save the completed version of the file. - */ - - if (trades.length > 0) { - tradesAtFileWhereHoleBegins = trades; - } - - readNextFile(); - - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> checkHolesInFile -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - function findEndOfHole() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> findEndOfHole -> Entering function."); } - - /* Here we will enter a loop where will try to find the next available file recorded and extract from it the Id and Datetime - from the first record. */ - - date = new Date(date.valueOf() + 60 * 1000); - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> findEndOfHole -> Date = " + date.toUTCString()); } - - if (date.valueOf() > datetimeLastFileLiveTrade.valueOf()) { - - /* - In this case we have an open hole produced by a missing file, and because live trades files contains zero records or are missing, we reached the - forward side of the market. The situation in unsolvable for now, we will leave it of future execution. - */ - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> findEndOfHole -> Head of the market reached -> Date = " + date.toUTCString()); } - - nextIntervalExecution = true; // Even if we didn-t find the end of the hole, we need to continue the execution of this month interval. - - writeStatusReport(lastGoodTradeDatetime, lastGoodTradeId, false, false, onStatusReportWritten); - - function onStatusReportWritten(err) { - - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> findEndOfHole -> onStatusReportWritten -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> findEndOfHole -> onStatusReportWritten -> err = " + err.stack); - callBackFunction(err); - return; - } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> findEndOfHole -> onStatusReportWritten -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - return; - } - - dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2); - - filePath = bot.filePathRoot + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath; - filePath += '/' + fileName - - fileStorage.getTextFile(filePath, onNextFileReceived, true); - - logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> findEndOfHole -> reading file at dateForPath = " + dateForPath); - - function onNextFileReceived(err, text) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> findEndOfHole -> onNextFileReceived -> Entering function."); } - - if (err.result === global.DEFAULT_FAIL_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> findEndOfHole -> onNextFileReceived -> err = " + err.stack); - callBackFunction(err); - return; - } - - if ( - (err.result === global.CUSTOM_FAIL_RESPONSE.result && - (err.message === 'Folder does not exist.' || err.message === 'File does not exist.')) || - err.code === 'The specified key does not exist.' - ) { - logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> findEndOfHole -> onNextFileReceived -> err = " + err.stack); - - /* The file does not exist, so this means we need to move forward */ - - findEndOfHole(); - return; - } - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - let trades; - - try { - - trades = JSON.parse(text); - - } catch (err) { - - /* If the file is corrupt, then we are in a similar situation as if it does not exist. */ - - logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> findEndOfHole -> onNextFileReceived -> Corrupt file with no records. -> Date = " + date.toUTCString()); - - findEndOfHole(); - return; - } - - if (trades.length === 0) { - - /* This is the same situation that if there is no file, move forward */ - - logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> findEndOfHole -> onNextFileReceived -> File with no records. -> Date = " + date.toUTCString()); - - findEndOfHole(); - return; - } - - logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> findEndOfHole -> onNextFileReceived -> Next available record found at date = " + date.toUTCString()); - - let firstTradeIdAtCurrentFile = trades[0][0]; // First position in each record. - - holeFinalId = firstTradeIdAtCurrentFile; - holeFinalDatetime = new Date(date.valueOf() + trades[0][5] * 1000); // Field #5 contains the seconds. - - getTheTrades(); - return; - } - - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> findEndOfHole -> onNextFileReceived -> Unhandled response received. err = " + err.stack); - callBackFunction(err); - return; - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> findEndOfHole -> onNextFileReceived -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> findEndOfHole -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - function getTheTrades() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getTheTrades -> Entering function."); } - - /* - We request to the Exchange API some more records than needed, anyway we will discard records out of the range we need. - To do this we substract 120 seconds and add 10 seconds to the already calculated current date. - */ - - let startTime = parseInt(holeInitialDatetime.valueOf() / 1000 - 65); - let endTime - - if (exchangeCalls === 0) { // this means that we still have not requested the trades from the exchange. - endTime = parseInt(holeFinalDatetime.valueOf() / 1000 + 65); - } else { - let lastRecordDate = new Date(consolidatedResponse[consolidatedResponse.length - 1].date + GMT_MILI_SECONDS) - endTime = lastRecordDate.valueOf() / 1000 + 2 // 2 seconds of overlap - - if (lastRecordDate.valueOf() < holeInitialDatetime.valueOf()) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getTheTrades -> No need to ask for more trades."); } - tradesReadyToBeSaved(consolidatedResponse); - return - } - } - - if (consolidatedResponse.length > 0) { - if (lastReceivedTradeId === consolidatedResponse[consolidatedResponse.length - 1].tradeID) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getTheTrades -> No new records at ID = " + lastReceivedTradeId); } - tradesReadyToBeSaved(consolidatedResponse); - return - } else { - lastReceivedTradeId = consolidatedResponse[consolidatedResponse.length - 1].tradeID - } - } - - exchangeCalls++ - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getTheTrades -> exchangeCalls = " + exchangeCalls); } - if (exchangeCalls > MAX_EXCHANGE_CALLS_FOR_GETTING_TRADES) { - tradesReadyToBeSaved(consolidatedResponse); - return - } - - exchangeCallTime = new Date(); - EXCHANGE_API.getPublicTradeHistory(market.baseAsset, market.quotedAsset, startTime, endTime, onExchangeCallReturned); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> getTheTrades -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - function onExchangeCallReturned(err, exchangeResponse) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> onExchangeCallReturned -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[WARN] start -> tradesReadyToBeSaved -> Somethinig is wrong with the Exchange Response. "); - logger.write(MODULE_NAME, "[WARN] start -> tradesReadyToBeSaved -> err.message = " + err.stack); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - if (exchangeResponse === undefined) { - logger.write(MODULE_NAME, "[WARN] start -> tradesReadyToBeSaved -> exchangeResponse is UNDEFINED. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - if (FULL_LOG === true) { - - let exchangeResponseTime = new Date(); - let timeDifference = (exchangeResponseTime.valueOf() - exchangeCallTime.valueOf()) / 1000; - logger.write(MODULE_NAME, "[INFO] start -> onExchangeCallReturned -> Call time recorded = " + timeDifference + " seconds."); - } - - consolidatedResponse = consolidatedResponse.concat(exchangeResponse); - getTheTrades() - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> onExchangeCallReturned -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - function tradesReadyToBeSaved(tradesRequested) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> Entering function."); } - - /* We will set true this next variable once we reach the point where we found the current minute where we had the last known verified trade - without holes and we already added the previous content to the file. Then that means there is no need to continue processing further records.*/ - let verifiedTradesAllreadyAdded = false - - /* - We have learnt that the records from the exchange dont always come in the right order, sorted by TradeId. That means the we need to sort them - by ourselves if we want that our verification of holes work. - */ - - let iterations = tradesRequested.length; - - for (let i = 0; i < iterations; i++) { - - for (let j = 0; j < iterations - 1; j++) { - - if (tradesRequested[j].tradeID < tradesRequested[j + 1].tradeID) { - - let trade = tradesRequested[j + 1]; - tradesRequested.splice(j + 1, 1); // Remove that trade from the array. - tradesRequested.splice(j, 0, trade); // Insert the trade removed. - } - } - } - /* - The trades received from the exchange might or might not be enough to fix the hole. We wont worry about that at this point. We will simple record the trades received - in the range where records where missing. - - We only have to take into account that the lowest id we have is already on a file that exist and it is partially verified, so we have to be carefull to overwrite this file - without introducing new holes. - */ - - let fileRecordCounter = 0; - let needSeparator; - let separator; - - let lastProcessMinute; // Stores the previous record minute during each iteration - let filesToSave = []; // Array where we will store all the content to be written to files - - needSeparator = false; - - let fileContent = ""; - - let currentProcessMinute = Math.trunc(holeFinalDatetime.valueOf() / 1000 / 60); // Number of minutes since the begining of time, where the process is pointing to. - let holeStartsMinute = Math.trunc(holeInitialDatetime.valueOf() / 1000 / 60); // Number of minutes since the begining of time, where the hole started. - - /* We will iterate through all the records received from the exchange. // We expect the exchange API to return the records ordered by ID DESC so we change it to ASC. So this is going to be going back in time as we advance. */ - - for (let i = 0; i < tradesRequested.length; i++) { - - if (verifiedTradesAllreadyAdded === true) { - break - } - - let record = tradesRequested[i]; - - const trade = { - tradeIdAtExchange: record.tradeID, - marketIdAtExchange: record.globalTradeID, - type: record.type, - rate: record.rate, - amountA: record.total, - amountB: record.amount, - datetime: new Date(record.date + GMT_MILI_SECONDS) - }; - - trade.seconds = trade.datetime.getUTCSeconds(); - - let currentExchangeTradeMinute = Math.trunc(trade.datetime.valueOf() / 1000 / 60); // This are the number of minutes since the begining of time of this trade. - - if (currentExchangeTradeMinute > currentProcessMinute) { - - /* We discard this trade, since it happened after the minute we want to record in the current file. */ - - continue; - } - - if (currentExchangeTradeMinute < currentProcessMinute) { - - /* - - As soon as i find the first trade that belongs to the previous minute in relation to the ones i was packing, i need to close that package, - and start going back in time. - - */ - - let minutesToGoBack = currentProcessMinute - currentExchangeTradeMinute; - - for (let j = 1; j <= minutesToGoBack; j++) { - - packageFileContent(); - currentProcessMinute--; - } - } - - if (currentExchangeTradeMinute === currentProcessMinute) { - - if (needSeparator === false) { - - needSeparator = true; - separator = ''; - - } else { - separator = ','; - } - - if (trade.tradeIdAtExchange > holeInitialId) { - - /* We only add trades with ids bigger that the last id verified without holes. */ - - fileContent = '[' + trade.tradeIdAtExchange + ',"' + trade.type + '",' + trade.rate + ',' + trade.amountA + ',' + trade.amountB + ',' + trade.seconds + ']' + separator + fileContent; - - fileRecordCounter++; - } - } - } - - if (fileContent !== "") { - - /* - Usually the last file Content must be discarded since it could belong to an incomplete file. But there is one exception: it a hole is found at a file and the previous minute is empty - then this will produce the exception in which the fileContent needs to saved. To figure out if we are in this situation we do the following: - */ - - if (currentProcessMinute === holeStartsMinute) { - - packageFileContent(); - } - } - - function packageFileContent() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> packageFileContent -> Entering function."); } - - let existingFileContent = ""; - let separator = ""; - - if (currentProcessMinute === holeStartsMinute) { - - /* - Here we are at the situation that the content already generated has to be added to the content already existing on the file where the hole was found. - */ - verifiedTradesAllreadyAdded = true - - for (let i = 0; i < tradesAtFileWhereHoleBegins.length; i++) { - - if (tradesAtFileWhereHoleBegins[i][0] <= holeInitialId && tradesAtFileWhereHoleBegins[i][0] !== 0) { // 0 because of the empty trade record signaling an incomplete file. - - /* We only add trades with ids smallers that the last id verified without holes. */ - - existingFileContent = existingFileContent + separator + '[' + tradesAtFileWhereHoleBegins[i][0] + ',"' + tradesAtFileWhereHoleBegins[i][1] + '",' + tradesAtFileWhereHoleBegins[i][2] + ',' + tradesAtFileWhereHoleBegins[i][3] + ',' + tradesAtFileWhereHoleBegins[i][4] + ',' + tradesAtFileWhereHoleBegins[i][5] + ']'; - fileRecordCounter++; - - if (separator === "") { - - separator = ","; - } - } - } - } - - if (existingFileContent === "") { - - fileContent = '[' + fileContent + ']'; - - } else { - - if (fileContent === "") { - fileContent = '[' + existingFileContent + ']'; - } else { - fileContent = '[' + existingFileContent + "," + fileContent + ']'; - } - } - - let fileRecord = { - datetime: currentProcessMinute, - content: fileContent, - records: fileRecordCounter - }; - - filesToSave.push(fileRecord); - - fileRecordCounter = 0; - needSeparator = false; - fileContent = ""; - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> tradesReadyToBeSaved -> packageFileContent -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - /* Now it is time to process all the information we stored at filesToSave. */ - - let i = 0; - let date; - - nextRecord(); - - function nextRecord() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> Entering function."); } - - if (filesToSave.length === 0) { - - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> No file to save."); - controlLoop(); - return - } - - let fileName = '' + market.baseAsset + '_' + market.quotedAsset + '.json'; - - date = new Date(filesToSave[i].datetime * 60 * 1000); - fileRecordCounter = filesToSave[i].records; - fileContent = filesToSave[i].content; - - dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2); - - filePath = bot.filePathRoot + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath; - filePath += '/' + fileName - - fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated); - - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> creating file at dateForPath = " + dateForPath); - - function onFileCreated(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> err = " + err.stack); - callBackFunction(err); - return; - } - - if (LOG_FILE_CONTENT === true) { - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Content written = " + fileContent); - } - - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Finished with File @ " + market.baseAsset + "_" + market.quotedAsset); - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Records inserted = " + fileRecordCounter); - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Path = " + filePath + "/" + fileName + ""); - - controlLoop(); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> tradesReadyToBeSaved -> nextRecord -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - function controlLoop() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> controlLoop -> Entering function."); } - - i++; - - if (i < filesToSave.length) { - - nextRecord(); - - } else { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> controlLoop -> Leaving function 'tradesReadyToBeSaved'."); } - - writeStatusReport(undefined, undefined, false, false, onStatusReportWritten); - - function onStatusReportWritten() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> controlLoop -> onStatusReportWritten -> Entering function."); } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> tradesReadyToBeSaved -> controlLoop -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> tradesReadyToBeSaved -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - function writeStatusReport(lastTradeDatetime, lastTradeId, monthChecked, atHeadOfMarket, callBack) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> Entering function."); } - - /* - If no parameters are provided, that means that last good information is the begining of the hole. If they are provided is because no hole was detected until the - forward end of the market. - */ - - let key = bot.dataMine + "-" + bot.codeName + "-" + bot.process + "-" + bot.dataSetVersion + "-" + year + "-" + month; - let statusReport = statusDependencies.statusReports.get(key); - - if (lastTradeId === undefined) { - - lastTradeId = holeInitialId; - lastTradeDatetime = holeInitialDatetime; - } - - let lastFileWithoutHoles = new Date(lastTradeDatetime.valueOf() - 60 * 1000); // It is the previous file where the last verified trade is. - - /* - Here we will calculate the "counter". The counter keeps track of how many times the process tried to fix the same hole. This allows - the process to know when a hole is not fixable. To do that we need to compare the current status report with the information we ve got - about the hole. If it is the same, we add to the counter. - */ - - let counter = 0; - - try { - - if (holeFixingStatusReport.lastTrade.id === lastTradeId) { - - counter = holeFixingStatusReport.lastTrade.counter; - - if (counter === undefined) { counter = 0; } - counter++; - } - - } catch (err) { // we are here when the status report did not exist. - counter = 0; - } - - statusReport.file = { - lastFile: { - year: lastFileWithoutHoles.getUTCFullYear(), - month: (lastFileWithoutHoles.getUTCMonth() + 1), - days: lastFileWithoutHoles.getUTCDate(), - hours: lastFileWithoutHoles.getUTCHours(), - minutes: lastFileWithoutHoles.getUTCMinutes() - }, - lastTrade: { - year: lastTradeDatetime.getUTCFullYear(), - month: (lastTradeDatetime.getUTCMonth() + 1), - days: lastTradeDatetime.getUTCDate(), - hours: lastTradeDatetime.getUTCHours(), - minutes: lastTradeDatetime.getUTCMinutes(), - seconds: lastTradeDatetime.getUTCSeconds(), - id: lastTradeId, - counter: counter - }, - monthChecked: monthChecked, - atHeadOfMarket: atHeadOfMarket - }; - - statusReport.save(onSaved); - - function onSaved(err) { - - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> onSaved -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> onSaved -> err = " + err.stack); - callBackFunction(err); - return; - } - - /* - If we are at the same month of the begining of the market, we need to create the main status report file. - We will re-create it even every time the month status report is created. When this month check finished, other months later - will update it. - */ - - if (processDate.getUTCMonth() === datetimeLastFileHistoricTrade.getUTCMonth() && processDate.getUTCFullYear() === datetimeLastFileHistoricTrade.getUTCFullYear()) { - - createMainStatusReport(lastTradeDatetime, lastTradeId, onMainReportCreated); - - function onMainReportCreated() { - - if (monthChecked === true) { - - let key = bot.dataMine + "-" + bot.codeName + "-" + bot.process + "-" + bot.dataSetVersion; - let statusReport = statusDependencies.statusReports.get(key); - statusReport.verifyMarketComplete(callBack); - return; - - } else { - callBack(global.DEFAULT_OK_RESPONSE); - return; - } - } - - } else { - - if (monthChecked === true) { - - let key = bot.dataMine + "-" + bot.codeName + "-" + bot.process + "-" + bot.dataSetVersion; - let statusReport = statusDependencies.statusReports.get(key); - statusReport.verifyMarketComplete(callBack); - return; - - } else { - callBack(global.DEFAULT_OK_RESPONSE); - return; - } - } - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> onSaved -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - function createMainStatusReport(lastTradeDatetime, lastTradeId, callBack) { - - try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> createMainStatusReport -> Entering function."); } - - let key = bot.dataMine + "-" + bot.codeName + "-" + bot.process + "-" + bot.dataSetVersion; - - let statusReport = statusDependencies.statusReports.get(key); - - let lastFileWithoutHoles = new Date(lastTradeDatetime.valueOf() - 60 * 1000); // It is the previous file where the last verified trade is. - - statusReport.file = { - lastFile: { - year: lastFileWithoutHoles.getUTCFullYear(), - month: (lastFileWithoutHoles.getUTCMonth() + 1), - days: lastFileWithoutHoles.getUTCDate(), - hours: lastFileWithoutHoles.getUTCHours(), - minutes: lastFileWithoutHoles.getUTCMinutes() - }, - lastTrade: { - year: lastTradeDatetime.getUTCFullYear(), - month: (lastTradeDatetime.getUTCMonth() + 1), - days: lastTradeDatetime.getUTCDate(), - hours: lastTradeDatetime.getUTCHours(), - minutes: lastTradeDatetime.getUTCMinutes(), - seconds: lastTradeDatetime.getUTCSeconds(), - id: lastTradeId - } - }; - - statusReport.save(onSaved); - - function onSaved(err) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> createMainStatusReport -> onSaved -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> createMainStatusReport -> onSaved -> err = " + err.stack); - callBackFunction(err); - return; - } - - callBack(global.DEFAULT_OK_RESPONSE); - return; - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> createMainStatusReport -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } -}; diff --git a/Live-Trades/User.Bot.js b/Live-Trades/User.Bot.js deleted file mode 100644 index 6fd2aaee4..000000000 --- a/Live-Trades/User.Bot.js +++ /dev/null @@ -1,377 +0,0 @@ -exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, fileStorage, STATUS_REPORT, EXCHANGE_API) { - - const FULL_LOG = true; - const LOG_FILE_CONTENT = false; - const GMT_SECONDS = ':00.000 GMT+0000'; - const GMT_MILI_SECONDS = '.000 GMT+0000'; - const MODULE_NAME = "User Bot"; - const TRADES_FOLDER_NAME = "Trades"; - - thisObject = { - initialize: initialize, - start: start - }; - - let utilities = UTILITIES.newCloudUtilities(bot, logger); - - let statusDependencies; - - return thisObject; - - function initialize(pStatusDependencies, pMonth, pYear, callBackFunction) { - try { - - logger.fileName = MODULE_NAME; - logger.initialize(); - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } - statusDependencies = pStatusDependencies; - callBackFunction(global.DEFAULT_OK_RESPONSE); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - /* - - We are going to generate 2 files: - - A. The first one will contain all the trades of the current minute and will be store in a folder that we will create for this if it does not exist. - This file will be incomplete, since we are at the current minute and some trades will happen after we retrieve the information from the exchange, - but this is not a problem, since the second file is going to fix this. This file is only usefull for viewing a partial candle as it is being built - at the head of the market. - - B. The second file will contain all the trades of the previous minute. This will override the incomplete file written a minute before. - - FILE FORMAT - ----------- - - Array of records with this information: - - 1. Trade Id provided by the exchange. - 2. Trade Type: "buy" or "sell" - 3. Trade Rate: the rate of the transaction. - 4. Amount Asset A - 5. Amount Asset B - 6. Time: Seconds and Milliseconds the trade happened. (the rest of the time and date whoever reads the file already know it since it is organized in folders according to this.) - - */ - - function start(callBackFunction) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); } - - let market = bot.market; - - let currentDate; // This will hold the current datetime of each execution. - let previousMinute; // This hold the current time minus 60 seconds. - let marketQueue; // This is the queue of all markets to be procesesd at each interval. - - let dateForPathA; - let dateForPathB; - - let filePathA; - let filePathB; - - let reportFilePath = bot.exchange + "/Processes/" + bot.process; - - let exchangeCallTime; - - firstSteps(); - - function firstSteps() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> firstSteps -> Entering function."); } - - currentDate = bot.processDatetime; - - previousMinute = new Date(currentDate.valueOf() - 60000); - - dateForPathA = currentDate.getUTCFullYear() + '/' + utilities.pad(currentDate.getUTCMonth() + 1, 2) + '/' + utilities.pad(currentDate.getUTCDate(), 2) + '/' + utilities.pad(currentDate.getUTCHours(), 2) + '/' + utilities.pad(currentDate.getUTCMinutes(), 2); - dateForPathB = previousMinute.getUTCFullYear() + '/' + utilities.pad(previousMinute.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousMinute.getUTCDate(), 2) + '/' + utilities.pad(previousMinute.getUTCHours(), 2) + '/' + utilities.pad(previousMinute.getUTCMinutes(), 2); - - filePathA = bot.filePathRoot + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPathA; - filePathB = bot.filePathRoot + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPathB; - - getTheTrades(); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> firstSteps -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function getTheTrades() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getTheTrades -> Entering function."); } - - const datetime = parseInt(currentDate.valueOf() / 1000); - - /* - We request to the Exchange API some more records than needed, anyway we will discard records out of the range we need. - To do this we substract 120 seconds and add 10 seconds to the already calculated current date. - */ - - const startTime = datetime - 120; - const endTime = datetime + 10; - - exchangeCallTime = new Date(); - - EXCHANGE_API.getPublicTradeHistory(market.baseAsset, market.quotedAsset, startTime, endTime, onExchangeCallReturned); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> getTheTrades -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function onExchangeCallReturned(err, exchangeResponse) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> onExchangeCallReturned -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[WARN] start -> tradesReadyToBeSaved -> Somethinig is wrong with the Exchange Response. "); - logger.write(MODULE_NAME, "[WARN] start -> tradesReadyToBeSaved -> err.message = " + err.stack); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - if (exchangeResponse === undefined) { - logger.write(MODULE_NAME, "[WARN] start -> tradesReadyToBeSaved -> exchangeResponse is UNDEFINED. "); - callBackFunction(global.DEFAULT_RETRY_RESPONSE); - return; - } - - if (FULL_LOG === true) { - - let exchangeResponseTime = new Date(); - let timeDifference = (exchangeResponseTime.valueOf() - exchangeCallTime.valueOf()) / 1000; - logger.write(MODULE_NAME, "[INFO] start -> onExchangeCallReturned -> Call time recorded = " + timeDifference + " seconds."); - } - - tradesReadyToBeSaved(exchangeResponse); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> onExchangeCallReturned -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function tradesReadyToBeSaved(exchangeResponse) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> Entering function."); } - - let fileRecordCounterA = 0; - let fileRecordCounterB = 0; - - let fileNameA = '' + market.baseAsset + '_' + market.quotedAsset + '.json'; - let fileNameB = '' + market.baseAsset + '_' + market.quotedAsset + '.json'; - - let needSeparator; - let separator; - - needSeparator = false; - - let fileContent = ""; - - /* First we create the file A */ - - fileContent = fileContent + '['; - - for (i = 0; i < exchangeResponse.length; i++) { - - let record = exchangeResponse[exchangeResponse.length - 1 - i]; // We expect the exchange API to return the records ordered by ID DESC so we change it to ASC - - const trade = { - tradeIdAtExchange: record.tradeID, - marketIdAtExchange: record.globalTradeID, - type: record.type, - rate: record.rate, - amountA: record.total, - amountB: record.amount, - datetime: new Date(record.date + GMT_MILI_SECONDS) - }; - - trade.seconds = trade.datetime.getUTCSeconds(); - - if (trade.datetime.getUTCMinutes() === currentDate.getUTCMinutes()) { - - if (needSeparator === false) { - - needSeparator = true; - separator = ''; - - } else { - separator = ','; - } - - fileContent = fileContent + separator + '[' + trade.tradeIdAtExchange + ',"' + trade.type + '",' + trade.rate + ',' + trade.amountA + ',' + trade.amountB + ',' + trade.seconds + ']'; - - fileRecordCounterA++; - } - } - - fileContent = fileContent + ']'; - - fileStorage.createTextFile(filePathA +'/'+ fileNameA, fileContent + '\n', onFirstFileACreated); - - function onFirstFileACreated(err) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> onFirstFileACreated -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> tradesReadyToBeSaved -> onFirstFileACreated -> err = " + err.stack); - callBackFunction(err); - return; - } - - if (LOG_FILE_CONTENT === true) { - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> onFirstFileACreated -> Content written = " + fileContent); - } - - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> onFirstFileACreated -> Finished with File A @ " + market.baseAsset + "_" + market.quotedAsset); - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> onFirstFileACreated -> Records inserted = " + fileRecordCounterA); - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> onFirstFileACreated -> Path = " + filePathA + "/" + fileNameA + ""); - - generateFileB(); - } - - function generateFileB() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> generateFileB -> Entering function."); } - - fileContent = ""; - needSeparator = false; - - fileContent = fileContent + '['; - - for (i = 0; i < exchangeResponse.length; i++) { - - let record = exchangeResponse[exchangeResponse.length - 1 - i]; // We expect the exchange API to return the records ordered by ID DESC so we change it to ASC - - const trade = { - tradeIdAtExchange: record.tradeID, - marketIdAtExchange: record.globalTradeID, - type: record.type, - rate: record.rate, - amountA: record.total, - amountB: record.amount, - datetime: new Date(record.date + GMT_MILI_SECONDS) - }; - - trade.seconds = trade.datetime.getUTCSeconds(); - - if (trade.datetime.getUTCMinutes() === previousMinute.getUTCMinutes()) { - - if (needSeparator === false) { - - needSeparator = true; - separator = ''; - - } else { - separator = ','; - } - - fileContent = fileContent + separator + '[' + trade.tradeIdAtExchange + ',"' + trade.type + '",' + trade.rate + ',' + trade.amountA + ',' + trade.amountB + ',' + trade.seconds + ']'; - - fileRecordCounterB++; - } - } - - fileContent = fileContent + ']'; - - fileStorage.createTextFile(filePathB + '/' + fileNameB, fileContent + '\n', onFileBCreated); - - function onFileBCreated(err) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> onFileBCreated -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> tradesReadyToBeSaved -> onFileBCreated -> err = " + err.stack); - callBackFunction(err); - return; - } - - if (LOG_FILE_CONTENT === true) { - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> onFileBCreated -> Content written = " + fileContent); - } - - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> onFileBCreated -> Finished with File B @ " + market.baseAsset + "_" + market.quotedAsset); - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> onFileBCreated -> Content written -> Records inserted = " + fileRecordCounterB); - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> onFileBCreated -> Content written -> Path = " + filePathB + "/" + fileNameB + ""); - - writeStatusReport(); - } - } - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> tradesReadyToBeSaved -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function writeStatusReport() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> Entering function."); } - - let key = bot.dataMine + "-" + bot.codeName + "-" + bot.process + "-" + bot.dataSetVersion; - - let statusReport = statusDependencies.statusReports.get(key); - - statusReport.file = { - firstFile: { // This date points to the file that might be incomplete. - year: currentDate.getUTCFullYear(), - month: (currentDate.getUTCMonth() + 1), - days: currentDate.getUTCDate(), - hours: currentDate.getUTCHours(), - minutes: currentDate.getUTCMinutes() - }, - lastFile: { // This will point to the last file written with is complete. That means it has all the trades in it. - year: previousMinute.getUTCFullYear(), - month: (previousMinute.getUTCMonth() + 1), - days: previousMinute.getUTCDate(), - hours: previousMinute.getUTCHours(), - minutes: previousMinute.getUTCMinutes() - } - }; - - statusReport.save(onSaved); - - function onSaved(err) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> onSaved -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> onSaved -> err = " + err.stack); - callBackFunction(err); - return; - } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } -};