From 19f03f8adb180540b749754523bf6eadcbca25ad Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 8 Aug 2019 15:04:31 +0200 Subject: [PATCH] Change not commited before --- Hole-Fixing/User.Bot.js | 80 ++++++++++++++++++----------------------- Live-Trades/User.Bot.js | 37 ++++++------------- 2 files changed, 45 insertions(+), 72 deletions(-) diff --git a/Hole-Fixing/User.Bot.js b/Hole-Fixing/User.Bot.js index 94ce6c2f8..cdf54cf2d 100644 --- a/Hole-Fixing/User.Bot.js +++ b/Hole-Fixing/User.Bot.js @@ -1,4 +1,4 @@ -exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, STATUS_REPORT, EXCHANGE_API) { +exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, fileStorage, STATUS_REPORT, EXCHANGE_API) { const FULL_LOG = true; const LOG_FILE_CONTENT = false; @@ -15,12 +15,8 @@ start: start }; - let charlyStorage = BLOB_STORAGE.newBlobStorage(bot, logger); - let utilities = UTILITIES.newCloudUtilities(bot, logger); - let statusReportModule = STATUS_REPORT.newStatusReport(bot, logger, BLOB_STORAGE, UTILITIES); - let year; let month; @@ -74,20 +70,7 @@ return; } - charlyStorage.initialize(bot.devTeam, onCharlyInizialized); - - function onCharlyInizialized(err) { - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> onCharlyInizialized -> Initialization Succeed."); } - callBackFunction(global.DEFAULT_OK_RESPONSE); - - } else { - logger.write(MODULE_NAME, "[ERROR] initialize -> onCharlyInizialized -> err = " + err.message); - callBackFunction(err); - } - } + callBackFunction(global.DEFAULT_OK_RESPONSE); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); @@ -96,41 +79,41 @@ } /* - + This process is going to do the following: - + It will try to find "holes" on the transaction history and fix them by retrieving the missing transactions from the exchange. - + What are holes? - + Holes are missing transactions in the history of the market and the reason there can be holes are the following: - + 1) The "Lives Trades" process can be stopped and later started again. The process will only retrieve live trades, leaving a "hole" between it was stopped and restarted. 2) The "Historic Trades" process might have received a "hole" of transactions directly from the exchange. 3) The "Historic Trades" process failed to write some files to the storage and now they are missing, together with the transaction in them. 4) The "Historic Trades" process wrote some files to the storage with corrupted content. 5) The "Historic Trades" process wrote files with transaction not properly ordered by Trade Id (sometimes the exchange sent this wrong). - + How does the process work? - + Transactions have a unique id generated by the exchange. It is a secuencial number. This process scans the transaction history looking for missing ids. When it finds some group of missing ids, it request the exchange the missing records and it writes the necesary files to fix the hole. But some things might happen: - + 1) The exchange responds, but does not provides the missing records. In this case the process will flag this hole for retry later, up to 3 times. After that it will flag the hole as "permanent". 2) The exchange call timeouts: the process will continue retrying later. 3) The exchange returns an unexpected error: The process will retry up to 3 times later under this condition. 4) The exchange returns neighboring transactions but not the missing ones. The process will retry 3 more times and mark the hole as permanent if the exchange refuses to send the missing trades. - + Since the process is run in an infinite loop but can be restarted any time, the status of its running is recorded in the Status Report file. - + What is the lastFile pointer? - + It is the Datetime since the begining of the hitory of a market that is considered without holes (or only with permanent ones, unfixables). In other words, the process starts from the begining of the history of a market, and only when its history is signaled as "Complete" by the "Historic Trades" process. From there, the process is advancing in time, validating the history and fixing holes, and moving the lastFile pointer forward as it gets sure that all the previous trades files are with no holes or unfixable ones. This pointer will later be used by Indicators bots that depends of these trades, and it will be considered the head of the market. - + */ function start(callBackFunction) { @@ -243,7 +226,7 @@ return; } - if (thisReport.completeHistory === true) { // We get from the file to know if this markets history is complete or not. + if (thisReport.completeHistory === true) { // We get from the file to know if this markets history is complete or not. lastHistoricTradeFile = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS); @@ -289,7 +272,7 @@ if (holeFixingStatusReport.lastFile === undefined) { - /* + /* The file does not exist, so this means this is the first time we are running this process. Then, we might me under two different possible situations: @@ -457,8 +440,9 @@ dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2); filePath = bot.filePathRoot + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath; + filePath += '/' + fileName - charlyStorage.getTextFile(filePath, fileName, onNextFileReceived, true); + fileStorage.getTextFile(bot.devTeam, filePath, onNextFileReceived, true); console.log("[INFO] start -> findNextHole -> readNextFile -> reading file at dateForPath = " + dateForPath); @@ -475,8 +459,9 @@ } if ( - err.result === global.CUSTOM_FAIL_RESPONSE.result && - (err.message === 'Folder does not exist.' || err.message === 'File does not exist.') + (err.result === global.CUSTOM_FAIL_RESPONSE.result && + (err.message === 'Folder does not exist.' || err.message === 'File does not exist.')) || + err.code === 'The specified key does not exist.' ) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> readNextFile -> onNextFileReceived -> err = " + err.message); @@ -588,7 +573,7 @@ /* We should usually try to fix the hole, but there is an exception. If the we tried this 3 times already, we must declare the problem - unsolvable and move forward. + unsolvable and move forward. */ let lastRecordedTradeId = 0; @@ -727,8 +712,9 @@ dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2); filePath = bot.filePathRoot + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath; + filePath += '/' + fileName - charlyStorage.getTextFile(filePath, fileName, onNextFileReceived, true); + fileStorage.getTextFile(bot.devTeam, filePath, onNextFileReceived, true); console.log("[INFO] start -> findNextHole -> findEndOfHole -> reading file at dateForPath = " + dateForPath); @@ -745,8 +731,9 @@ } if ( - err.result === global.CUSTOM_FAIL_RESPONSE.result && - (err.message === 'Folder does not exist.' || err.message === 'File does not exist.') + (err.result === global.CUSTOM_FAIL_RESPONSE.result && + (err.message === 'Folder does not exist.' || err.message === 'File does not exist.')) || + err.code === 'The specified key does not exist.' ) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> findEndOfHole -> onNextFileReceived -> err = " + err.message); @@ -887,7 +874,7 @@ /* We have learnt that the records from the exchange dont always come in the right order, sorted by TradeId. That means the we need to sort them - by ourselves if we want that our verification of holes work. + by ourselves if we want that our verification of holes work. */ let iterations = tradesRequested.length; @@ -909,7 +896,7 @@ in the range where records where missing. We only have to take into account that the lowest id we have is already on a file that exist and it is partially verified, so we have to be carefull to overwrite this file - without introducing new holes. + without introducing new holes. */ let fileRecordCounter = 0; @@ -955,7 +942,7 @@ if (currentRecordMinute < currentProcessMinute) { - /* + /* The information is older that the current time. We must store the current info and reset the pointer to the current time to match the one on the information currently being processd. We know this can lead to a 'hole' or some empty files being skipped, but we solve that problem with the next loop. @@ -994,7 +981,7 @@ if (fileContent !== "") { - /* + /* Usually the last file Content must be discarded since it could belong to an incomplete file. But there is one exception: it a hole is found at a file and the previous minute is empty then this will produce the exception in which the fileContent needs to saved. To figure out if we are in this situation we do the following: */ @@ -1098,8 +1085,9 @@ dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2); filePath = bot.filePathRoot + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath; + filePath += '/' + fileName - charlyStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + fileStorage.createTextFile(bot.devTeam, filePath, fileContent + '\n', onFileCreated); console.log("[INFO] start -> tradesReadyToBeSaved -> nextRecord -> creating file at dateForPath = " + dateForPath); @@ -1258,7 +1246,7 @@ return; } - /* + /* If we are at the same month of the begining of the market, we need to create the main status report file. We will re-create it even every time the month status report is created. When this month check finished, other months later will update it. diff --git a/Live-Trades/User.Bot.js b/Live-Trades/User.Bot.js index 1012de4c4..da9050bc5 100644 --- a/Live-Trades/User.Bot.js +++ b/Live-Trades/User.Bot.js @@ -1,4 +1,4 @@ -exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, STATUS_REPORT, EXCHANGE_API) { +exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, fileStorage, STATUS_REPORT, EXCHANGE_API) { const FULL_LOG = true; const LOG_FILE_CONTENT = false; @@ -15,8 +15,6 @@ start: start }; - let charlyStorage = BLOB_STORAGE.newBlobStorage(bot, logger); - let utilities = UTILITIES.newCloudUtilities(bot, logger); let statusDependencies; @@ -34,20 +32,7 @@ statusDependencies = pStatusDependencies; - charlyStorage.initialize(bot.devTeam, onCharlyInizialized); - - function onCharlyInizialized(err) { - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> onCharlyInizialized -> Initialization Succeed."); } - callBackFunction(global.DEFAULT_OK_RESPONSE); - - } else { - logger.write(MODULE_NAME, "[ERROR] initialize -> onCharlyInizialized -> err = " + err.message); - callBackFunction(err); - } - } + callBackFunction(global.DEFAULT_OK_RESPONSE); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); @@ -56,28 +41,28 @@ } /* - + We are going to generate 2 files: - + A. The first one will contain all the trades of the current minute and will be store in a folder that we will create for this if it does not exist. This file will be incomplete, since we are at the current minute and some trades will happen after we retrieve the information from the exchange, but this is not a problem, since the second file is going to fix this. This file is only usefull for viewing a partial candle as it is being built at the head of the market. - + B. The second file will contain all the trades of the previous minute. This will override the incomplete file written a minute before. - + FILE FORMAT ----------- - + Array of records with this information: - + 1. Trade Id provided by the exchange. 2. Trade Type: "buy" or "sell" 3. Trade Rate: the rate of the transaction. 4. Amount Asset A 5. Amount Asset B 6. Time: Seconds and Milliseconds the trade happened. (the rest of the time and date whoever reads the file already know it since it is organized in folders according to this.) - + */ function start(callBackFunction) { @@ -246,7 +231,7 @@ fileContent = fileContent + ']'; - charlyStorage.createTextFile(filePathA, fileNameA, fileContent + '\n', onFirstFileACreated); + fileStorage.createTextFile(bot.devTeam, filePathA +'/'+ fileNameA, fileContent + '\n', onFirstFileACreated); function onFirstFileACreated(err) { @@ -313,7 +298,7 @@ fileContent = fileContent + ']'; - charlyStorage.createTextFile(filePathB, fileNameB, fileContent + '\n', onFileBCreated); + fileStorage.createTextFile(bot.devTeam, filePathB + '/' + fileNameB, fileContent + '\n', onFileBCreated); function onFileBCreated(err) {