From b0987d0961ed87bf42d5f6ff9c88bef31e7d9707 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 1 Apr 2019 08:33:07 +0200 Subject: [PATCH] Mati Changes to run in develop --- Poloniex-Hole-Fixing/User.bot.js | 121 ++++++++++++++----------------- this.bot.config.json | 6 +- 2 files changed, 59 insertions(+), 68 deletions(-) diff --git a/Poloniex-Hole-Fixing/User.bot.js b/Poloniex-Hole-Fixing/User.bot.js index e4fa528e0..6fd698d0c 100644 --- a/Poloniex-Hole-Fixing/User.bot.js +++ b/Poloniex-Hole-Fixing/User.bot.js @@ -1,4 +1,4 @@ -exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, STATUS_REPORT, POLONIEX_CLIENT_MODULE) { +exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, STATUS_REPORT, EXCHANGE_API) { const FULL_LOG = true; const LOG_FILE_CONTENT = false; @@ -20,8 +20,7 @@ let charlyStorage = BLOB_STORAGE.newBlobStorage(bot, logger); let utilities = UTILITIES.newCloudUtilities(bot, logger); - let poloniexApiClient = POLONIEX_CLIENT_MODULE.newPoloniexAPIClient(global.EXCHANGE_KEYS[global.EXCHANGE_NAME].Key, global.EXCHANGE_KEYS[global.EXCHANGE_NAME].Secret); - + let statusReportModule = STATUS_REPORT.newStatusReport(bot, logger, BLOB_STORAGE, UTILITIES); let year; @@ -98,43 +97,43 @@ } } -/* - -This process is going to do the following: - -It will try to find "holes" on the transaction history and fix them by retrieving the missing transactions from the exchange. - -What are holes? - - Holes are missing transactions in the history of the market and the reason there can be holes are the following: - - 1) The "Lives Trades" process can be stopped and later started again. The process will only retrieve live trades, leaving a "hole" between it was stopped and restarted. - 2) The "Historic Trades" process might have received a "hole" of transactions directly from the exchange. - 3) The "Historic Trades" process failed to write some files to the storage and now they are missing, together with the transaction in them. - 4) The "Historic Trades" process wrote some files to the storage with corrupted content. - 5) The "Historic Trades" process wrote files with transaction not properly ordered by Trade Id (sometimes the exchange sent this wrong). - -How does the process work? - - Transactions have a unique id generated by the exchange. It is a secuencial number. This process scans the transaction history looking for missing ids. When it finds - some group of missing ids, it request the exchange the missing records and it writes the necesary files to fix the hole. But some things might happen: - - 1) The exchange responds, but does not provides the missing records. In this case the process will flag this hole for retry later, up to 3 times. After that it will flag - the hole as "permanent". - 2) The exchange call timeouts: the process will continue retrying later. - 3) The exchange returns an unexpected error: The process will retry up to 3 times later under this condition. - 4) The exchange returns neighboring transactions but not the missing ones. The process will retry 3 more times and mark the hole as permanent if the exchange refuses to send the missing trades. - -Since the process is run in an infinite loop but can be restarted any time, the status of its running is recorded in the Status Report file. - -What is the lastFile pointer? - - It is the Datetime since the begining of the hitory of a market that is considered without holes (or only with permanent ones, unfixables). In other words, the process - starts from the begining of the history of a market, and only when its history is signaled as "Complete" by the "Historic Trades" process. From there, the process is advancing - in time, validating the history and fixing holes, and moving the lastFile pointer forward as it gets sure that all the previous trades files are with no holes or unfixable ones. - This pointer will later be used by Indicators bots that depends of these trades, and it will be considered the head of the market. - -*/ + /* + + This process is going to do the following: + + It will try to find "holes" on the transaction history and fix them by retrieving the missing transactions from the exchange. + + What are holes? + + Holes are missing transactions in the history of the market and the reason there can be holes are the following: + + 1) The "Lives Trades" process can be stopped and later started again. The process will only retrieve live trades, leaving a "hole" between it was stopped and restarted. + 2) The "Historic Trades" process might have received a "hole" of transactions directly from the exchange. + 3) The "Historic Trades" process failed to write some files to the storage and now they are missing, together with the transaction in them. + 4) The "Historic Trades" process wrote some files to the storage with corrupted content. + 5) The "Historic Trades" process wrote files with transaction not properly ordered by Trade Id (sometimes the exchange sent this wrong). + + How does the process work? + + Transactions have a unique id generated by the exchange. It is a secuencial number. This process scans the transaction history looking for missing ids. When it finds + some group of missing ids, it request the exchange the missing records and it writes the necesary files to fix the hole. But some things might happen: + + 1) The exchange responds, but does not provides the missing records. In this case the process will flag this hole for retry later, up to 3 times. After that it will flag + the hole as "permanent". + 2) The exchange call timeouts: the process will continue retrying later. + 3) The exchange returns an unexpected error: The process will retry up to 3 times later under this condition. + 4) The exchange returns neighboring transactions but not the missing ones. The process will retry 3 more times and mark the hole as permanent if the exchange refuses to send the missing trades. + + Since the process is run in an infinite loop but can be restarted any time, the status of its running is recorded in the Status Report file. + + What is the lastFile pointer? + + It is the Datetime since the begining of the hitory of a market that is considered without holes (or only with permanent ones, unfixables). In other words, the process + starts from the begining of the history of a market, and only when its history is signaled as "Complete" by the "Historic Trades" process. From there, the process is advancing + in time, validating the history and fixing holes, and moving the lastFile pointer forward as it gets sure that all the previous trades files are with no holes or unfixable ones. + This pointer will later be used by Indicators bots that depends of these trades, and it will be considered the head of the market. + + */ function start(callBackFunction) { @@ -265,7 +264,7 @@ What is the lastFile pointer? logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); callBackFunction(customOK); return; - } + } } else { logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Trade History is not complete."); @@ -278,7 +277,7 @@ What is the lastFile pointer? callBackFunction(customOK); return; } - + reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Poloniex-Hole-Fixing" + "-" + "dataSet.V1" + "-" + year + "-" + month; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } @@ -377,7 +376,7 @@ What is the lastFile pointer? let fileCheckedCounter = 0; - date = new Date(lastHoleFixingFile.valueOf()); + date = new Date(lastHoleFixingFile.valueOf()); readNextFile(); @@ -421,7 +420,7 @@ What is the lastFile pointer? return; } - /* Lets check if we have reached the end of the month. */ + /* Lets check if we have reached the end of the month. */ if (date.valueOf() > lastMinuteOfMonth.valueOf()) { @@ -458,7 +457,7 @@ What is the lastFile pointer? } dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2); - + filePath = bot.filePathRoot + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath; charlyStorage.getTextFile(filePath, fileName, onNextFileReceived, true); @@ -490,7 +489,7 @@ What is the lastFile pointer? findEndOfHole(); return; - } + } if (err.result === global.DEFAULT_OK_RESPONSE.result) { try { @@ -548,11 +547,11 @@ What is the lastFile pointer? We will need these trades at the end of the process. */ - tradesWithHole = trades; + tradesWithHole = trades; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> checkHolesInFile -> Checking File '" + fileName + "' @ " + filePath + " - " + trades.length + " records in it."); } - if (currentTradeId === FIRST_TRADE_RECORD_ID || currentTradeId === UNKNOWN_TRADE_RECORD_ID) { + if (currentTradeId === FIRST_TRADE_RECORD_ID || currentTradeId === UNKNOWN_TRADE_RECORD_ID) { /* Here we dont know the currentTradeId, so we might take it directly from the file. There will be a problem though if the market starts @@ -714,7 +713,7 @@ What is the lastFile pointer? return; } - callBackFunction(global.DEFAULT_OK_RESPONSE); + callBackFunction(global.DEFAULT_OK_RESPONSE); return; } catch (err) { logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> findEndOfHole -> onStatusReportWritten -> err = " + err.message); @@ -753,10 +752,9 @@ What is the lastFile pointer? findEndOfHole(); return; - } + } - if (err.result === global.DEFAULT_OK_RESPONSE.result) - { + if (err.result === global.DEFAULT_OK_RESPONSE.result) { let trades; try { @@ -771,7 +769,7 @@ What is the lastFile pointer? findEndOfHole(); return; - } + } if (trades.length === 0) { @@ -835,9 +833,7 @@ What is the lastFile pointer? exchangeCallTime = new Date(); - let poloniexApiClient = POLONIEX_CLIENT_MODULE.newPoloniexAPIClient(global.EXCHANGE_KEYS[global.EXCHANGE_NAME].Key, global.EXCHANGE_KEYS[global.EXCHANGE_NAME].Secret); - - poloniexApiClient.API.returnPublicTradeHistory(market.assetA, market.assetB, startTime, endTime, onExchangeCallReturned); + EXCHANGE_API.getPublicTradeHistory(market.assetA, market.assetB, startTime, endTime, onExchangeCallReturned); } catch (err) { logger.write(MODULE_NAME, "[ERROR] start -> getTheTrades -> err = " + err.message); @@ -859,12 +855,7 @@ What is the lastFile pointer? logger.write(MODULE_NAME, "[INFO] start -> onExchangeCallReturned -> Call time recorded = " + timeDifference + " seconds."); } - poloniexApiClient.API.analizeResponse(logger, err, exchangeResponse, callBackFunction, onResponseOk); - - function onResponseOk() { - - tradesReadyToBeSaved(exchangeResponse); - } + tradesReadyToBeSaved(exchangeResponse); } catch (err) { logger.write(MODULE_NAME, "[ERROR] start -> onExchangeCallReturned -> err = " + err.message); @@ -1104,9 +1095,9 @@ What is the lastFile pointer? logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Content written = " + fileContent); } - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Finished with File @ " + market.assetA + "_" + market.assetB); - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Records inserted = " + fileRecordCounter); - logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Path = " + filePath + "/" + fileName + ""); + logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Finished with File @ " + market.assetA + "_" + market.assetB); + logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Records inserted = " + fileRecordCounter); + logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Path = " + filePath + "/" + fileName + ""); controlLoop(); diff --git a/this.bot.config.json b/this.bot.config.json index 934212209..de4f22a8f 100644 --- a/this.bot.config.json +++ b/this.bot.config.json @@ -57,12 +57,12 @@ "allMonths": { "run": "false", "minYear": "2014", - "maxYear": "2018" + "maxYear": "2020" }, "oneMonth": { "run": "true", "year": "2018", - "month": "04" + "month": "01" }, "noTime": { "run": "false" @@ -106,7 +106,7 @@ "allMonths": { "run": "true", "minYear": "2018", - "maxYear": "2018" + "maxYear": "2019" }, "oneMonth": { "run": "false",