From 52cffcd4d1ca9eeb6ad8edc6f10732d8537e8f94 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 16 Mar 2018 12:13:44 +0100 Subject: [PATCH] getTrades ready --- .vs/AACharly-Extraction-Bot/v15/.suo | Bin 22016 -> 23040 bytes Poloniex-Live-Trades/User.bot.js | 310 ++++++--------------------- 2 files changed, 68 insertions(+), 242 deletions(-) diff --git a/.vs/AACharly-Extraction-Bot/v15/.suo b/.vs/AACharly-Extraction-Bot/v15/.suo index 3bddabaa78bc4c6ecb72fcd3dc70417c42f227d0..e570a2dc34acaa8edea9918ca937062d88d0bd20 100644 GIT binary patch delta 522 zcmZoz!`QHeae{#?69W_oFfgzHnT$aE@Bjb*AW`PYjvU?_6EYYlG3BrZT&fZk-CW2N z%*X?j{twazq9-#-s!aaE9KkCJ6c7VqNgx*99LXZg#Hz68w-!(fTM#2JBM(rR6^LPa zHh*FFXXRz!1`6>Z3zzVQuyF7qiA???cz&|8Q0!&}VHL*BE_y7Cn>|FU1tD6+Q55oo zaDc3VTT#M$0peH=6j6SEpnLd%LW)SjlfMXraBw0ElnC}r5^>=HIhq|wd~=BiNWUOZ zND4`q6(qnh`KO>i2Q!lJ;j(-3eCLME_uR5L@yZ7%u`UW? F0RYVwqIm!S delta 502 zcmZqJ!q~8eae{#?0}Kc-FbD$Kj6nSF|Ns9WVaCah9NrrfG8iW@<**)*?-A$ST*ws6 z$ioAa;{sx!vj3AWN~lc!!W_XX1{4qlVqqYb+#Jaw%*5&tn;giqxsWZ0k(U9ahZTrn zdNzMy_h;n=N-!|+Aq$uAhOkUNp=~iq=7D#V zEr18;4F+Z;wUbMDuW*12fLT1bMB~cj6HGdjOZYGFC_p6;l3F1g9LTD_2n6te+|7wB zSRxp}1JVL=JzVP-L4Tl&V<#`rTg=NK4wM0f2wb*Azh{$(2P20xilA-?ha{4~<}acl zOdLW;B9k4gSTt=EKU@8C!ie2 Entering function."); } + + let market = global.MARKET; + + let currentDate; // This will hold the current datetime of each execution. + let previousMinute; // This hold the current time minus 60 seconds. + let marketQueue; // This is the queue of all markets to be procesesd at each interval. let dateForPathA; let dateForPathB; @@ -114,206 +114,93 @@ Array of records with this information: let exchangeCallTime; - marketsLoop(); + createFolders(); - /* - - At every run, the process needs to loop through all the markets at this exchange. - The problem is that we can not fire all the requests at once to the exchange or we would get banned. - For that reason we create a queue with the markets to process and we do one at the time. - The following functions marketsLoop(), openMarket(), closeMarket() and closeAndOpenMarket() controls the serialization of this processing. - - */ - - function marketsLoop() { + function createFolders() { try { - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'marketsLoop'"); - } + if (FULL_LOG === true) { logger.write("[INFO] start -> createFolders -> Entering function."); } - if (FORCED_INITIAL_TIME !== undefined) { + currentDate = new Date(); + + previousMinute = new Date(currentDate.valueOf() - 60000); - currentDate = new Date(FORCED_INITIAL_TIME); + dateForPathA = currentDate.getUTCFullYear() + '/' + utilities.pad(currentDate.getUTCMonth() + 1, 2) + '/' + utilities.pad(currentDate.getUTCDate(), 2) + '/' + utilities.pad(currentDate.getUTCHours(), 2) + '/' + utilities.pad(currentDate.getUTCMinutes(), 2); + dateForPathB = previousMinute.getUTCFullYear() + '/' + utilities.pad(previousMinute.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousMinute.getUTCDate(), 2) + '/' + utilities.pad(previousMinute.getUTCHours(), 2) + '/' + utilities.pad(previousMinute.getUTCMinutes(), 2); - } else { + filePathA = EXCHANGE_NAME + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPathA; + filePathB = EXCHANGE_NAME + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPathB; - currentDate = new Date(); + utilities.createFolderIfNeeded(filePathA, charlyFileStorage, onFolderACreated); - } + function onFolderACreated(err) { - markets.getMarketsByExchange(EXCHANGE_ID, onMarketsReady); + if (FULL_LOG === true) { logger.write("[INFO] start -> createFolders -> onFolderACreated -> Entering function."); } - function onMarketsReady(marketsArray) { - - marketQueue = JSON.parse(marketsArray); - - /* - - First thing to do is to create the folders where all the files are going to be placed. This is done only once per market. - - */ - - previousMinute = new Date(currentDate.valueOf() - 60000); - - dateForPathA = currentDate.getUTCFullYear() + '/' + utilities.pad(currentDate.getUTCMonth() + 1, 2) + '/' + utilities.pad(currentDate.getUTCDate(), 2) + '/' + utilities.pad(currentDate.getUTCHours(), 2) + '/' + utilities.pad(currentDate.getUTCMinutes(), 2); - dateForPathB = previousMinute.getUTCFullYear() + '/' + utilities.pad(previousMinute.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousMinute.getUTCDate(), 2) + '/' + utilities.pad(previousMinute.getUTCHours(), 2) + '/' + utilities.pad(previousMinute.getUTCMinutes(), 2); - - filePathA = EXCHANGE_NAME + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPathA; - filePathB = EXCHANGE_NAME + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPathB; - - utilities.createFolderIfNeeded(filePathA, charlyFileStorage, onFolderACreated); - - function onFolderACreated() { - - utilities.createFolderIfNeeded(filePathB, charlyFileStorage, onFolderBCreated); - - function onFolderBCreated() { - - utilities.createFolderIfNeeded(reportFilePath, charlyFileStorage, onFolderCreated); - - function onFolderCreated() { - - openMarket(); // First execution and entering into the real loop. - } - } - } - } - } - catch (err) { - const logText = "[ERROR] 'marketsLoop' - ERROR : " + err.message; - logger.write(logText); - } - } - - function openMarket() { - - // To open a Market means picking a new market from the queue. - - try { - - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'openMarket'"); - } - - - if (marketQueue.length === 0) { - - if (LOG_INFO === true) { - logger.write("[INFO] 'openMarket' - marketQueue.length === 0"); + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write("[ERROR] start -> createFolders -> onFolderACreated -> err = " + err.message); + callBackFunction(err); + return; } - const logText = "[WARN] We processed all the markets."; - logger.write(logText); - - closeMarket(); - - return; + utilities.createFolderIfNeeded(filePathB, charlyFileStorage, onFolderBCreated); } - if (GO_RANDOM === true) { - const index = parseInt(Math.random() * (marketQueue.length - 1)); + function onFolderBCreated(err) { - market.id = marketQueue[index][0]; - market.assetA = marketQueue[index][1]; - market.assetB = marketQueue[index][2]; - market.status = marketQueue[index][3]; + if (FULL_LOG === true) { logger.write("[INFO] start -> createFolders -> onFolderBCreated -> Entering function."); } - marketQueue.splice(index, 1); - } else { - let marketRecord = marketQueue.shift(); - - market.id = marketRecord[0]; - market.assetA = marketRecord[1]; - market.assetB = marketRecord[2]; - market.status = marketRecord[3]; - - if (FORCE_MARKET > 0) { - if (FORCE_MARKET !== market.id) { - closeAndOpenMarket(); - return; - } + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write("[ERROR] start -> createFolders -> onFolderBCreated -> err = " + err.message); + callBackFunction(err); + return; } + + utilities.createFolderIfNeeded(reportFilePath, charlyFileStorage, onFolderCreated); } - if (LOG_INFO === true) { - logger.write("[INFO] 'openMarket' - marketQueue.length = " + marketQueue.length); - logger.write("[INFO] 'openMarket' - market sucessfully opened : " + market.assetA + "_" + market.assetB); + function onFolderCreated(err) { + + if (FULL_LOG === true) { logger.write("[INFO] start -> createFolders -> onFolderCreated -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write("[ERROR] start -> createFolders -> onFolderCreated -> err = " + err.message); + callBackFunction(err); + return; + } + getTheTrades(); } - if (market.status === markets.ENABLED) { - - getTheTrades(); - - } else { - - logger.write("[INFO] 'openMarket' - market " + market.assetA + "_" + market.assetB + " skipped because its status is not valid. Status = " + market.status); - closeAndOpenMarket(); - return; - - } - - - } - catch (err) { - const logText = "[ERROR] 'openMarket' - ERROR : " + err.message; - logger.write(logText); - closeMarket(); + } catch (err) { + logger.write("[ERROR] start -> createFolders -> err = " + err.message); + callBack(global.DEFAULT_FAIL_RESPONSE); } } - function closeMarket() { - - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'closeMarket'"); - } - - } - - function closeAndOpenMarket() { - - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'closeAndOpenMarket'"); - } - - openMarket(); - } - - - /* - - The following code executes for each market, getting the trades from the exchange and saving them in files at the cloud storage. - - */ - function getTheTrades() { try { - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'getTheTrades'"); - } + if (FULL_LOG === true) { logger.write("[INFO] start -> getTheTrades -> Entering function."); } const datetime = parseInt(currentDate.valueOf() / 1000); - /* We request to the Exchange API some more records than needed, anyway we will discard records out of the range we need. To do this we substract 120 seconds and add 10 - seconds to the already calculated UNIX timestamps. */ + /* + We request to the Exchange API some more records than needed, anyway we will discard records out of the range we need. + To do this we substract 120 seconds and add 10 seconds to the already calculated current date. + */ const startTime = datetime - 120; const endTime = datetime + 10; exchangeCallTime = new Date(); - let poloniexApiClient = new POLONIEX_CLIENT_MODULE(); - poloniexApiClient.returnPublicTradeHistory(market.assetA, market.assetB, startTime, endTime, onExchangeCallReturned); - } - catch (err) { - const logText = "[ERROR] 'getTheTrades' - ERROR : " + err.message; - logger.write(logText); - closeMarket(); + } catch (err) { + logger.write("[ERROR] start -> getTheTrades -> err = " + err.message); + callBack(global.DEFAULT_FAIL_RESPONSE); } } @@ -321,85 +208,26 @@ Array of records with this information: try { - if (LOG_INFO === true) { + if (FULL_LOG === true) { logger.write("[INFO] start -> onExchangeCallReturned -> Entering function."); } + + if (FULL_LOG === true) { let exchangeResponseTime = new Date(); let timeDifference = (exchangeResponseTime.valueOf() - exchangeCallTime.valueOf()) / 1000; - logger.write("[INFO] Entering function 'onExchangeCallReturned' - Call time recorded = " + timeDifference + " seconds."); + logger.write("[INFO] start -> onExchangeCallReturned -> Call time recorded = " + timeDifference + " seconds."); } - if (err || tradesRequested.error !== undefined) { - try { + poloniexApiClient.analizeResponse(logger, err, exchangeResponse, callBackFunction, onResponseOk); - if (err.message.indexOf("ETIMEDOUT") > 0) { - - const logText = "[WARN] onExchangeCallReturned - Timeout reached while trying to access the Exchange API. Trying again. : ERROR = " + err.message; - logger.write(logText); - - /* We try to reconnect to the exchange and fetch the data again. */ - - getTheTrades(); - return; - - } else { - - if (err.message.indexOf("ECONNRESET") > 0) { - - const logText = "[WARN] onExchangeCallReturned - The exchange reseted the connection. Trying again. : ERROR = " + err.message; - logger.write(logText); - - /* We try to reconnect to the exchange and fetch the data again. */ - - getTheTrades(); - return; - - } else { - - - const logText = "[ERROR] onExchangeCallReturned - Unexpected error trying to contact the Exchange. Giving up. : ERROR = " + err.message; - logger.write(logText); - closeMarket(); - return; - } - } - - } catch (err) { - const logText = "[ERROR] onExchangeCallReturned : ERROR : tradesRequested.error = " + tradesRequested.error; - logger.write(logText); - - if (tradesRequested.error === "Invalid currency pair.") { - - markets.disableMarket(EXCHANGE_ID, market.id, onMarketDeactivated) - - function onMarketDeactivated() { - - logger.write("[INFO] Market " + market.assetA + "_" + market.assetB + " deactivated. Id = " + market.id); - - closeAndOpenMarket(); - return; - - } - - } - - closeMarket(); - return; - } - - closeMarket(); - return; - - } else { + function onResponseOk() { tradesReadyToBeSaved(tradesRequested); } - } - catch (err) { - const logText = "[ERROR] 'onExchangeCallReturned' - ERROR : " + err.message; - logger.write(logText); - closeMarket(); - } + } catch (err) { + logger.write("[ERROR] start -> onExchangeCallReturned -> err = " + err.message); + callBack(global.DEFAULT_FAIL_RESPONSE); + } } function tradesReadyToBeSaved(tradesRequested) { @@ -665,11 +493,9 @@ Array of records with this information: } - - } - catch (err) { - const logText = "[ERROR] 'Start' - ERROR : " + err.message; - logger.write(logText); + } catch (err) { + logger.write("[ERROR] start -> err = " + err.message); + callBack(global.DEFAULT_FAIL_RESPONSE); } } };