From 5825195fa5a86fa09c2aae6e3e122afbfb57ce32 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 10 Feb 2018 15:25:56 +0100 Subject: [PATCH] All Interval module implemented --- .vs/AAOlivia/v15/.suo | Bin 20992 -> 18432 bytes Index-Files/Interval.js | 488 ++++++++++++++++++++-------------------- 2 files changed, 248 insertions(+), 240 deletions(-) diff --git a/.vs/AAOlivia/v15/.suo b/.vs/AAOlivia/v15/.suo index 2edce14cd97169f8073fef2d7e28fdebaacfe6dc..1c704d14248a4409b1b4418d1281903558734133 100644 GIT binary patch delta 115 zcmZoz!q_l@ae{#?0}Kc-FbD$Kj6nSF|Ns9WVaCah9NrrfG8iW@<**hwPh+XxT*&l? ynPn6Ix2nyBY?l=`yNEO}Za$)UUsBQ;qyY-Lfph^7^8hjPaae{#?69W_oFfgzHnT$aE@Bjb*AW`PYjvU?_6EYYlG3Brdlo{Sl++4`? zhneNgvt`Mf3)wC!@^AuG0TqM6 maxCandleFile.valueOf()) { - date = new Date(firstTradeFile.valueOf() - 60 * 1000); + const logText = "[INFO] 'buildCandles' - Head of the market found at " + filePath + " for market " + market.assetA + '_' + market.assetB + ". "; + logger.write(logText); + + closeAndOpenMarket(); + + return; } - if (lastTradeFile !== undefined) { + nextCandleFile(); - date = new Date(lastTradeFile.valueOf()); + } - } + function nextCandleFile() { - let candles = []; - let volumes = []; + let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2); + let fileName = market.assetA + '_' + market.assetB + ".json" + let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath; - if (previousCandles !== undefined && canAddPrevious === true) { + bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true); - for (let i = 0; i < previousCandles.length; i++) { + function onFileReceived(text) { + + let candlesFile; + + try { + + candlesFile = JSON.parse(text); + + } catch (err) { + + const logText = "[ERR] 'buildCandles' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; + logger.write(logText); + + closeAndOpenMarket(); + + return; + } + + let indexCandle = { + open: 0, + close: 0, + min: 0, + max: 0, + begin: 0, + end: 0 + }; + + if (candlesFile.length > 0) { + + indexCandle.open = candlesFile[0][2]; + indexCandle.close = candlesFile[candlesFile.length - 1][3]; + + indexCandle.min = candlesFile[0][0]; + indexCandle.max = candlesFile[0][1]; + + indexCandle.begin = candlesFile[0][4]; + indexCandle.end = candlesFile[candlesFile.length - 1][5]; + + } else { + + const logText = "[ERR] 'buildCandles' - No candles found at file " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; + logger.write(logText); + + closeAndOpenMarket(); + + return; + + } + + for (let i = 0; i < candlesFile.length; i++) { let candle = { - open: previousCandles[i][2], - close: previousCandles[i][3], - min: previousCandles[i][0], - max: previousCandles[i][1], - begin: previousCandles[i][4], - end: previousCandles[i][5] + open: candlesFile[i][2], + close: candlesFile[i][3], + min: candlesFile[i][0], + max: candlesFile[i][1], + begin: candlesFile[i][4], + end: candlesFile[i][5] }; - candles.push(candle); + if (candle.min < indexCandle.min) { + + indexCandle.min = candle.min; + + } + + if (candle.max > indexCandle.max) { + + indexCandle.max = candle.max; + + } + } + indexCandles.push(indexCandle); + } - if (previousVolumes !== undefined && canAddPrevious === true) { + nextVolumeFile(); - for (let i = 0; i < previousVolumes.length; i++) { + } + + function nextVolumeFile() { + + let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2); + let fileName = market.assetA + '_' + market.assetB + ".json" + let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath; + + bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(text) { + + let volumesFile; + + try { + + volumesFile = JSON.parse(text); + + } catch (err) { + + const logText = "[ERR] 'buildCandles' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; + logger.write(logText); + + closeAndOpenMarket(); + + return; + } + + let indexVolume = { + buy: 0, + sell: 0, + begin: 0, + end: 0 + }; + + if (volumesFile.length > 0) { + + indexVolume.begin = volumesFile[0][2]; + indexVolume.end = volumesFile[volumesFile.length - 1][3]; + + } else { + + const logText = "[ERR] 'buildCandles' - No volumes found at file " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; + logger.write(logText); + + closeAndOpenMarket(); + + return; + + } + + for (let i = 0; i < volumesFile.length; i++) { let volume = { - begin: previousVolumes[i][2], - end: previousVolumes[i][3], - buy: previousVolumes[i][0], - sell: previousVolumes[i][1] + buy: volumesFile[i][0], + sell: volumesFile[i][1], + begin: volumesFile[i][2], + end: volumesFile[i][3] }; - volumes.push(volume); + indexVolume.buy = indexVolume.buy + volume.buy; + indexVolume.sell = indexVolume.sell + volume.sell; + } + indexVolumes.push(indexVolume); + } - canAddPrevious = false; // We add them only onece. + writeFiles(lastCandleFile, indexCandles, indexVolumes, advanceTime); - nextDate(); - - function nextDate() { - - date = new Date(date.valueOf() + 60 * 1000); - - /* Check if we are outside the current Day / File */ - - if (date.getUTCDate() !== lastCandleFile.getUTCDate()) { - - writeFiles(lastCandleFile, candles, volumes, true, onFilesWritten); - - return; - - function onFilesWritten() { - - nextCandleFile(); - - } - - } - - /* Check if we are outside the currrent Month */ - - if (date.getUTCMonth() + 1 !== parseInt(month)) { - - lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); - - writeStatusReport(lastCandleFile, lastTradeFile, lastCandleClose, true, true, onStatusReportWritten); - - return; - - function onStatusReportWritten() { - - const logText = "[ERR] 'buildCandles' - Finishing processing the whole month for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; - logger.write(logText); - - closeAndOpenMarket(); - - } - } - - /* Check if we have past the most recent hole fixed file */ - - if (date.valueOf() > lastFileWithoutHoles.valueOf()) { - - writeFiles(lastCandleFile, candles, volumes, false, onFilesWritten); - - return; - - function onFilesWritten() { - - nextIntervalExecution = true; - - const logText = "[ERR] 'buildCandles' - Head of the market reached for market " + market.assetA + '_' + market.assetB + " . "; - logger.write(logText); - - closeAndOpenMarket(); - - } - } - - readTrades(); - } - - function readTrades() { - - lastTradeFile = new Date(date.valueOf()); - - let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2); - let fileName = market.assetA + '_' + market.assetB + ".json" - let filePath = EXCHANGE_NAME + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath; - - charlyAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true); - - function onFileReceived(text) { - - let tradesFile; - - try { - - let candle = { - open: lastCandleClose, - close: lastCandleClose, - min: lastCandleClose, - max: lastCandleClose, - begin: date.valueOf(), - end: date.valueOf() + 60 * 1000 - 1 - }; - - let volume = { - begin: date.valueOf(), - end: date.valueOf() + 60 * 1000 - 1, - buy: 0, - sell: 0 - }; - - tradesFile = JSON.parse(text); - - let tradesCount = utilities.pad(tradesFile.length, 5); - - const logText = "[INFO] 'buildCandles' - " + tradesCount +" trades found at " + filePath + " for market " + market.assetA + '_' + market.assetB + ". "; - logger.write(logText); - console.log(logText); - - if (tradesFile.length > 0) { - - /* Candle open and close Calculations */ - - candle.open = tradesFile[0][2]; - candle.close = tradesFile[tradesFile.length - 1][2]; - - lastCandleClose = candle.close; - - } - - for (let i = 0; i < tradesFile.length; i++) { - - const trade = { - id: tradesFile[i][0], - type: tradesFile[i][1], - rate: tradesFile[i][2], - amountA: tradesFile[i][3], - amountB: tradesFile[i][4], - seconds: tradesFile[i][5] - }; - - /* Candle min and max Calculations */ - - if (trade.rate < candle.min) { - - candle.min = trade.rate; - - } - - if (trade.rate > candle.max) { - - candle.max = trade.rate; - - } - - /* Volume Calculations */ - - if (trade.type === "sell") { - volume.sell = volume.sell + trade.amountA; - } else { - volume.buy = volume.buy + trade.amountA; - } - - } - - candles.push(candle); - - volumes.push(volume); - - nextDate(); - - } catch (err) { - - const logText = "[ERR] 'buildCandles' - Empty or corrupt trade file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; - logger.write(logText); - - closeAndOpenMarket(); - } - } - } } + } catch (err) { @@ -694,11 +717,11 @@ Read the candles and volumes from Bruce and produce a single Index File for each } - function writeFiles(date, candles, volumes, isFileComplete, callBack) { + function writeFiles(date, candles, volumes, callBack) { /* - Here we will write the contents of the Candles and Volumens files. If the File is declared as complete, we will also write the status report. + Here we will write the contents of the Candles and Volumens files. */ @@ -729,15 +752,13 @@ Read the candles and volumes from Bruce and produce a single Index File for each let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2); + let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME; - let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath; - - utilities.createFolderIfNeeded(filePath, bruceAzureFileStorage, onFolderCreated); + utilities.createFolderIfNeeded(filePath, oliviaAzureFileStorage, onFolderCreated); function onFolderCreated() { - bruceAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + oliviaAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); function onFileCreated() { @@ -774,15 +795,13 @@ Read the candles and volumes from Bruce and produce a single Index File for each let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2); + let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME; - let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath; - - utilities.createFolderIfNeeded(filePath, bruceAzureFileStorage, onFolderCreated); + utilities.createFolderIfNeeded(filePath, oliviaAzureFileStorage, onFolderCreated); function onFolderCreated() { - bruceAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + oliviaAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); function onFileCreated() { @@ -797,7 +816,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each function writeReport() { - writeStatusReport(date, lastTradeFile, lastCandleClose, isFileComplete, false, onStatusReportWritten); + writeStatusReport(date, onStatusReportWritten); function onStatusReportWritten() { @@ -814,8 +833,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each } } - function writeStatusReport(lastFileDate, lastTradeFile, candleClose, isFileComplete, isMonthComplete, callBack) { - + function writeStatusReport(lastFileDate, callBack) { if (LOG_INFO === true) { logger.write("[INFO] Entering function 'writeStatusReport'"); @@ -825,7 +843,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each let reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process + "/" + year + "/" + month; - utilities.createFolderIfNeeded(reportFilePath, bruceAzureFileStorage, onFolderCreated); + utilities.createFolderIfNeeded(reportFilePath, oliviaAzureFileStorage, onFolderCreated); function onFolderCreated() { @@ -838,22 +856,12 @@ Read the candles and volumes from Bruce and produce a single Index File for each year: lastFileDate.getUTCFullYear(), month: (lastFileDate.getUTCMonth() + 1), days: lastFileDate.getUTCDate() - }, - lastTradeFile: { - year: lastTradeFile.getUTCFullYear(), - month: (lastTradeFile.getUTCMonth() + 1), - days: lastTradeFile.getUTCDate(), - hours: lastTradeFile.getUTCHours(), - minutes: lastTradeFile.getUTCMinutes() - }, - candleClose: candleClose, - monthCompleted: isMonthComplete, - fileComplete: isFileComplete + } }; let fileContent = JSON.stringify(report); - bruceAzureFileStorage.createTextFile(reportFilePath, fileName, fileContent + '\n', onFileCreated); + oliviaAzureFileStorage.createTextFile(reportFilePath, fileName, fileContent + '\n', onFileCreated); function onFileCreated() {