From 5ad5544de30dfca12ed28eb5e103f1cb434958c0 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 18 Feb 2018 15:28:33 +0100 Subject: [PATCH] Multi Period Daily process implemented --- .vs/AAOlivia/v15/.suo | Bin 36864 -> 37376 bytes AAOlivia.njsproj | 2 + Multi-Period-Daily/Interval.js | 840 ++++++++++++++++++++++++++++++++ Multi-Period-Market/Interval.js | 2 +- 4 files changed, 843 insertions(+), 1 deletion(-) create mode 100644 Multi-Period-Daily/Interval.js diff --git a/.vs/AAOlivia/v15/.suo b/.vs/AAOlivia/v15/.suo index 7cc25cf48ac06f52df13a86755ffd8b1db61bf16..ba82208a3f1ddc2694be6bc4895382b7f35a257d 100644 GIT binary patch delta 1691 zcmchXZA@EL7{|{!yBBVGDTTI-F@!S5D6Dj)fI-H~ZMmpq1G~i_xJJs`q>Lf28*Kzi z(Wny%xzGHd+Ys*1Ep8u_oA|}(s{3M`FWWT6Op{Ihu=vSE<-=kkJ_jp`@tcV^`SqOV z|D3nyJWo$#kwuo-d^Jsdt0s+60rFX%(U+YO`{4ZN)J6!26hWAU>1(jcW|&ieaAH^t zqW&e^-BK3J!rmHwBDSu0pWFP~j`r5LX2v!H$wjP49+HV{L;e`Oww8_k-_Xd%b_a4V zQh;!OF&q%ga9%Q5_n=}#%HkgGtwQhiZPlpo`8C+qB6Wz2JjlItFHpnxD%&3qF|cBA z@em7fK&%i$Y=q01rF21GWyw^PvdVMNVL|wc>DVSBd@pYEuM%wm*@c+mKG-kp6qC@9 z_?Du9%kYwL2>gNt1kplM@DR7vq;AtI=sJa*M$RC5AZeDWNdeaa8vk_iEnh`_5m`bs zeLMxd9<_ z5v|1J-vcqTAeExVFPX0;zKk6ShU&}^2v>qqQN7|)sN_bKXA*}vV7&A>HM?G~-}UZa zXDNx#r@b&{oT5*_D*hgZv9uS|uuL_eGiL%$(nm%}YTLxm?D~y!(YbTqoG8fo0Vi%0 z#h{LY0`<%C2fLcW#>S-^XLqX83)dg(qwDlqhmO@zF@7ccX9LS9TGq#sXRB7vt< zhT3pcHJy}sqIkA(K~jY9Tje;g&(G>acyaME=JE3Yd6D79>to>y(|pQfXdFVi5g1Ij zS$g8$KUd=|ET{Y7{oH&#Klf&$wu7k_MM8OPb0XX(-Bf&sYT0bX{t$x?nL{6ST&18? zxx@-w^#X*PztY_>wr5H#Mw`K1>y`C{Ze;nF%oO_X$*|jWTk|cdRS7HZ%j~~oTY*?z zf^PKvB&uGSF6kccC91iZs+7@-;E7Hd6w*P0q@RRf&Qp+DiMimF$axz5J>c`&L~bo^ z9<4P?uoN+i>m>j?8(QFcg9V}w)atPYnW-)JS%b%d9nOKik-^~$m}vfjF!DAGwN8pk zN3cJn9(8M82K2~#bdy>xFrsWyWacauA0Qw&-m*=;K2C(T3?FVWrz7t7PaV}u}c)b z>&%6~gab}Qis9qwQMK0zY9s?bji%sR#Gjv8(nYGsm1lmOx@ONa*>BAh-t-mN-TsV+xjsqod*|M@qi1g6Kgf)gWNm`H$x>4U!mEVc$t delta 974 zcmbu7-%C?b9Kd(Z?%bW5+tp1?CvH>IF*m#ZKsKE=^D2vm(W^%fQakMlj}`{Nw-Ayt>? zohWfU>2XjZfscCZUxU{oYb$xuWH1=iMwp>9^Np>~<@ll?*G4xkT z*e-el8%7zvnHzi+xFI0bNI6o4@G>j~c?h#Pd&Giu0HqCaB9+JvMi!&2MffScJg9pS zA5w<|culY(NTqhPWTXq(4TY@DQ`L=DPr=iR@-PxZ+V^_d(uzmfqU{37CguIAz29OW z7{=!X$Itm8V}A$nsvu+>3psC`3cO~%!7`0+iO74j7Cc&wz^0@G5qS~j-%es+-@DCJ z`}=B^u`dv7IPFXy#M)UyH+7!``dHC3P zNv2j*mf3XIO_FmdMC2wPKJEo!s(fAxNKEQw)Iyrk%g45RY7vzWkaZt}jXnjI+)>zY zMzwHU;NN1e)u9EW-X-Len80Gg>jZ93m%^>1b*z2#wei1G=|iG67PBtn*W{g0!(&(< zzYMDrWzd}p!1M8^4tx>9Uxlypu|M{W)S&Pf=(zY^AEqX`wX>rWBI)|#vlz_3xUxUq z%GOV*1hOL@7`t>2 + @@ -28,6 +29,7 @@ + diff --git a/Multi-Period-Daily/Interval.js b/Multi-Period-Daily/Interval.js new file mode 100644 index 000000000..ead80db00 --- /dev/null +++ b/Multi-Period-Daily/Interval.js @@ -0,0 +1,840 @@ +exports.newInterval = function newInterval(BOT, UTILITIES, AZURE_FILE_STORAGE, DEBUG_MODULE, MARKETS_MODULE, POLONIEX_CLIENT_MODULE) { + + let bot = BOT; + + const GMT_SECONDS = ':00.000 GMT+0000'; + const GMT_MILI_SECONDS = '.000 GMT+0000'; + const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; + + const MODULE_NAME = "Interval"; + const LOG_INFO = true; + + const EXCHANGE_NAME = "Poloniex"; + const EXCHANGE_ID = 1; + + const TRADES_FOLDER_NAME = "Trades"; + + const CANDLES_FOLDER_NAME = "Candles"; + const CANDLES_ONE_MIN = "One-Min"; + + const VOLUMES_FOLDER_NAME = "Volumes"; + const VOLUMES_ONE_MIN = "One-Min"; + + const GO_RANDOM = false; + const FORCE_MARKET = 2; // This allows to debug the execution of an specific market. Not intended for production. + + const logger = DEBUG_MODULE.newDebugLog(); + logger.fileName = MODULE_NAME; + logger.bot = bot; + + interval = { + initialize: initialize, + start: start + }; + + let markets; + + let charlyAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot); + let bruceAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot); + let oliviaAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot); + + let utilities = UTILITIES.newUtilities(bot); + + return interval; + + function initialize(yearAssigend, monthAssigned, callBackFunction) { + + try { + + /* IMPORTANT NOTE: + + We are ignoring in this Interval the received Year and Month. This interval is not depending on Year Month since it procecess the whole market at once. + + */ + + logger.fileName = MODULE_NAME; + + const logText = "[INFO] initialize - Entering function 'initialize' "; + console.log(logText); + logger.write(logText); + + charlyAzureFileStorage.initialize("Charly"); + bruceAzureFileStorage.initialize("Bruce"); + oliviaAzureFileStorage.initialize("Olivia"); + + markets = MARKETS_MODULE.newMarkets(bot); + markets.initialize(callBackFunction); + + + } catch (err) { + + const logText = "[ERROR] initialize - ' ERROR : " + err.message; + console.log(logText); + logger.write(logText); + + } + } + +/* + +This process is going to do the following: + +Read the candles and volumes from Bruce and produce a single Index File for each market with daily candles and volumes. + +*/ + + function start(callBackFunction) { + + try { + + if (LOG_INFO === true) { + logger.write("[INFO] Entering function 'start'"); + } + + let nextIntervalExecution = false; // This tell weather the Interval module will be executed again or not. By default it will not unless some hole have been found in the current execution. + let nextIntervalLapse; // With this we can request the next execution wait time. + + let marketQueue; // This is the queue of all markets to be procesesd at each interval. + let market = { // This is the current market being processed after removing it from the queue. + id: 0, + assetA: "", + assetB: "" + }; + + let lastCandleFile; // Datetime of the last file included on the Index File. + let firstTradeFile; // Datetime of the first trade file in the whole market history. + let maxCandleFile; // Datetime of the last file available to be included in the Index File. + + let periods = + '[' + + '[' + 55 * 60 * 1000 + ',' + '"55-min"' + ']' + ',' + + '[' + 50 * 60 * 1000 + ',' + '"50-min"' + ']' + ',' + + '[' + 45 * 60 * 1000 + ',' + '"45-min"' + ']' + ',' + + '[' + 40 * 60 * 1000 + ',' + '"40-min"' + ']' + ',' + + '[' + 35 * 60 * 1000 + ',' + '"35-min"' + ']' + ',' + + '[' + 30 * 60 * 1000 + ',' + '"30-min"' + ']' + ',' + + '[' + 25 * 60 * 1000 + ',' + '"25-min"' + ']' + ',' + + '[' + 20 * 60 * 1000 + ',' + '"20-min"' + ']' + ',' + + '[' + 15 * 60 * 1000 + ',' + '"15-min"' + ']' + ',' + + '[' + 10 * 60 * 1000 + ',' + '"10-min"' + ']' + ',' + + '[' + 05 * 60 * 1000 + ',' + '"05-min"' + ']' + ',' + + '[' + 04 * 60 * 1000 + ',' + '"04-min"' + ']' + ',' + + '[' + 03 * 60 * 1000 + ',' + '"03-min"' + ']' + ',' + + '[' + 02 * 60 * 1000 + ',' + '"02-min"' + ']' + ',' + + '[' + 01 * 60 * 1000 + ',' + '"01-min"' + ']' + ']'; + + const outputPeriods = JSON.parse(periods); + + marketsLoop(); + + /* + + At every run, the process needs to loop through all the markets at this exchange. + The following functions marketsLoop(), openMarket(), closeMarket() and closeAndOpenMarket() controls the serialization of this processing. + + */ + + function marketsLoop() { + + try { + + if (LOG_INFO === true) { + logger.write("[INFO] Entering function 'marketsLoop'"); + } + + markets.getMarketsByExchange(EXCHANGE_ID, onMarketsReady); + + function onMarketsReady(marketsArray) { + + marketQueue = JSON.parse(marketsArray); + + openMarket(); // First execution and entering into the real loop. + + } + } + catch (err) { + const logText = "[ERROR] 'marketsLoop' - ERROR : " + err.message; + logger.write(logText); + } + } + + function openMarket() { + + // To open a Market means picking a new market from the queue. + + try { + + if (LOG_INFO === true) { + logger.write("[INFO] Entering function 'openMarket'"); + } + + + if (marketQueue.length === 0) { + + if (LOG_INFO === true) { + logger.write("[INFO] 'openMarket' - marketQueue.length === 0"); + } + + const logText = "[WARN] We processed all the markets."; + logger.write(logText); + + callBackFunction(nextIntervalExecution, nextIntervalLapse); + + return; + } + + if (GO_RANDOM === true) { + const index = parseInt(Math.random() * (marketQueue.length - 1)); + + market.id = marketQueue[index][0]; + market.assetA = marketQueue[index][1]; + market.assetB = marketQueue[index][2]; + market.status = marketQueue[index][3]; + + marketQueue.splice(index, 1); + } else { + let marketRecord = marketQueue.shift(); + + market.id = marketRecord[0]; + market.assetA = marketRecord[1]; + market.assetB = marketRecord[2]; + market.status = marketRecord[3]; + + if (FORCE_MARKET > 0) { + if (FORCE_MARKET !== market.id) { + closeAndOpenMarket(); + return; + } + } + } + + if (LOG_INFO === true) { + logger.write("[INFO] 'openMarket' - marketQueue.length = " + marketQueue.length); + logger.write("[INFO] 'openMarket' - market sucessfully opened : " + market.assetA + "_" + market.assetB); + } + + if (market.status === markets.ENABLED) { + + getStatusReport(); + + } else { + + logger.write("[INFO] 'openMarket' - market " + market.assetA + "_" + market.assetB + " skipped because its status is not valid. Status = " + market.status); + closeAndOpenMarket(); + return; + + } + + + } + catch (err) { + const logText = "[ERROR] 'openMarket' - ERROR : " + err.message; + logger.write(logText); + closeMarket(); + } + } + + function closeMarket() { + + if (LOG_INFO === true) { + logger.write("[INFO] Entering function 'closeMarket'"); + } + + } + + function closeAndOpenMarket() { + + if (LOG_INFO === true) { + logger.write("[INFO] Entering function 'closeAndOpenMarket'"); + } + + openMarket(); + } + + /* + + The following code executes for each market. + + */ + + function getStatusReport() { + + try { + + let reportFilePath; + let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json" + + getHistoricTrades(); + + function getHistoricTrades() { + + /* + + We need to know where is the begining of the market, since that will help us know where the Index Files should start. + + */ + + reportFilePath = EXCHANGE_NAME + "/Processes/" + "Poloniex-Historic-Trades"; + + charlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); + + function onStatusReportReceived(text) { + + let statusReport; + + try { + + statusReport = JSON.parse(text); + + firstTradeFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS); + + getOneMinDailyCandlesVolumes(); + + } catch (err) { + + const logText = "[INFO] 'getStatusReport' - Failed to read main Historic Trades Status Report for market " + market.assetA + '_' + market.assetB + " . Skipping it. "; + logger.write(logText); + + closeAndOpenMarket(); + } + } + } + + function getOneMinDailyCandlesVolumes() { + + /* We need to discover the maxCandle file, which is the last file with candles we can use as input. */ + + let date = new Date(); + let currentYear = date.getUTCFullYear(); + let currentMonth = utilities.pad(date.getUTCMonth() + 1,2); + + reportFilePath = EXCHANGE_NAME + "/Processes/" + "One-Min-Daily-Candles-Volumes" + "/" + currentYear + "/" + currentMonth; + + bruceAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); + + function onStatusReportReceived(text) { + + let statusReport; + + try { + + statusReport = JSON.parse(text); + + maxCandleFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + "00:00" + GMT_SECONDS); + + getThisProcessReport(); + + } catch (err) { + + /* + + It might happen that the file content is corrupt or it does not exist. In either case we will point our lastCandleFile + to the last day of the previous month. + + */ + + maxCandleFile = new Date(date.valueOf() - ONE_DAY_IN_MILISECONDS); + getThisProcessReport(); + + } + } + } + + function getThisProcessReport() { + + /* If the process run and was interrupted, there should be a status report that allows us to resume execution. */ + + reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process; + + oliviaAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); + + function onStatusReportReceived(text) { + + let statusReport; + + try { + + statusReport = JSON.parse(text); + + lastCandleFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + "00:00" + GMT_SECONDS); + + buildCandles(); + + } catch (err) { + + /* + + It might happen that the file content is corrupt or it does not exist. In either case we will point our lastCandleFile + to the begining of the market. + + */ + + lastCandleFile = new Date(firstTradeFile.getUTCFullYear() + "-" + (firstTradeFile.getUTCMonth() + 1) + "-" + firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS); + + lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well. + + buildCandles(); + + } + } + } + + } + catch (err) { + const logText = "[ERROR] 'getStatusReport' - ERROR : " + err.message; + logger.write(logText); + closeMarket(); + } + } + + function buildCandles() { + + try { + + /* + + Firstly we prepere the arrays that will accumulate all the information for each output file. + + */ + + let outputCandles = []; + let outputVolumes = []; + + for (n = 0; n < outputPeriods.length; n++) { + + const emptyArray1 = []; + const emptyArray2 = []; + + outputCandles.push(emptyArray1); + outputVolumes.push(emptyArray2); + + } + + advanceTime(); + + function advanceTime() { + + lastCandleFile = new Date(lastCandleFile.valueOf() + ONE_DAY_IN_MILISECONDS); + + const logText = "[INFO] New processing time @ " + lastCandleFile.getUTCFullYear() + "/" + (lastCandleFile.getUTCMonth() + 1) + "/" + lastCandleFile.getUTCDate() + "."; + console.log(logText); + logger.write(logText); + + /* Validation that we are not going past the head of the market. */ + + if (lastCandleFile.valueOf() > maxCandleFile.valueOf()) { + + nextIntervalExecution = true; // we request a new interval execution. + + const logText = "[INFO] 'buildCandles' - Head of the market found @ " + lastCandleFile.getUTCFullYear() + "/" + (lastCandleFile.getUTCMonth() + 1) + "/" + lastCandleFile.getUTCDate() + "."; + logger.write(logText); + + closeAndOpenMarket(); + + return; + + } + + periodsLoop(); + + } + + function periodsLoop() { + + /* + + We will iterate through all posible periods. + + */ + + let n = 0 // loop Variable representing each possible period as defined at the periods array. + + loopBody(); + + function loopBody() { + + const outputPeriod = outputPeriods[n][0]; + const folderName = outputPeriods[n][1]; + + nextCandleFile(); + + function nextCandleFile() { + + let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2); + let fileName = market.assetA + '_' + market.assetB + ".json" + let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath; + + bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(text) { + + let candlesFile; + + try { + + candlesFile = JSON.parse(text); + + } catch (err) { + + const logText = "[ERR] 'buildCandles' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; + logger.write(logText); + + closeAndOpenMarket(); + + nextIntervalExecution = true; // we request a new interval execution. + nextIntervalLapse = 30000; + + return; + } + + const inputCandlesPerdiod = 60 * 1000; // 1 min + const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs + + let totalOutputCandles = inputFilePeriod / outputPeriod; + let beginingOutputTime = lastCandleFile.valueOf(); + + for (let i = 0; i < totalOutputCandles; i++) { + + let outputCandle = { + open: 0, + close: 0, + min: 0, + max: 0, + begin: 0, + end: 0 + }; + + let saveCandle = false; + + outputCandle.begin = beginingOutputTime + i * outputPeriod; + outputCandle.end = beginingOutputTime + (i + 1) * outputPeriod - 1; + + for (let j = 0; j < candlesFile.length; j++) { + + let candle = { + open: candlesFile[j][2], + close: candlesFile[j][3], + min: candlesFile[j][0], + max: candlesFile[j][1], + begin: candlesFile[j][4], + end: candlesFile[j][5] + }; + + /* Here we discard all the candles out of range. */ + + if (candle.begin >= outputCandle.begin && candle.end <= outputCandle.end) { + + if (saveCandle === false) { // this will set the value only once. + + outputCandle.open = candle.open; + outputCandle.min = candle.min; + outputCandle.max = candle.max; + + } + + saveCandle = true; + + outputCandle.close = candle.close; // only the last one will be saved + + if (candle.min < outputCandle.min) { + + outputCandle.min = candle.min; + + } + + if (candle.max > outputCandle.max) { + + outputCandle.max = candle.max; + + } + } + } + + if (saveCandle === true) { // then we have a valid candle, otherwise it means there were no candles to fill this one in its time range. + + outputCandles[n].push(outputCandle); + + } + } + + nextVolumeFile(); + + } + } + + function nextVolumeFile() { + + let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2); + let fileName = market.assetA + '_' + market.assetB + ".json" + let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath; + + bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(text) { + + let volumesFile; + + try { + + volumesFile = JSON.parse(text); + + } catch (err) { + + const logText = "[ERR] 'buildCandles' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; + logger.write(logText); + + closeAndOpenMarket(); + + return; + } + + const inputVolumesPerdiod = 60 * 1000; // 1 min + const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs + + let totalOutputVolumes = inputFilePeriod / outputPeriod; // this should be 2 in this case. + let beginingOutputTime = lastCandleFile.valueOf(); + + for (let i = 0; i < totalOutputVolumes; i++) { + + let outputVolume = { + buy: 0, + sell: 0, + begin: 0, + end: 0 + }; + + let saveVolume = false; + + outputVolume.begin = beginingOutputTime + i * outputPeriod; + outputVolume.end = beginingOutputTime + (i + 1) * outputPeriod - 1; + + for (let j = 0; j < volumesFile.length; j++) { + + let volume = { + buy: volumesFile[j][0], + sell: volumesFile[j][1], + begin: volumesFile[j][2], + end: volumesFile[j][3] + }; + + /* Here we discard all the Volumes out of range. */ + + if (volume.begin >= outputVolume.begin && volume.end <= outputVolume.end) { + + saveVolume = true; + + outputVolume.buy = outputVolume.buy + volume.buy; + outputVolume.sell = outputVolume.sell + volume.sell; + + } + } + + if (saveVolume === true) { + + outputVolumes[n].push(outputVolume); + + } + } + + writeFiles(outputCandles[n], outputVolumes[n], folderName, controlLoop); + + } + } + } + + function controlLoop() { + + n++; + + if (n < outputPeriods.length) { + + loopBody(); + + } else { + + writeStatusReport(lastCandleFile, advanceTime); + + } + } + } + } + + catch (err) { + const logText = "[ERROR] 'buildCandles' - ERROR : " + err.message; + logger.write(logText); + closeMarket(); + } + + } + + function writeFiles(candles, volumes, folderName, callBack) { + + /* + + Here we will write the contents of the Candles and Volumens files. + + */ + + try { + + writeCandles(); + + function writeCandles() { + + let separator = ""; + let fileRecordCounter = 0; + + let fileContent = ""; + + for (i = 0; i < candles.length; i++) { + + let candle = candles[i]; + + fileContent = fileContent + separator + '[' + candles[i].min + "," + candles[i].max + "," + candles[i].open + "," + candles[i].close + "," + candles[i].begin + "," + candles[i].end + "]"; + + if (separator === "") { separator = ","; } + + fileRecordCounter++; + + } + + fileContent = "[" + fileContent + "]"; + + let fileName = '' + market.assetA + '_' + market.assetB + '.json'; + + let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2); + + let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + "/" + bot.process + "/" + folderName + "/" + dateForPath; + + utilities.createFolderIfNeeded(filePath, oliviaAzureFileStorage, onFolderCreated); + + function onFolderCreated() { + + oliviaAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + + function onFileCreated() { + + const logText = "[WARN] Finished with File @ " + market.assetA + "_" + market.assetB + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName + ""; + console.log(logText); + logger.write(logText); + + writeVolumes(); + } + } + + } + + function writeVolumes() { + + let separator = ""; + let fileRecordCounter = 0; + + let fileContent = ""; + + for (i = 0; i < volumes.length; i++) { + + let candle = volumes[i]; + + fileContent = fileContent + separator + '[' + volumes[i].buy + "," + volumes[i].sell + "," + volumes[i].begin + "," + volumes[i].end + "]"; + + if (separator === "") { separator = ","; } + + fileRecordCounter++; + + } + + fileContent = "[" + fileContent + "]"; + + let fileName = '' + market.assetA + '_' + market.assetB + '.json'; + + let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2); + + let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + "/" + bot.process + "/" + folderName + "/" + dateForPath; + + utilities.createFolderIfNeeded(filePath, oliviaAzureFileStorage, onFolderCreated); + + function onFolderCreated() { + + oliviaAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); + + function onFileCreated() { + + const logText = "[WARN] Finished with File @ " + market.assetA + "_" + market.assetB + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName + ""; + console.log(logText); + logger.write(logText); + + callBack(); + } + } + } + } + + catch (err) { + const logText = "[ERROR] 'writeFiles' - ERROR : " + err.message; + logger.write(logText); + closeMarket(); + } + } + + function writeStatusReport(lastFileDate, callBack) { + + if (LOG_INFO === true) { + logger.write("[INFO] Entering function 'writeStatusReport'"); + } + + try { + + let reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process; + + utilities.createFolderIfNeeded(reportFilePath, oliviaAzureFileStorage, onFolderCreated); + + function onFolderCreated() { + + try { + + let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json"; + + let report = { + lastFile: { + year: lastFileDate.getUTCFullYear(), + month: (lastFileDate.getUTCMonth() + 1), + days: lastFileDate.getUTCDate() + } + }; + + let fileContent = JSON.stringify(report); + + oliviaAzureFileStorage.createTextFile(reportFilePath, fileName, fileContent + '\n', onFileCreated); + + function onFileCreated() { + + if (LOG_INFO === true) { + logger.write("[INFO] 'writeStatusReport' - Content written: " + fileContent); + } + + callBack(); + } + } + catch (err) { + const logText = "[ERROR] 'writeStatusReport - onFolderCreated' - ERROR : " + err.message; + logger.write(logText); + closeMarket(); + } + } + + } + catch (err) { + const logText = "[ERROR] 'writeStatusReport' - ERROR : " + err.message; + logger.write(logText); + closeMarket(); + } + + } + + } + catch (err) { + const logText = "[ERROR] 'Start' - ERROR : " + err.message; + logger.write(logText); + } + } +}; diff --git a/Multi-Period-Market/Interval.js b/Multi-Period-Market/Interval.js index bd031526e..ee8260040 100644 --- a/Multi-Period-Market/Interval.js +++ b/Multi-Period-Market/Interval.js @@ -311,7 +311,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each function getOneMinDailyCandlesVolumes() { - /* If the process run and was interrupted, there should be a status report that allows us to resume execution. */ + /* We need to discover the maxCandle file, which is the last file with candles we can use as input. */ let date = new Date(); let currentYear = date.getUTCFullYear();