diff --git a/.vs/AABruce/v15/.suo b/.vs/AABruce/v15/.suo index 6cc17f5d0..bb898aa2a 100644 Binary files a/.vs/AABruce/v15/.suo and b/.vs/AABruce/v15/.suo differ diff --git a/One-Min-Daily-Candles/Interval.js b/One-Min-Daily-Candles/Interval.js index f01bd7ee6..e455331c7 100644 --- a/One-Min-Daily-Candles/Interval.js +++ b/One-Min-Daily-Candles/Interval.js @@ -4,6 +4,7 @@ const ROOT_DIR = '../'; const GMT_SECONDS = ':00.000 GMT+0000'; const GMT_MILI_SECONDS = '.000 GMT+0000'; + const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; const MODULE_NAME = "Interval"; const LOG_INFO = true; @@ -77,38 +78,7 @@ This process is going to do the following: -It will iterate through all the markets and will try to find "holes" on the transaction history and fix them by retrieving the missing transactions from the exchange. - -What are holes? - - Holes are missing transactions in the history of the market and the reason there can be holes are the following: - - 1) The "Lives Trades" process can be stopped and later started again. The process will only retrieve live trades, leaving a "hole" between it was stopped and restarted. - 2) The "Historic Trades" process might have received a "hole" of transactions directly from the exchange. - 3) The "Historic Trades" process failed to write some files to the storage and now they are missing, together with the transaction in them. - 4) The "Historic Trades" process wrote some files to the storage with corrupted content. - 5) The "Historic Trades" process wrote files with transaction not properly ordered by Trade Id (sometimes the exchange sent this wrong). - -How does the process work? - - Transactions have a unique id generated by the exchange. It is a secuencial number. This process scans the transaction history looking for missing ids. When it finds - some group of missing ids, it request the exchange the missing records and it writes the necesary files to fix the hole. But some things might happen: - - 1) The exchange responds, but does not provides the missing records. In this case the process will flag this hole for retry later, up to 3 times. After that it will flag - the hole as "permanent". - 2) The exchange call timeouts: the process will continue retrying later. - 3) The exchange returns an unexpected error: The process will retry up to 3 times later under this condition. - 4) The exchange returns neighboring transactions but not the missing ones. The process will retry 3 more times and mark the hole as permanent if the exchange refuses to send the missing trades. - -Since the process is run in an infinite loop but can be restarted any time, the status of is running is recorded in the Process.Status file. - -What is the lastFile pointer? - - It is the Datetime since the begining of the hitory of a market that is considered without holes (or only with permanent ones, unfixables). In other words, the process - starts from the begining of the history of a market, and only when its history is signaled as "Complete" by the "Historic Trades" process. From there, the process is advancing - in time, validating the history and fixing holes, and moving the lastFile pointer forward as it gets sure that all the previous trades files are with no holes or unfixable ones. - This pointer will later be used by Indicators bots that depends of these trades, to calculate their final version of indicators up to the lastFile pointer datetime. - +Read the trades from Charly's Output and pack them into daily files with candles of one minute. */ @@ -136,6 +106,19 @@ What is the lastFile pointer? } + let atHeadOfMarket; // This tell us if we are at the month which includes the head of the market according to current datetime. + + if ((year === thisDatetime.getUTCFullYear() && month === thisDatetime.getUTCMonth() + 1)) { + + atHeadOfMarket = true; + + } else { + + atHeadOfMarket = false; + + } + + let nextIntervalExecution = false; // This tell weather the Interval module will be executed again or not. By default it will not unless some hole have been found in the current execution. let currentDate; // This will hold the current datetime of each execution. @@ -152,12 +135,6 @@ What is the lastFile pointer? let filePath; - let exchangeCallTime; - - const MAX_EXCHANGE_CALL_RETRIES = 3; - let exchangeCallRetries = 0; - - const MAX_HOLE_FIXING_RETRIES = 3; const FIRST_TRADE_RECORD_ID = -1; const UNKNOWN_TRADE_RECORD_ID = -2; @@ -171,7 +148,7 @@ What is the lastFile pointer? let lastLiveTradeFile; // Datetime of the last complete trades file written by the Live Trades process. let lastHistoricTradeFile; // Datatime of the last trades file written by the Historic Trades process. - let lastHoleFixingFile; // Datetime of the last file certified by the Hole Fixing process as without permanent holes. + let lastCandleFile; // Datetime of the last file certified by the Hole Fixing process as without permanent holes. /* The next 4 variables hold the results of the search of the next hole. */ @@ -188,8 +165,6 @@ What is the lastFile pointer? /* At every run, the process needs to loop through all the markets at this exchange. - The problem is that we can not fire all the requests at once to the exchange or we would get banned. - For that reason we create a queue with the markets to process and we do one at the time. The following functions marketsLoop(), openMarket(), closeMarket() and closeAndOpenMarket() controls the serialization of this processing. */ @@ -276,8 +251,6 @@ What is the lastFile pointer? if (market.status === markets.ENABLED) { - exchangeCallRetries = 0; // For each market we will count the retries, so we need to reset this here. - getStatusReport(); } else { @@ -324,144 +297,67 @@ What is the lastFile pointer? function getStatusReport() { - /* - - In order to know where to start the process (which datetime) we need to read the StatusReport of the process. This file will tell - us where we left after the last execution. If this file does not exist, this could mean that this process was never sucessfuly ran - for this market. This is normal when the market does not have the "complete" flag at the Trades History status report. - - We will also get the status report from the Live Trade process, to know which is highest possible datetime the process can work on. - - */ - try { let reportFilePath; let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json" - getLivesTrades(); + getHistoricTrades(); - function getLivesTrades() { - - reportFilePath = EXCHANGE_NAME + "/Processes/" + "Live-Trades"; - - azureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); - - function onStatusReportReceived(text) { - - if (text === undefined) { - - /* The file does not exist, so this means it is too early to run this process. */ - - const logText = "[INFO] 'getStatusReport' - Market " + market.assetA + '_' + market.assetB + " is too early too prcess it, since there are no Live Trades reports about it yet. Skipping it. "; - logger.write(logText); - - closeAndOpenMarket(); - - } else { - - let statusReport; - - try { - - statusReport = JSON.parse(text); - - lastLiveTradeFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS); - - } catch (err) { - - /* - - It might happen the the file contect is corrupt if it is read exactly at the moment the other process is writting it. - In this situation we just assume the current datetime is a good engough value. - - */ - - lastLiveTradeFile = new Date(); - - } - - /* We get from the file the datetime of the last file created. */ - - getHistoricTrades(); - - } - } - } - function getHistoricTrades() { - reportFilePath = EXCHANGE_NAME + "/Processes/" + "Historic-Trades"; + /* - azureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); + We need to know where is the begining of the market, since that will help us know how to estimate the value of the last close. + If we are at the begining of the market, the last close should be zero. + + */ + + reportFilePath = EXCHANGE_NAME + "/Processes/" + "Hiistoric-Trades"; + + charlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); function onStatusReportReceived(text) { - if (text === undefined) { + let statusReport; - /* The file does not exist, so this means it is too early to run this process. */ + try { - const logText = "[INFO] 'getStatusReport' - Market " + market.assetA + '_' + market.assetB + " is too early too prcess it, since there are no Historic Trades reports about it yet. Skipping it. "; + statusReport = JSON.parse(text); + + firstTradeFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS); + + getHoleFixing(); + + } catch (err) { + + const logText = "[INFO] 'getStatusReport' - Failed to read main Historic Trades Status Report for market " + market.assetA + '_' + market.assetB + " . Skipping it. "; logger.write(logText); closeAndOpenMarket(); - - } else { - - let statusReport = JSON.parse(text); - - /* We get from the file to know if this markets history is complete or not. */ - - if (statusReport.completeHistory === true) { - - lastHistoricTradeFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS); - - /* - - Before processing this month we need to check if it is not too far in the past. - - */ - - if ( - processDate.getUTCFullYear() < lastHistoricTradeFile.getUTCFullYear() - || - (processDate.getUTCFullYear() === lastHistoricTradeFile.getUTCFullYear() && processDate.getUTCMonth() < lastHistoricTradeFile.getUTCMonth()) - ) { - - const logText = "[INFO] 'getStatusReport' - The current year / month is before the start of the market history for market " + market.assetA + '_' + market.assetB + " . Skipping it. "; - logger.write(logText); - - closeAndOpenMarket(); - - } else { - - getHoleFixing(); - - } - - } else { - - const logText = "[INFO] 'getStatusReport' - Market " + market.assetA + '_' + market.assetB + " is history is not complete. Skipping it. "; - logger.write(logText); - - closeAndOpenMarket(); - - } - } } } function getHoleFixing() { + /* + + The limit in the future as of which candles to include is determined by the Hole Fixing process. We wont include + trades not certified to be without hole. + + */ + reportFilePath = EXCHANGE_NAME + "/Processes/" + "Hole-Fixing" + "/" + year + "/" + month; - azureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); + cahrlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); function onStatusReportReceived(text) { + let statusReport; + try { - holeFixingStatusReport = JSON.parse(text); + statusReport = JSON.parse(text); } catch (err) { text = undefined; // If the content of the file is corrupt, this equals as if the file did not exist. @@ -469,61 +365,82 @@ What is the lastFile pointer? if (text === undefined) { - /* + const logText = "[INFO] 'getStatusReport' - The current year / month was not yet hole-fixed for market " + market.assetA + '_' + market.assetB + " . Skipping it. "; + logger.write(logText); - The file does not exist, so this means this is the first time we are running this process. - Then, we might me under two different possible situations: - - 1. We are at the first month of the market. In this case, the starting date is not the begining of the month but the first date of - the history of the market. - - 2. We are at some other month different than the first one. In this case, the starting date is the first minute of the market. - - */ - - if (processDate.valueOf() < lastHistoricTradeFile.valueOf()) { - - lastHoleFixingFile = new Date(lastHistoricTradeFile.valueOf() - 60 * 1000); // One minute less that the begining of market history. - - currentTradeId = FIRST_TRADE_RECORD_ID; - currentDatetime = new Date(lastHistoricTradeFile.valueOf()); - - } else { - - lastHoleFixingFile = new Date(processDate.valueOf() - 60 * 1000); // One minute less that the begining of the month. - - currentTradeId = UNKNOWN_TRADE_RECORD_ID; - currentDatetime = new Date(processDate.valueOf()); - - } - - findNextHole(); + closeAndOpenMarket(); } else { if (holeFixingStatusReport.monthChecked === true) { - const logText = "[INFO] 'getStatusReport' - The current year / month was already fully checked for market " + market.assetA + '_' + market.assetB + " . Skipping it. "; - logger.write(logText); - - closeAndOpenMarket(); + lastFileWithoutHoles = new Date(); // We need this with a valid value. + getOneMinDailyCandles(); } else { - /* We get from the file the datetime of the last file without holes. */ + /* - lastHoleFixingFile = new Date(holeFixingStatusReport.lastFile.year + "-" + holeFixingStatusReport.lastFile.month + "-" + holeFixingStatusReport.lastFile.days + " " + holeFixingStatusReport.lastFile.hours + ":" + holeFixingStatusReport.lastFile.minutes + GMT_SECONDS); + If the hole report is incomplete, we are only interested if we are at the head of the market. + Otherwise, we are not going to calculate the candles of a month which was not fully checked for holes. - currentTradeId = holeFixingStatusReport.lastTrade.id; - currentDatetime = new Date(holeFixingStatusReport.lastTrade.year + "-" + holeFixingStatusReport.lastTrade.month + "-" + holeFixingStatusReport.lastTrade.days + " " + holeFixingStatusReport.lastTrade.hours + ":" + holeFixingStatusReport.lastTrade.minutes + ":" + holeFixingStatusReport.lastTrade.seconds + GMT_MILI_SECONDS); + */ - findNextHole(); + if (atHeadOfMarket === true) { + lastFileWithoutHoles = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS); + getOneMinDailyCandles(); + + } else { + + const logText = "[INFO] 'getStatusReport' - The current year / month was not completely hole-fixed for market " + market.assetA + '_' + market.assetB + " . Skipping it. "; + logger.write(logText); + + closeAndOpenMarket(); + + } } } } } + function getOneMinDailyCandles() { + + /* If the process run and was interrupted, there should be a status report that allows us to resume execution. */ + + reportFilePath = EXCHANGE_NAME + "/Processes/" + "One-Min-Daily-Candles" + "/" + year + "/" + month; + + bruceAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); + + function onStatusReportReceived(text) { + + let statusReport; + + try { + + statusReport = JSON.parse(text); + + lastCandleFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS); + lastCandleClose = statusReport.lastFile.candleClose; + + buildCandles(); + + } catch (err) { + + /* + + It might happen that the file content is corrupt or it does not exist. In either case we will point our lastCandleFile + to the last day of the previous month. + + */ + + lastCandleFile = new Date(processDate.valueOf() - ONE_DAY_IN_MILISECONDS); + findLastCandleCloseValue(); + + } + } + } + } catch (err) { const logText = "[ERROR] 'getStatusReport' - ERROR : " + err.message; @@ -532,6 +449,100 @@ What is the lastFile pointer? } } + function findLastCandleCloseValue() { + + /* + + We will search and find for the last trade before the begining of the current candle and that will give us the last close value. + Before going backwards, we need to be sure we are not at the begining of the market. + + */ + + if ((year === firstTradeFile.getUTCFullYear() && month === firstTradeFile.getUTCMonth() + 1)) { + + /* + + We are at the begining of the market, so we will set everyting to build the first candle. + + */ + + lastCandleFile = new Date(firstTradeFile.getUTCFullYear() + "-" + (firstTradeFile.getUTCMonth() + 1) + "-" + firstTradeFile.getUTCDay() + " " + "00:00" + GMT_SECONDS); + lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); + + lastCandleClose = 0; + + buildCandles(); + + } else { + + /* + + We are not at the begining of the market, so we need scan backwards the trade files until we find a non empty one and get the last trade. + + */ + + let date = new Date(processDate.valueOf()); + + function loopStart() { + + date = new Date(date.valueOf() - 60 * 1000); + + let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2); + let fileName = market.assetA + '_' + market.assetB + ".json" + let filePath = EXCHANGE_NAME + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath; + + charlyAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true); + + function onFileReceived(text) { + + let tradesFile; + + try { + + tradesFile = JSON.parse(text); + + if (tradesFile.length > 0) { + + lastCandleClose = tradesFile[tradesFile.length - 1][2]; // Position 2 is the rate at which the trade was executed. + + const logText = "[INFO] 'findLastCandleCloseValue' - Trades found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . lastCandleClose = " + lastCandleClose; + logger.write(logText); + + buildCandles(); + + } else { + + const logText = "[INFO] 'findLastCandleCloseValue' - No trades found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " ."; + logger.write(logText); + + loopStart(); + + } + + } catch (err) { + + const logText = "[INFO] 'findLastCandleCloseValue' - Empy of corrupt trade file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping it. "; + logger.write(logText); + + closeAndOpenMarket(); + } + } + } + } + } + + function buildCandles() { + + /* + + Here we are going to scan the trades files packing them in candles files every one day. + We need for this the last close value, bacause all candles that are empty of trades at the begining, they need to + have a valid open and close value. + + */ + + } + function findNextHole() { try { @@ -552,7 +563,7 @@ What is the lastFile pointer? let fileCheckedCounter = 0; - date = new Date(lastHoleFixingFile.valueOf()); + date = new Date(lastCandleFile.valueOf()); readNextFile();