diff --git a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js index 1e6f79e12..4b7ede69d 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js +++ b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js @@ -1,5 +1,4 @@ - -exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { +exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { const MODULE_NAME = "Historic OHLCVs"; const CANDLES_FOLDER_NAME = "Candles/One-Min"; @@ -57,8 +56,8 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { maxTradesPerFetch - this is set to a number that indicates the maximum number of trades the exchange will return in the fetchTrades method (e.g. for the Luno exchange, this number is 100). */ - let useFetchTradesForFetchOHLCVs = false - let maxTradesPerFetch = 100 + let useFetchTradesForFetchOHLCVs = false + let maxTradesPerFetch = 100 return thisObject; @@ -94,15 +93,15 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { if (API.rateLimit !== undefined) { rateLimit = API.rateLimit } + if (API.limit !== undefined) { + limit = API.limit + } if (API.hostname !== undefined) { hostname = API.hostname } if (API.fetchType !== undefined) { fetchType = API.fetchType } - if (API.limit !== undefined) { - limit = API.limit - } if (API.useFetchTradesForFetchOHLCVs !== undefined) { useFetchTradesForFetchOHLCVs = API.useFetchTradesForFetchOHLCVs if (API.maxTradesPerFetch !== undefined) { @@ -200,6 +199,11 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { return; } + /* Check if the uiStartDate is Invalid, if so lets set it to the Epoch and let the getMarketStart function later take care of it */ + if(isNaN(uiStartDate)) { + uiStartDate = new Date(0) + } + thisReport = statusDependencies.statusReports.get(reportKey) if (thisReport.file.beginingOfMarket !== undefined) { // This means this is not the first time this process has run. @@ -294,7 +298,7 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { } } } - } + } } catch (err) { TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, @@ -345,9 +349,12 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { let previousSince let fromDate = new Date(since) let lastDate = new Date() + let changeToMarketStart = false while (true) { + let invalidSince = false + /* Reporting we are doing well */ function heartBeat(noNewInternalLoop) { let processingDate = new Date(since) @@ -365,8 +372,6 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { } } - heartBeat() - /* Defining if we will query the exchange by Date or Id */ if (fetchType === "by Id") { /* @@ -377,11 +382,120 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { since = lastId } + /* + If this is the first time the process has run, check if to see if the start date the user + has entered is before the start of the Market on the Exchange. If this is the case, jump + forward to the start date of the Market. + */ + if(firstTimeThisProcessRun && changeToMarketStart === false && useFetchTradesForFetchOHLCVs === false) { + let marketStartSince + + marketStartSince = await findMarketStart() + + if (initialProcessTimestamp < marketStartSince) { + since = marketStartSince + fromDate = new Date(since) + changeToMarketStart = true + } + } + + heartBeat() + /* Fetching the OHLCVs from the exchange.*/ await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit const OHLCVs = useFetchTradesForFetchOHLCVs ? - await fetchTradesForOHLCV(symbol, '1m', since, limit, params) : - await exchange.fetchOHLCV(symbol, '1m', since, limit, params) + await fetchTradesForOHLCV(symbol, '1m', since, limit, params) : + await exchange.fetchOHLCV(symbol, '1m', since, limit, params) + + + if (OHLCVs.length === 0) { + /* + If OHLCVs is empty we might be dealing with extended maintenance longer than our OHLCV limit. + If this is the case jump forward to the next valid OHLCV record. + */ + since = await findNextValidOHLCV() + } + + /* + CCXT for certain Exchanges will return an empty or partial array if the since date is too + far in the past if this occurs this function will recursively step backwards in the Exchange + and Market in 1D intervals until it finds the earliest date with values. + */ + + async function findMarketStart() { + + let earliestMarketSince = undefined + let foundDate = false + let searchSize = limit + let previousMarketSince = undefined + while (foundDate !== true) { + + await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit + const earlyOHLCVs = useFetchTradesForFetchOHLCVs ? + await fetchTradesForOHLCV(symbol, '1d', earliestMarketSince, searchSize, params) : + await exchange.fetchOHLCV(symbol, '1d', earliestMarketSince, searchSize, params) + + if (earlyOHLCVs.length === searchSize && previousMarketSince !== earlyOHLCVs[0][0]) { // If array is full, and the date returned isn't the same then we are still working in valid data, search further back + earliestMarketSince = (earlyOHLCVs[0][0] - (TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS * searchSize)) + previousMarketSince = earlyOHLCVs[0][0] + + } else if (earlyOHLCVs.length === searchSize && previousMarketSince === earlyOHLCVs[0][0]) { // If array is full, but the first OHLCV date is the same as last time then we've found our Market Start + foundDate = true + invalidSince = true + return earlyOHLCVs[0][0] + + } else if (earlyOHLCVs.length < searchSize && earlyOHLCVs.length > 0) { // If the array is partially filled then, take the earliest data and we've found our Market Start + foundDate = true + invalidSince = true + return earlyOHLCVs[0][0] + + } else if ((earlyOHLCVs.length === 0 && searchSize !== 1)) { // If array is empty, we've gone too far, half the search size and retry + earliestMarketSince = (earliestMarketSince + (TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS * searchSize)) + if (searchSize > 1) { + searchSize = Math.floor(searchSize / 2) + } + + } else { // Otherwise, the array is empty and we're at the lowest searchSize then we've found our Market Start + foundDate = true + invalidSince = true + return earliestMarketSince + + } + + let processingDate = new Date(earliestMarketSince) + processingDate = processingDate.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processingDate.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processingDate.getUTCDate(), 2); + TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, "Invalid or No Start Date. Finding Market Start @ " + processingDate, 0) + } + } + + /* + CCXT will return an empty array if there are no OHLCVs for the limit. + This can happen when the Exchange has extended maintenance i.e. over + 1,000 minutes (example see Binance on 2018-02-08). This function will + return the next valid returned OHLCV. + */ + + async function findNextValidOHLCV() { + + let foundDate = false + + while (foundDate !== true) { + + await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit + const nextValidOHLCVs = useFetchTradesForFetchOHLCVs ? + await fetchTradesForOHLCV(symbol, '1m', since, limit, params) : + await exchange.fetchOHLCV(symbol, '1m', since, limit, params) + + if (nextValidOHLCVs.length === 0) { + since = (since + (TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * limit)) + + } else { + foundDate = true + invalidSince = true + return nextValidOHLCVs[0][0] + } + } + } /* OHLCV Structure @@ -418,16 +532,13 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { fromDate = new Date(beginingOfMarket.valueOf()) firstTimeThisProcessRun = false } - } - if (OHLCVs.length > 1) { previousSince = since since = OHLCVs[OHLCVs.length - 1][0] // 'timestamp' if (since === previousSince) { since++ // this prevents requesting in a loop OHLCVs with the same timestamp, that can happen when all the records fetched come with exactly the same timestamp. } - lastId = OHLCVs[OHLCVs.length - 1]['id'] for (let i = 0; i < OHLCVs.length; i++) { @@ -444,8 +555,19 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { heartBeat(true) } + let currentDate = new Date(since) + + /* + If we've pulled less OHLCVs than the limit, this either means: + (1) we reached the current date and are now pulling the latest + (2) there was exchange maintenance on the date being processed + (3) we've found zero records as the start date was way in the past + + If (1) is the case, or if we're stopping the task or have hit the max + limit. then break and stop till the next loop. + */ if ( - OHLCVs.length < limit - 1 || + (OHLCVs.length < limit - 1 && invalidSince === false && TS.projects.superalgos.utilities.dateTimeFunctions.areTheseDatesEqual(currentDate, new Date())) || TS.projects.superalgos.globals.taskVariables.IS_TASK_STOPPING === true || rawDataArray.length >= MAX_OHLCVs_PER_EXECUTION ) { @@ -868,7 +990,7 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { if (dataLength > 0) { // first get the start of the day after this day we are checking let timestamp = (day * TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) + - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS + TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS if (rawDataArray[dataLength - 1][0] < timestamp) { // there is no data for the next day, so now trim this day's data to remove anything before it timestamp -= TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS