From 0b9af11663fd77ae90349ef4bc0d93109f1c361b Mon Sep 17 00:00:00 2001 From: Smidy13 <77650800+Smidy13@users.noreply.github.com> Date: Wed, 28 Apr 2021 17:13:45 +1200 Subject: [PATCH 1/4] Bug Fix: Unnecessary Halt in getOHLCVs, Handling for Invalid Start Date on getOHLCVs & Handling for Exchange maintenance over limit --- .../Exchange-Raw-Data/HistoricOHLCVs.js | 92 +++++++++++++++++-- 1 file changed, 85 insertions(+), 7 deletions(-) diff --git a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js index 1e6f79e12..03f85c18d 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js +++ b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js @@ -1,5 +1,4 @@ - -exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { +exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { const MODULE_NAME = "Historic OHLCVs"; const CANDLES_FOLDER_NAME = "Candles/One-Min"; @@ -100,9 +99,6 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { if (API.fetchType !== undefined) { fetchType = API.fetchType } - if (API.limit !== undefined) { - limit = API.limit - } if (API.useFetchTradesForFetchOHLCVs !== undefined) { useFetchTradesForFetchOHLCVs = API.useFetchTradesForFetchOHLCVs if (API.maxTradesPerFetch !== undefined) { @@ -110,6 +106,9 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { } } } + if (API.limit !== undefined) { + limit = API.limit + } } } @@ -343,6 +342,7 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { let lastOHLCVKey = '' let params = undefined let previousSince + let invalidDate = false let fromDate = new Date(since) let lastDate = new Date() @@ -383,6 +383,73 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { await fetchTradesForOHLCV(symbol, '1m', since, limit, params) : await exchange.fetchOHLCV(symbol, '1m', since, limit, params) + /* + If no results are returned, check if to see if the date the user has entered is before + the start of the Market on the Exchange. If this isn't the case, jump forward to the + next available OHLCV. + */ + if (OHLCVs.length === 0) { + let earliestSince + + earliestSince = await findEarliestDate() + + if (initialProcessTimestamp < earliestSince) { + since = earliestSince + } else { + since = await findNextValidOHLCV() + } + } + + /* + CCXT will return an empty array if the since date is too far in the past + if this occurs this function will recursively step backwards in the Exchange + and Market in 1D intervals until it finds the earliest date with values. + */ + + async function findEarliestDate() { + + let earliestSince = undefined + let foundDate = false + + while (foundDate !== true) { + + const earlyOHLCVs = await exchange.fetchOHLCV(symbol, '1d', earliestSince, limit, params) + + if (earlyOHLCVs.length === limit) { + earliestSince = (earlyOHLCVs[0][0] - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS * limit) + } else { + foundDate = true + invalidDate = true + return earlyOHLCVs[0][0] + } + } + } + + /* + CCXT will return an empty array if there are no candles for the limit. + This can happen when the Exchange has extended maintenance i.e. over + 1,000 minutes (example see Binance on 2018-02-08). This function will + skip to the next valid returned OHLCV. + */ + + async function findNextValidOHLCV() { + + let foundDate = false + + while (foundDate !== true) { + const nextValidOHLCVs = await exchange.fetchOHLCV(symbol, '1m', since, limit, params) + + if (nextValidOHLCVs.length === 0) { + since = (since + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * limit) + } else { + foundDate = true + invalidDate = true + return nextValidOHLCVs[0][0] + } + } + + } + /* OHLCV Structure The fetchOHLCV method shown above returns a list (a flat array) of OHLCV candles represented by the following structure: @@ -420,7 +487,7 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { } } - if (OHLCVs.length > 1) { + if (OHLCVs.length >= 1) { previousSince = since since = OHLCVs[OHLCVs.length - 1][0] // 'timestamp' if (since === previousSince) { @@ -444,8 +511,19 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) { heartBeat(true) } + let currentDate = new Date(since) + + /* + If we've pulled less OHLCVs than the limit, this either means: + (1) we reached the current date and are now pulling the latest + (2) there was exchange maintenance on the date being processed + (3) we've found zero records as the start date was way in the past + + If (1) is the case, or if we're stopping the task or have hit the max + limit. then break and stop till the next loop. + */ if ( - OHLCVs.length < limit - 1 || + (OHLCVs.length < limit - 1 && invalidDate === false && TS.projects.superalgos.utilities.dateTimeFunctions.areTheseDatesEqual(currentDate, new Date())) || TS.projects.superalgos.globals.taskVariables.IS_TASK_STOPPING === true || rawDataArray.length >= MAX_OHLCVs_PER_EXECUTION ) { From d052098b66996c8b121589073886a93039757937 Mon Sep 17 00:00:00 2001 From: Smidy13 <77650800+Smidy13@users.noreply.github.com> Date: Wed, 28 Apr 2021 17:45:02 +1200 Subject: [PATCH 2/4] Moving back if statement moved during testing, setting InvalidDate at right point --- .../Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js index 03f85c18d..7a885399f 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js +++ b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js @@ -93,6 +93,9 @@ if (API.rateLimit !== undefined) { rateLimit = API.rateLimit } + if (API.limit !== undefined) { + limit = API.limit + } if (API.hostname !== undefined) { hostname = API.hostname } @@ -106,9 +109,6 @@ } } } - if (API.limit !== undefined) { - limit = API.limit - } } } @@ -342,12 +342,13 @@ let lastOHLCVKey = '' let params = undefined let previousSince - let invalidDate = false let fromDate = new Date(since) let lastDate = new Date() while (true) { + let invalidDate = false + /* Reporting we are doing well */ function heartBeat(noNewInternalLoop) { let processingDate = new Date(since) From be094ba536a5823b244d02d88fa8663dc2decdd7 Mon Sep 17 00:00:00 2001 From: Smidy13 <77650800+Smidy13@users.noreply.github.com> Date: Wed, 28 Apr 2021 22:50:51 +1200 Subject: [PATCH 3/4] Renaming Variables to More Accurate Names, Adding in Promises for Functions --- .../Exchange-Raw-Data/HistoricOHLCVs.js | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js index 7a885399f..f0f2fcaf2 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js +++ b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js @@ -199,6 +199,11 @@ return; } + /* Check if the uiStartDate is Invalid, if so lets set it to the Epoch and let the getMarketStart function later take care of it */ + if(isNaN(uiStartDate)) { + uiStartDate = new Date(0) + } + thisReport = statusDependencies.statusReports.get(reportKey) if (thisReport.file.beginingOfMarket !== undefined) { // This means this is not the first time this process has run. @@ -347,7 +352,7 @@ while (true) { - let invalidDate = false + let invalidSince = false /* Reporting we are doing well */ function heartBeat(noNewInternalLoop) { @@ -385,18 +390,23 @@ await exchange.fetchOHLCV(symbol, '1m', since, limit, params) /* - If no results are returned, check if to see if the date the user has entered is before - the start of the Market on the Exchange. If this isn't the case, jump forward to the - next available OHLCV. + If no results are returned, check if to see if the start date the user has entered is + before the start of the Market on the Exchange. If this is the case, jump forward to the + next valid OHLCV record. */ if (OHLCVs.length === 0) { - let earliestSince - earliestSince = await findEarliestDate() + let marketStartSince + marketStartSince = await findMarketStart() - if (initialProcessTimestamp < earliestSince) { - since = earliestSince + if (initialProcessTimestamp < marketStartSince) { + since = marketStartSince } else { + + /* + Otherwise we might be dealing with extended maintenance longer than our OHLCV limit. + If this is the case jump forward to the next valid OHLCV record. + */ since = await findNextValidOHLCV() } } @@ -407,30 +417,31 @@ and Market in 1D intervals until it finds the earliest date with values. */ - async function findEarliestDate() { + async function findMarketStart() { - let earliestSince = undefined + let earliestMarketSince = undefined let foundDate = false while (foundDate !== true) { - const earlyOHLCVs = await exchange.fetchOHLCV(symbol, '1d', earliestSince, limit, params) + await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit + const earlyOHLCVs = await exchange.fetchOHLCV(symbol, '1d', earliestMarketSince, limit, params) if (earlyOHLCVs.length === limit) { - earliestSince = (earlyOHLCVs[0][0] - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS * limit) + earliestMarketSince = (earlyOHLCVs[0][0] - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS * limit) } else { foundDate = true - invalidDate = true + invalidSince = true return earlyOHLCVs[0][0] } } } /* - CCXT will return an empty array if there are no candles for the limit. + CCXT will return an empty array if there are no OHLCVs for the limit. This can happen when the Exchange has extended maintenance i.e. over 1,000 minutes (example see Binance on 2018-02-08). This function will - skip to the next valid returned OHLCV. + return the next valid returned OHLCV. */ async function findNextValidOHLCV() { @@ -438,13 +449,15 @@ let foundDate = false while (foundDate !== true) { + + await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit const nextValidOHLCVs = await exchange.fetchOHLCV(symbol, '1m', since, limit, params) if (nextValidOHLCVs.length === 0) { since = (since + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * limit) } else { foundDate = true - invalidDate = true + invalidSince = true return nextValidOHLCVs[0][0] } } @@ -486,16 +499,13 @@ fromDate = new Date(beginingOfMarket.valueOf()) firstTimeThisProcessRun = false } - } - if (OHLCVs.length >= 1) { previousSince = since since = OHLCVs[OHLCVs.length - 1][0] // 'timestamp' if (since === previousSince) { since++ // this prevents requesting in a loop OHLCVs with the same timestamp, that can happen when all the records fetched come with exactly the same timestamp. } - lastId = OHLCVs[OHLCVs.length - 1]['id'] for (let i = 0; i < OHLCVs.length; i++) { @@ -524,7 +534,7 @@ limit. then break and stop till the next loop. */ if ( - (OHLCVs.length < limit - 1 && invalidDate === false && TS.projects.superalgos.utilities.dateTimeFunctions.areTheseDatesEqual(currentDate, new Date())) || + (OHLCVs.length < limit - 1 && invalidSince === false && TS.projects.superalgos.utilities.dateTimeFunctions.areTheseDatesEqual(currentDate, new Date())) || TS.projects.superalgos.globals.taskVariables.IS_TASK_STOPPING === true || rawDataArray.length >= MAX_OHLCVs_PER_EXECUTION ) { From 09fa1cca5d5747618095782092c96cbd075876a0 Mon Sep 17 00:00:00 2001 From: Smidy13 <77650800+Smidy13@users.noreply.github.com> Date: Sun, 2 May 2021 20:18:13 +1200 Subject: [PATCH 4/4] Further changes to HistoricOHLCVs.js after extensive testing --- .../Exchange-Raw-Data/HistoricOHLCVs.js | 103 ++++++++++++------ 1 file changed, 68 insertions(+), 35 deletions(-) diff --git a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js index f0f2fcaf2..4b7ede69d 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js +++ b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js @@ -56,8 +56,8 @@ maxTradesPerFetch - this is set to a number that indicates the maximum number of trades the exchange will return in the fetchTrades method (e.g. for the Luno exchange, this number is 100). */ - let useFetchTradesForFetchOHLCVs = false - let maxTradesPerFetch = 100 + let useFetchTradesForFetchOHLCVs = false + let maxTradesPerFetch = 100 return thisObject; @@ -298,7 +298,7 @@ } } } - } + } } catch (err) { TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, @@ -349,6 +349,7 @@ let previousSince let fromDate = new Date(since) let lastDate = new Date() + let changeToMarketStart = false while (true) { @@ -371,8 +372,6 @@ } } - heartBeat() - /* Defining if we will query the exchange by Date or Id */ if (fetchType === "by Id") { /* @@ -383,57 +382,89 @@ since = lastId } - /* Fetching the OHLCVs from the exchange.*/ - await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit - const OHLCVs = useFetchTradesForFetchOHLCVs ? - await fetchTradesForOHLCV(symbol, '1m', since, limit, params) : - await exchange.fetchOHLCV(symbol, '1m', since, limit, params) - /* - If no results are returned, check if to see if the start date the user has entered is - before the start of the Market on the Exchange. If this is the case, jump forward to the - next valid OHLCV record. + If this is the first time the process has run, check if to see if the start date the user + has entered is before the start of the Market on the Exchange. If this is the case, jump + forward to the start date of the Market. */ - if (OHLCVs.length === 0) { - + if(firstTimeThisProcessRun && changeToMarketStart === false && useFetchTradesForFetchOHLCVs === false) { let marketStartSince + marketStartSince = await findMarketStart() if (initialProcessTimestamp < marketStartSince) { since = marketStartSince - } else { - - /* - Otherwise we might be dealing with extended maintenance longer than our OHLCV limit. - If this is the case jump forward to the next valid OHLCV record. - */ - since = await findNextValidOHLCV() + fromDate = new Date(since) + changeToMarketStart = true } } + heartBeat() + + /* Fetching the OHLCVs from the exchange.*/ + await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit + const OHLCVs = useFetchTradesForFetchOHLCVs ? + await fetchTradesForOHLCV(symbol, '1m', since, limit, params) : + await exchange.fetchOHLCV(symbol, '1m', since, limit, params) + + + if (OHLCVs.length === 0) { + /* + If OHLCVs is empty we might be dealing with extended maintenance longer than our OHLCV limit. + If this is the case jump forward to the next valid OHLCV record. + */ + since = await findNextValidOHLCV() + } + /* - CCXT will return an empty array if the since date is too far in the past - if this occurs this function will recursively step backwards in the Exchange - and Market in 1D intervals until it finds the earliest date with values. + CCXT for certain Exchanges will return an empty or partial array if the since date is too + far in the past if this occurs this function will recursively step backwards in the Exchange + and Market in 1D intervals until it finds the earliest date with values. */ async function findMarketStart() { let earliestMarketSince = undefined let foundDate = false - + let searchSize = limit + let previousMarketSince = undefined while (foundDate !== true) { await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit - const earlyOHLCVs = await exchange.fetchOHLCV(symbol, '1d', earliestMarketSince, limit, params) + const earlyOHLCVs = useFetchTradesForFetchOHLCVs ? + await fetchTradesForOHLCV(symbol, '1d', earliestMarketSince, searchSize, params) : + await exchange.fetchOHLCV(symbol, '1d', earliestMarketSince, searchSize, params) - if (earlyOHLCVs.length === limit) { - earliestMarketSince = (earlyOHLCVs[0][0] - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS * limit) - } else { + if (earlyOHLCVs.length === searchSize && previousMarketSince !== earlyOHLCVs[0][0]) { // If array is full, and the date returned isn't the same then we are still working in valid data, search further back + earliestMarketSince = (earlyOHLCVs[0][0] - (TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS * searchSize)) + previousMarketSince = earlyOHLCVs[0][0] + + } else if (earlyOHLCVs.length === searchSize && previousMarketSince === earlyOHLCVs[0][0]) { // If array is full, but the first OHLCV date is the same as last time then we've found our Market Start foundDate = true invalidSince = true return earlyOHLCVs[0][0] + + } else if (earlyOHLCVs.length < searchSize && earlyOHLCVs.length > 0) { // If the array is partially filled then, take the earliest data and we've found our Market Start + foundDate = true + invalidSince = true + return earlyOHLCVs[0][0] + + } else if ((earlyOHLCVs.length === 0 && searchSize !== 1)) { // If array is empty, we've gone too far, half the search size and retry + earliestMarketSince = (earliestMarketSince + (TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS * searchSize)) + if (searchSize > 1) { + searchSize = Math.floor(searchSize / 2) + } + + } else { // Otherwise, the array is empty and we're at the lowest searchSize then we've found our Market Start + foundDate = true + invalidSince = true + return earliestMarketSince + } + + let processingDate = new Date(earliestMarketSince) + processingDate = processingDate.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processingDate.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processingDate.getUTCDate(), 2); + TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, "Invalid or No Start Date. Finding Market Start @ " + processingDate, 0) } } @@ -451,17 +482,19 @@ while (foundDate !== true) { await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit - const nextValidOHLCVs = await exchange.fetchOHLCV(symbol, '1m', since, limit, params) + const nextValidOHLCVs = useFetchTradesForFetchOHLCVs ? + await fetchTradesForOHLCV(symbol, '1m', since, limit, params) : + await exchange.fetchOHLCV(symbol, '1m', since, limit, params) if (nextValidOHLCVs.length === 0) { - since = (since + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * limit) + since = (since + (TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * limit)) + } else { foundDate = true invalidSince = true return nextValidOHLCVs[0][0] } } - } /* @@ -957,7 +990,7 @@ if (dataLength > 0) { // first get the start of the day after this day we are checking let timestamp = (day * TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) + - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS + TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS if (rawDataArray[dataLength - 1][0] < timestamp) { // there is no data for the next day, so now trim this day's data to remove anything before it timestamp -= TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS