From be094ba536a5823b244d02d88fa8663dc2decdd7 Mon Sep 17 00:00:00 2001
From: Smidy13 <77650800+Smidy13@users.noreply.github.com>
Date: Wed, 28 Apr 2021 22:50:51 +1200
Subject: [PATCH] Renaming Variables to More Accurate Names, Adding in Promises
 for Functions

---
 .../Exchange-Raw-Data/HistoricOHLCVs.js       | 50 +++++++++++--------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js
index 7a885399f..f0f2fcaf2 100644
--- a/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js
+++ b/Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js
@@ -199,6 +199,11 @@
                         return;
                     }
 
+                    /* Check if the uiStartDate is Invalid, if so lets set it to the Epoch and let the getMarketStart function later take care of it */
+                    if(isNaN(uiStartDate)) {
+                        uiStartDate = new Date(0)
+                    }
+
                     thisReport = statusDependencies.statusReports.get(reportKey)
 
                     if (thisReport.file.beginingOfMarket !== undefined) { // This means this is not the first time this process has run.
@@ -347,7 +352,7 @@
 
                     while (true) {
 
-                        let invalidDate = false
+                        let invalidSince = false
 
                         /* Reporting we are doing well */
                         function heartBeat(noNewInternalLoop) {
@@ -385,18 +390,23 @@
                                        await exchange.fetchOHLCV(symbol, '1m', since, limit, params)
 
                         /*
-                        If no results are returned, check if to see if the date the user has entered is before
-                        the start of the Market on the Exchange. If this isn't the case, jump forward to the
-                        next available OHLCV.
+                        If no results are returned, check if to see if the start date the user has entered is
+                        before the start of the Market on the Exchange. If this is the case, jump forward to the
+                        next valid OHLCV record.
                          */
                         if (OHLCVs.length === 0) {
-                            let earliestSince
 
-                            earliestSince = await findEarliestDate()
+                            let marketStartSince
+                            marketStartSince = await findMarketStart()
 
-                            if (initialProcessTimestamp < earliestSince) {
-                                since = earliestSince
+                            if (initialProcessTimestamp < marketStartSince) {
+                                since = marketStartSince
                             } else {
+
+                                /*
+                                Otherwise we might be dealing with extended maintenance longer than our OHLCV limit.
+                                If this is the case jump forward to the next valid OHLCV record.
+                                 */
                                 since = await findNextValidOHLCV()
                             }
                         }
@@ -407,30 +417,31 @@
                         and Market in 1D intervals until it finds the earliest date with values.
                         */
 
-                        async function findEarliestDate() {
+                        async function findMarketStart() {
 
-                            let earliestSince = undefined
+                            let earliestMarketSince = undefined
                             let foundDate = false
 
                             while (foundDate !== true) {
 
-                                const earlyOHLCVs = await exchange.fetchOHLCV(symbol, '1d', earliestSince, limit, params)
+                                await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit
+                                const earlyOHLCVs = await exchange.fetchOHLCV(symbol, '1d', earliestMarketSince, limit, params)
 
                                 if (earlyOHLCVs.length === limit) {
-                                    earliestSince = (earlyOHLCVs[0][0] - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS * limit)
+                                    earliestMarketSince = (earlyOHLCVs[0][0] - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS * limit)
                                 } else {
                                     foundDate = true
-                                    invalidDate = true
+                                    invalidSince = true
                                     return earlyOHLCVs[0][0]
                                 }
                             }
                         }
 
                         /*
-                        CCXT will return an empty array if there are no candles for the limit.
+                        CCXT will return an empty array if there are no OHLCVs for the limit.
                         This can happen when the Exchange has extended maintenance i.e. over
                         1,000 minutes (example see Binance on 2018-02-08). This function will
-                        skip to the next valid returned OHLCV.
+                        return the next valid returned OHLCV.
                         */
 
                         async function findNextValidOHLCV() {
@@ -438,13 +449,15 @@
                             let foundDate = false
 
                             while (foundDate !== true) {
+
+                                await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit
                                 const nextValidOHLCVs = await exchange.fetchOHLCV(symbol, '1m', since, limit, params)
 
                                 if (nextValidOHLCVs.length === 0) {
                                     since = (since + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * limit)
                                 } else {
                                     foundDate = true
-                                    invalidDate = true
+                                    invalidSince = true
                                     return nextValidOHLCVs[0][0]
                                 }
                             }
@@ -486,16 +499,13 @@
                                 fromDate = new Date(beginingOfMarket.valueOf())
                                 firstTimeThisProcessRun = false
                             }
-                        }
 
-                        if (OHLCVs.length >= 1) {
                             previousSince = since
                             since = OHLCVs[OHLCVs.length - 1][0] // 'timestamp'
                             if (since === previousSince) {
                                 since++ // this prevents requesting in a loop OHLCVs with the same timestamp, that can happen when all the records fetched come with exactly the same timestamp.
                             }
 
-
                             lastId = OHLCVs[OHLCVs.length - 1]['id']
 
                             for (let i = 0; i < OHLCVs.length; i++) {
@@ -524,7 +534,7 @@
                         limit. then break and stop till the next loop.
                          */
                         if (
-                            (OHLCVs.length < limit - 1 && invalidDate === false && TS.projects.superalgos.utilities.dateTimeFunctions.areTheseDatesEqual(currentDate, new Date())) ||
+                            (OHLCVs.length < limit - 1 && invalidSince === false && TS.projects.superalgos.utilities.dateTimeFunctions.areTheseDatesEqual(currentDate, new Date())) ||
                             TS.projects.superalgos.globals.taskVariables.IS_TASK_STOPPING === true ||
                             rawDataArray.length >= MAX_OHLCVs_PER_EXECUTION
                         ) {