fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug Fix

Browse Source
This commit is contained in:
Luis Fernando Molina committed 2021-01-21 13:33:07 +01:00
1 parent 9522b7b848
commit 1e1fa783ca
1 file changed
+231 -80
Download Patch File Download Diff File Expand all files Collapse all files
Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/HistoricOHLCVs.js
+231 -80
View File
@@ -20,7 +20,6 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
let fetchType = "by Time"
let lastId
let firstId
let allOHLCVs = []
let thisReport;
let since
let initialProcessTimestamp
@@ -119,6 +118,11 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
function start(callBackFunction) {
try {
/*
Initialize the array where we are going to receive the
data from the exchange.
*/
let rawDataArray = []
if (TS.projects.superalgos.globals.taskVariables.IS_TASK_STOPPING === true) {
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE);
@@ -252,7 +256,7 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
processingDate = processingDate.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processingDate.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processingDate.getUTCDate(), 2);
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> getOHLCVs -> Fetching OHLCVs @ " + processingDate + "-> exchange = " + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.name + " -> symbol = " + symbol + " -> since = " + since + " -> limit = " + limit)
let heartBeatText = "Fetching " + allOHLCVs.length.toFixed(0) + " OHLCVs from " + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.name + " " + symbol + " @ " + processingDate
let heartBeatText = "Fetching " + rawDataArray.length.toFixed(0) + " OHLCVs from " + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.name + " " + symbol + " @ " + processingDate
let currentDate = new Date(since)
let percentage = TS.projects.superalgos.utilities.dateTimeFunctions.getPercentage(fromDate, currentDate, lastDate)
TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, heartBeatText, percentage) // tell the world we are alive and doing well
@@ -332,7 +336,7 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
let OHLCVKey = OHLCV[0] + '-' + OHLCV[1].toFixed(16) + '-' + OHLCV[2].toFixed(16) + '-' + OHLCV[3].toFixed(16) + '-' + OHLCV[4].toFixed(16) + '-' + OHLCV[5].toFixed(16)
if (OHLCVKey !== lastOHLCVKey) {
allOHLCVs.push(OHLCV)
rawDataArray.push(OHLCV)
}
lastOHLCVKey = OHLCVKey
}
@@ -343,7 +347,7 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
if (
OHLCVs.length < limit - 1 ||
TS.projects.superalgos.globals.taskVariables.IS_TASK_STOPPING === true ||
allOHLCVs.length >= MAX_OHLCVs_PER_EXECUTION
rawDataArray.length >= MAX_OHLCVs_PER_EXECUTION
) {
break
}
@@ -376,18 +380,27 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
}
async function saveOHLCVs() {
/*
What we are going to do in this function is to save all the candles
received from the exchage. We need to partition the batch of candles
into 1 day files. At the same time we need to take care of the situation
that some exchanges send inconsitent data. We have detected some cases
where candles do not begin at second 0 of the minute but are a little
bit shifted. We will try to detectthis and fix it as we go.
We have the data received from the exchage at the arrary rawDataArray
*/
try {
let candlesFileContent = '['
let volumesFileContent = '['
let previousDay = Math.trunc((initialProcessTimestamp - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) / TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS)
let currentDay = Math.trunc((initialProcessTimestamp - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) / TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS)
let needSeparator = false
let error
let separator
let heartBeatCounter = 0
let headOfTheMarketReached = false
let savingProcedureFinished = false
let endOfTheOHLCVArrayReached = false
let currentDay = Math.trunc((initialProcessTimestamp - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) / TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS)
let lastCandle = {
begin: 0,
@@ -405,55 +418,104 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
sell: 0
}
let i = 0
let ohlcvArrayIndex = 0
lastId = undefined
if (lastCandleOfTheDay !== undefined) {
lastCandle = JSON.parse(JSON.stringify(lastCandleOfTheDay))
}
/*
At a macro level, we will be creating one file per day, so we will
run into a loop that each run will represent one day.
*/
controlLoop()
function loop() {
/*
Here we will try to match the OHLCVs received with each possible
candle of a single day.
*/
let filesToCreate = 0
let filesCreated = 0
/* Go through all possible 1 min candles of a day. */
for (let j = 0; j < 60 * 24; j++) {
/*
We will loop aroud all the possible 1 minute candles that be found
on a single day. For each minute of the day, we will try to find
the matching OHLCV. This would be an easy task if the exchanges
would allways return consistent data, but that is not the case.
Sometimes, some candles are missing. We have seen also candles that
do not start at an UTC minute, they are shifted in time some seconds.
In order to address all these inconsistencies the process gets a little
bit complicated.
*/
for (let minuteOfTheDay = 0; minuteOfTheDay < 60 * 24; minuteOfTheDay++) {
/*
We initialize our candle and volume objecs positioning them
at the current minute of the day of the current day. We also
initialize them with the last candle and volume property values.
The reason we do this last thing is because we don't knwo if
we are going to find or not a matching OHLCV. If we do find one
these property values will be overwritten, and if not, they will
hold at least the last know value.
*/
let candle = {
begin: currentDay * TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * j,
end: currentDay * TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * j + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS - 1,
begin: currentDay * TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * minuteOfTheDay,
end: currentDay * TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * minuteOfTheDay + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS - 1,
open: lastCandle.close,
close: lastCandle.close,
min: lastCandle.close,
max: lastCandle.close
}
let volume = {
begin: currentDay * TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * j,
end: currentDay * TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * j + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS - 1,
begin: currentDay * TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * minuteOfTheDay,
end: currentDay * TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS * minuteOfTheDay + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS - 1,
buy: lastVolume.buy,
sell: lastVolume.sell
}
let processingDate = new Date(candle.begin)
processingDate = processingDate.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processingDate.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processingDate.getUTCDate(), 2);
/*
Here we will check that the current candle is not going into the future.
Remember we are looping around all possible minutes of a day, and when
that day is the current actual day, and the current minute is in the future,
it manke no more sense to continue inside this loop since we are not going
to find more OHLCVs matchings.
if (candle.begin > (new Date()).valueOf()) {
At the same time, we are going to check that we haven't processed the whole
OHLCV array. Once we reached the end of it, it makes no sense to continue
inside this loop.
*/
if (
candle.begin > (new Date()).valueOf() ||
endOfTheOHLCVArrayReached === true
) {
/* We stop when the current candle is pointing to a time in the future.*/
headOfTheMarketReached = true
savingProcedureFinished = true
/*
This will be our last file saved.
*/
saveFile(currentDay)
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> saveOHLCVs -> Before Fetch -> Saving OHLCVs @ " + processingDate + " -> i = " + i + " -> total = " + allOHLCVs.length)
TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, "Saving " + i.toFixed(0) + " / " + allOHLCVs.length + " OHLCVs from " + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.name + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
/*
We will produce our last log and heartbeat, since we have just
reached the head of the market.
*/
logAndHeartBeat()
/* We exit the loop and we aint comming back*/
return
}
let OHLCV = { // these values should be overrided unless there are no OHLVCs fetched from the exchange.
/*
We initialize here the OHLCV object. These initial
values should be overrided unless there are no
OHLVCs fetched from the exchange. We need the
timestamp in order to calculate OHLCVMinute.
*/
let OHLCV = {
timestamp: (new Date()).valueOf() + TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS,
open: 0,
hight: 0,
@@ -462,9 +524,16 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
volume: 0
}
let record = allOHLCVs[i]
let record = rawDataArray[ohlcvArrayIndex]
if (record !== undefined) { // can be undefined if there is no candle fetched from the exchange
/*
We will check that we can have a record to
analize. It might happen that we don't have one
in the situation that we could not get a single
record from the exchange. We still want to be
here so that everything is properly logged.
*/
if (record !== undefined) {
OHLCV = {
timestamp: record[0],
open: record[1],
@@ -477,48 +546,73 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
let candleMinute = Math.trunc(candle.begin / TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS)
let OHLCVMinute
/*
Some exchanges return inconsistent data. It is not guaranteed
that each candle will have a timeStamp exactly at the begining of an
UTC minute. It is also not guaranteed that the distance
between timestamps will be the same. To fix this, we will do this.
*/
checkOHLCVMinute()
OHLCVMinute = Math.trunc(OHLCV.timestamp / TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS)
function checkOHLCVMinute() {
/*
Some exchanges return inconsistent data. It is not guaranteed that each candle will have a timeStamp exactly at the begining of an
UTC minute. It is also not guaranteed that the distance between timestamps will be the same. To fix this, we will do this.
*/
/*
If the minute of the record item received from the exchange is
less than the minute of the current minute in our loop,
that means that we need to reposition the inded at the rawDataArray
array, moving it one record fordward, and that is what we are
doing here.
*/
while (OHLCVMinute < candleMinute) {
OHLCVMinute = Math.trunc(OHLCV.timestamp / TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS)
/* Move fordward at the rawDataArray array. */
ohlcvArrayIndex++
if (OHLCVMinute < candleMinute) {
if (i >= allOHLCVs.length - 1) {
/* We run out of OHLCVs, we can not move to the next OHLCV, we leave this function. */
return
}
i++
record = allOHLCVs[i]
OHLCV = {
timestamp: record[0],
open: record[1],
hight: record[2],
low: record[3],
close: record[4],
volume: record[5],
id: record[6]
}
setImmediate (checkOHLCVMinute)
/* Check that we have not passed the end of the array */
if (ohlcvArrayIndex > rawDataArray.length - 1) {
/*
We run out of OHLCVs, we can not move to the next OHLCV,
we need to leave this loop.
*/
break
}
record = rawDataArray[ohlcvArrayIndex]
/*
Once this loop is broken, this is the OHLCV that needs
to be considered. All the ones in the past are ignored.
*/
OHLCV = {
timestamp: record[0],
open: record[1],
hight: record[2],
low: record[3],
close: record[4],
volume: record[5],
id: record[6]
}
/* Recalculate this to see if we need to break the loop*/
OHLCVMinute = Math.trunc(OHLCV.timestamp / TS.projects.superalgos.globals.timeConstants.ONE_MIN_IN_MILISECONDS)
}
/* Reporting we are doing well */
heartBeatCounter--
if (heartBeatCounter <= 0) {
heartBeatCounter = 1440
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> saveOHLCVs -> After Fetch -> Saving OHLCVs @ " + processingDate + " -> i = " + i + " -> total = " + allOHLCVs.length)
TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, "Saving " + i.toFixed(0) + " / " + allOHLCVs.length + " OHLCVs from " + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.name + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
}
/* End Reporting */
/*
If the candleMinute and the OHLCVMinute matches, then
we transfer the properties of the OHLCV into the
candle object and the volume object. Two things to
consider here:
1. If they do not match, at this point it could only means
that the OHLCVMinute is in the future, in which case the
candle and volume will keep their initialization values
which in turn are equal to the latest candle and volumes.
2. They might be equal even though the OHLCV timestamp
did not match exactly the UTC minute, but since we are
comparing truncated values, then we force the matching
and we correct the shifting in time that sometimes happens
with exchange data.
*/
if (candleMinute === OHLCVMinute) {
candle.open = OHLCV.open
candle.close = OHLCV.close
@@ -527,14 +621,25 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
volume.buy = OHLCV.volume / 2
volume.sell = OHLCV.volume / 2
if (i < allOHLCVs.length - 1) {
i++
/*
Since we extracted this OHLCV value, we move
fordward our array index.
*/
if (ohlcvArrayIndex < rawDataArray.length - 1) {
ohlcvArrayIndex++
} else {
endOfTheOHLCVArrayReached = true
}
lastId = OHLCV.id
}
/*
Here we remeber the last candle and volumen, in case
we need it.
*/
lastCandle = candle
lastVolume = volume
if (needSeparator === false) {
needSeparator = true;
@@ -548,14 +653,40 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
volumesFileContent = volumesFileContent + separator + '[' + volume.buy + "," + volume.sell + "," + volume.begin + "," + volume.end + "]";
/* We store the last candle of the day in order to have a previous candles during next execution. */
if (j === 1440 - 1) {
if (minuteOfTheDay === 1440 - 1) {
lastCandleOfTheDay = JSON.parse(JSON.stringify(candle))
}
/* Reporting we are doing well */
heartBeatCounter--
if (heartBeatCounter <= 0) {
heartBeatCounter = 1440
logAndHeartBeat()
}
function logAndHeartBeat() {
/* We need thi processing date for logging purposes only */
let processingDate = new Date(candle.begin)
processingDate =
processingDate.getUTCFullYear() + '-' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processingDate.getUTCMonth() + 1, 2) + '-' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processingDate.getUTCDate(), 2);
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> saveOHLCVs -> Before Fetch -> Saving OHLCVs @ " + processingDate + " -> ohlcvArrayIndex = " + ohlcvArrayIndex + " -> total = " + rawDataArray.length)
TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, "Saving " + (ohlcvArrayIndex + 1).toFixed(0) + " / " + rawDataArray.length + " OHLCVs from " + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.name + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
}
}
previousDay = currentDay
if (i > 0) { // Only start saving at the day of the first candle.
/*
When the bot is processing historical information, it
happens that during the whole processing of one day
it did not reach either the head of the market (a future time)
nor the end of the Raw Data Array. In this situation
we still need to save the full day of content, and
we do it only if at least one candle has been processed.
*/
if (ohlcvArrayIndex > 0) {
saveFile(currentDay)
return
}
@@ -565,7 +696,7 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
function saveFile(day) {
candlesFileContent = candlesFileContent + ']'
volumesFileContent = volumesFileContent + ']'
let fileName = 'Data.json'
filesToCreate++
@@ -577,7 +708,6 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
candlesFileContent = '['
volumesFileContent = '['
needSeparator = false
}
function onFileCreated(err) {
@@ -605,39 +735,60 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
}
function controlLoop() {
/*
This loop advances one day, and if it does not need to stop
for any reason, it will execute the loop function so as to process
the next day.
*/
currentDay++
/*
It might have happened that the User is stopping the Task. If that
is the case, we need to stop this processing here.
*/
if (TS.projects.superalgos.globals.taskVariables.IS_TASK_STOPPING === true) {
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE);
return
}
/*
If we had any problem saving the latest file, we will also abort
this process here, so that we can record our progress until it stopped
working.
*/
if (error) {
callBackFunction(error);
return;
}
currentDay++
/*
One possible exit is when we reached the amount of candles downloaded. This not necesary happens at the end of the market
if the process was canceled for any reason at the middle.
One possible exit is when we reached the amount of candles downloaded.
This does not necesary happens at the end of the market
if the process was canceled for any reason at the middle,
or the exchange became unavailable.
*/
if (i >= allOHLCVs.length - 1) {
if (ohlcvArrayIndex >= rawDataArray.length - 1) {
writeStatusReport()
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> saveOHLCVs -> controlLoop -> Exit because i reached the end of the candles array. ")
"[INFO] start -> saveOHLCVs -> controlLoop -> Exit because i reached the end of the rawDataArray array. ")
return
}
/* The natural exit is at the end of the market */
if (headOfTheMarketReached === false) {
setImmediate(loop)
} else {
/*
The most common exit is at the end of the market. There is
nothing else to save, so we finish here.
*/
if (savingProcedureFinished === true) {
writeStatusReport()
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> saveOHLCVs -> controlLoop -> Exit because we reached the end of the market. ")
writeStatusReport()
return
}
/*
If there is no reson to exit, we will process the next day.
*/
setImmediate(loop)
}
} catch (err) {