fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Mati Changes to run in develop

Browse Source
This commit is contained in:
Luis Fernando Molina committed 2019-04-01 08:33:07 +02:00
1 parent cccbc4002a
commit b0987d0961
2 files changed
+59 -68
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Poloniex-Hole-Fixing/User.bot.js
+56 -65
View File
@@ -1,4 +1,4 @@
exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, STATUS_REPORT, POLONIEX_CLIENT_MODULE) {
exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, BLOB_STORAGE, STATUS_REPORT, EXCHANGE_API) {
const FULL_LOG = true;
const LOG_FILE_CONTENT = false;
@@ -20,8 +20,7 @@
let charlyStorage = BLOB_STORAGE.newBlobStorage(bot, logger);
let utilities = UTILITIES.newCloudUtilities(bot, logger);
let poloniexApiClient = POLONIEX_CLIENT_MODULE.newPoloniexAPIClient(global.EXCHANGE_KEYS[global.EXCHANGE_NAME].Key, global.EXCHANGE_KEYS[global.EXCHANGE_NAME].Secret);
let statusReportModule = STATUS_REPORT.newStatusReport(bot, logger, BLOB_STORAGE, UTILITIES);
let year;
@@ -98,43 +97,43 @@
}
}
/*
This process is going to do the following:
It will try to find "holes" on the transaction history and fix them by retrieving the missing transactions from the exchange.
What are holes?
Holes are missing transactions in the history of the market and the reason there can be holes are the following:
1) The "Lives Trades" process can be stopped and later started again. The process will only retrieve live trades, leaving a "hole" between it was stopped and restarted.
2) The "Historic Trades" process might have received a "hole" of transactions directly from the exchange.
3) The "Historic Trades" process failed to write some files to the storage and now they are missing, together with the transaction in them.
4) The "Historic Trades" process wrote some files to the storage with corrupted content.
5) The "Historic Trades" process wrote files with transaction not properly ordered by Trade Id (sometimes the exchange sent this wrong).
How does the process work?
Transactions have a unique id generated by the exchange. It is a secuencial number. This process scans the transaction history looking for missing ids. When it finds
some group of missing ids, it request the exchange the missing records and it writes the necesary files to fix the hole. But some things might happen:
1) The exchange responds, but does not provides the missing records. In this case the process will flag this hole for retry later, up to 3 times. After that it will flag
the hole as "permanent".
2) The exchange call timeouts: the process will continue retrying later.
3) The exchange returns an unexpected error: The process will retry up to 3 times later under this condition.
4) The exchange returns neighboring transactions but not the missing ones. The process will retry 3 more times and mark the hole as permanent if the exchange refuses to send the missing trades.
Since the process is run in an infinite loop but can be restarted any time, the status of its running is recorded in the Status Report file.
What is the lastFile pointer?
It is the Datetime since the begining of the hitory of a market that is considered without holes (or only with permanent ones, unfixables). In other words, the process
starts from the begining of the history of a market, and only when its history is signaled as "Complete" by the "Historic Trades" process. From there, the process is advancing
in time, validating the history and fixing holes, and moving the lastFile pointer forward as it gets sure that all the previous trades files are with no holes or unfixable ones.
This pointer will later be used by Indicators bots that depends of these trades, and it will be considered the head of the market.
*/
/*
This process is going to do the following:
It will try to find "holes" on the transaction history and fix them by retrieving the missing transactions from the exchange.
What are holes?
Holes are missing transactions in the history of the market and the reason there can be holes are the following:
1) The "Lives Trades" process can be stopped and later started again. The process will only retrieve live trades, leaving a "hole" between it was stopped and restarted.
2) The "Historic Trades" process might have received a "hole" of transactions directly from the exchange.
3) The "Historic Trades" process failed to write some files to the storage and now they are missing, together with the transaction in them.
4) The "Historic Trades" process wrote some files to the storage with corrupted content.
5) The "Historic Trades" process wrote files with transaction not properly ordered by Trade Id (sometimes the exchange sent this wrong).
How does the process work?
Transactions have a unique id generated by the exchange. It is a secuencial number. This process scans the transaction history looking for missing ids. When it finds
some group of missing ids, it request the exchange the missing records and it writes the necesary files to fix the hole. But some things might happen:
1) The exchange responds, but does not provides the missing records. In this case the process will flag this hole for retry later, up to 3 times. After that it will flag
the hole as "permanent".
2) The exchange call timeouts: the process will continue retrying later.
3) The exchange returns an unexpected error: The process will retry up to 3 times later under this condition.
4) The exchange returns neighboring transactions but not the missing ones. The process will retry 3 more times and mark the hole as permanent if the exchange refuses to send the missing trades.
Since the process is run in an infinite loop but can be restarted any time, the status of its running is recorded in the Status Report file.
What is the lastFile pointer?
It is the Datetime since the begining of the hitory of a market that is considered without holes (or only with permanent ones, unfixables). In other words, the process
starts from the begining of the history of a market, and only when its history is signaled as "Complete" by the "Historic Trades" process. From there, the process is advancing
in time, validating the history and fixing holes, and moving the lastFile pointer forward as it gets sure that all the previous trades files are with no holes or unfixable ones.
This pointer will later be used by Indicators bots that depends of these trades, and it will be considered the head of the market.
*/
function start(callBackFunction) {
@@ -265,7 +264,7 @@ What is the lastFile pointer?
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
callBackFunction(customOK);
return;
}
}
} else {
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Trade History is not complete.");
@@ -278,7 +277,7 @@ What is the lastFile pointer?
callBackFunction(customOK);
return;
}
reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Poloniex-Hole-Fixing" + "-" + "dataSet.V1" + "-" + year + "-" + month;
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
@@ -377,7 +376,7 @@ What is the lastFile pointer?
let fileCheckedCounter = 0;
date = new Date(lastHoleFixingFile.valueOf());
date = new Date(lastHoleFixingFile.valueOf());
readNextFile();
@@ -421,7 +420,7 @@ What is the lastFile pointer?
return;
}
/* Lets check if we have reached the end of the month. */
/* Lets check if we have reached the end of the month. */
if (date.valueOf() > lastMinuteOfMonth.valueOf()) {
@@ -458,7 +457,7 @@ What is the lastFile pointer?
}
dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2);
filePath = bot.filePathRoot + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath;
charlyStorage.getTextFile(filePath, fileName, onNextFileReceived, true);
@@ -490,7 +489,7 @@ What is the lastFile pointer?
findEndOfHole();
return;
}
}
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
try {
@@ -548,11 +547,11 @@ What is the lastFile pointer?
We will need these trades at the end of the process.
*/
tradesWithHole = trades;
tradesWithHole = trades;
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findNextHole -> checkHolesInFile -> Checking File '" + fileName + "' @ " + filePath + " - " + trades.length + " records in it."); }
if (currentTradeId === FIRST_TRADE_RECORD_ID || currentTradeId === UNKNOWN_TRADE_RECORD_ID) {
if (currentTradeId === FIRST_TRADE_RECORD_ID || currentTradeId === UNKNOWN_TRADE_RECORD_ID) {
/*
Here we dont know the currentTradeId, so we might take it directly from the file. There will be a problem though if the market starts
@@ -714,7 +713,7 @@ What is the lastFile pointer?
return;
}
callBackFunction(global.DEFAULT_OK_RESPONSE);
callBackFunction(global.DEFAULT_OK_RESPONSE);
return;
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> findNextHole -> findEndOfHole -> onStatusReportWritten -> err = " + err.message);
@@ -753,10 +752,9 @@ What is the lastFile pointer?
findEndOfHole();
return;
}
}
if (err.result === global.DEFAULT_OK_RESPONSE.result)
{
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
let trades;
try {
@@ -771,7 +769,7 @@ What is the lastFile pointer?
findEndOfHole();
return;
}
}
if (trades.length === 0) {
@@ -835,9 +833,7 @@ What is the lastFile pointer?
exchangeCallTime = new Date();
let poloniexApiClient = POLONIEX_CLIENT_MODULE.newPoloniexAPIClient(global.EXCHANGE_KEYS[global.EXCHANGE_NAME].Key, global.EXCHANGE_KEYS[global.EXCHANGE_NAME].Secret);
poloniexApiClient.API.returnPublicTradeHistory(market.assetA, market.assetB, startTime, endTime, onExchangeCallReturned);
EXCHANGE_API.getPublicTradeHistory(market.assetA, market.assetB, startTime, endTime, onExchangeCallReturned);
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> getTheTrades -> err = " + err.message);
@@ -859,12 +855,7 @@ What is the lastFile pointer?
logger.write(MODULE_NAME, "[INFO] start -> onExchangeCallReturned -> Call time recorded = " + timeDifference + " seconds.");
}
poloniexApiClient.API.analizeResponse(logger, err, exchangeResponse, callBackFunction, onResponseOk);
function onResponseOk() {
tradesReadyToBeSaved(exchangeResponse);
}
tradesReadyToBeSaved(exchangeResponse);
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> onExchangeCallReturned -> err = " + err.message);
@@ -1104,9 +1095,9 @@ What is the lastFile pointer?
logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Content written = " + fileContent);
}
logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Finished with File @ " + market.assetA + "_" + market.assetB);
logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Records inserted = " + fileRecordCounter);
logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Path = " + filePath + "/" + fileName + "");
logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Finished with File @ " + market.assetA + "_" + market.assetB);
logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Records inserted = " + fileRecordCounter);
logger.write(MODULE_NAME, "[INFO] start -> tradesReadyToBeSaved -> nextRecord -> onFileCreated -> Path = " + filePath + "/" + fileName + "");
controlLoop();
this.bot.config.json
+3 -3
View File
@@ -57,12 +57,12 @@
"allMonths": {
"run": "false",
"minYear": "2014",
"maxYear": "2018"
"maxYear": "2020"
},
"oneMonth": {
"run": "true",
"year": "2018",
"month": "04"
"month": "01"
},
"noTime": {
"run": "false"
@@ -106,7 +106,7 @@
"allMonths": {
"run": "true",
"minYear": "2018",
"maxYear": "2018"
"maxYear": "2019"
},
"oneMonth": {
"run": "false",