COMMONS module introduced

This commit is contained in:
Luis Fernando Molina committed 2018-03-15 20:31:57 +01:00
1 parent db97ce597b
commit e134af2dd3
6 files changed
+415 -433

No files matched your search

Binary file not shown.
+264
View File
@@ -0,0 +1,264 @@
exports.newCommons = function newCommons(BOT, DEBUG_MODULE, UTILITIES) {
const FULL_LOG = true;
const LOG_FILE_CONTENT = false;
const MODULE_NAME = "Commons";
const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000;
const GMT_SECONDS = ':00.000 GMT+0000';
let thisObject = {
initializeStorage: initializeStorage,
getContextVariables: getContextVariables
};
let bot = BOT;
const logger = DEBUG_MODULE.newDebugLog();
logger.fileName = MODULE_NAME;
logger.bot = bot;
let utilities = UTILITIES.newUtilities(bot);
return thisObject;
function initializeStorage(charlyFileStorage, bruceFileStorage, oliviaFileStorage, callBackFunction) {
try {
if (FULL_LOG === true) { logger.write("[INFO] initializeStorage -> Entering function."); }
initializeCharlyStorage();
function initializeCharlyStorage() {
charlyFileStorage.initialize("AACharly", onCharlyInizialized);
function onCharlyInizialized(err) {
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
initializeBruceStorage();
} else {
logger.write("[ERROR] initializeStorage -> initializeCharlyStorage -> onCharlyInizialized -> err = " + err.message);
callBackFunction(err);
}
}
}
function initializeBruceStorage() {
bruceFileStorage.initialize("AABruce", onBruceInizialized);
function onBruceInizialized(err) {
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
initializeOliviaStorage();
} else {
logger.write("[ERROR] initializeStorage -> initializeBruceStorage -> onBruceInizialized -> err = " + err.message);
callBackFunction(err);
}
}
}
function initializeOliviaStorage() {
oliviaFileStorage.initialize("AAOlivia", onOliviaInizialized);
function onOliviaInizialized(err) {
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
callBackFunction(global.DEFAULT_OK_RESPONSE);
} else {
logger.write("[ERROR] initializeStorage -> initializeOliviaStorage -> onOliviaInizialized -> err = " + err.message);
callBackFunction(err);
}
}
}
}
catch (err) {
logger.write("[ERROR] initializeStorage -> err = " + err.message);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function getContextVariables(charlyFileStorage, bruceFileStorage, oliviaFileStorage, contextVariables, statusReportModule, callBackFunction) {
try {
if (FULL_LOG === true) { logger.write("[INFO] getContextVariables -> Entering function."); }
let reportFilePath;
let fileName = "Status.Report." + global.MARKET.assetA + '_' + global.MARKET.assetB + ".json"
if (FULL_LOG === true) { logger.write("[INFO] getContextVariables -> fileName = " + fileName); }
getHistoricTrades();
function getHistoricTrades() {
if (FULL_LOG === true) { logger.write("[INFO] getContextVariables -> getHistoricTrades -> Entering function."); }
/*
We need to know where is the begining of the market, since that will help us know where the Index Files should start.
*/
reportFilePath = EXCHANGE_NAME + "/Processes/" + "Poloniex-Historic-Trades";
if (FULL_LOG === true) { logger.write("[INFO] getContextVariables -> getHistoricTrades -> reportFilePath = " + reportFilePath); }
charlyFileStorage.getTextFile(reportFilePath, fileName, onFileReceived);
function onFileReceived(err, text) {
if (FULL_LOG === true) { logger.write("[INFO] getContextVariables -> getHistoricTrades -> onFileReceived -> Entering function."); }
if (FULL_LOG === true) { logger.write("[INFO] getContextVariables -> getHistoricTrades -> onFileReceived -> text = " + text); }
let thisReport;
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
try {
thisReport = JSON.parse(text);
contextVariables.firstTradeFile = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS);
getOneMinDailyCandlesVolumes();
} catch (err) {
logger.write("[ERROR] getContextVariables -> getHistoricTrades -> onFileReceived -> err = " + err.message);
logger.write("[ERROR] getContextVariables -> getHistoricTrades -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
}
} else {
logger.write("[ERROR] getContextVariables -> getHistoricTrades -> onFileReceived -> err = " + err.message);
callBackFunction(err);
}
}
}
function getOneMinDailyCandlesVolumes() {
if (FULL_LOG === true) { logger.write("[INFO] getContextVariables -> getOneMinDailyCandlesVolumes -> Entering function."); }
/* We need to discover the maxCandle file, which is the last file with candles we can use as input. */
let date = new Date();
let currentYear = date.getUTCFullYear();
let currentMonth = utilities.pad(date.getUTCMonth() + 1, 2);
reportFilePath = EXCHANGE_NAME + "/Processes/" + "One-Min-Daily-Candles-Volumes" + "/" + currentYear + "/" + currentMonth;
if (FULL_LOG === true) { logger.write("[INFO] getContextVariables -> getOneMinDailyCandlesVolumes -> reportFilePath = " + reportFilePath); }
bruceFileStorage.getTextFile(reportFilePath, fileName, onFileReceived);
function onFileReceived(err, text) {
if (FULL_LOG === true) { logger.write("[INFO] getContextVariables -> getOneMinDailyCandlesVolumes -> onFileReceived -> Entering function."); }
if (FULL_LOG === true) { logger.write("[INFO] getContextVariables -> getOneMinDailyCandlesVolumes -> onFileReceived -> text = " + text); }
let thisReport;
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
try {
thisReport = JSON.parse(text);
contextVariables.maxCandleFile = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + "00:00" + GMT_SECONDS);
getThisProcessReport();
} catch (err) {
logger.write("[ERROR] getContextVariables -> getOneMinDailyCandlesVolumes -> onFileReceived -> err = " + err.message);
logger.write("[ERROR] getContextVariables -> getOneMinDailyCandlesVolumes -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
}
} else {
logger.write("[ERROR] getContextVariables -> getHistoricTrades -> onFileReceived -> err = " + err.message);
callBackFunction(err);
}
}
}
function getThisProcessReport() {
if (FULL_LOG === true) { logger.write("[INFO] getContextVariables -> getThisProcessReport -> Entering function."); }
statusReportModule.initialize(bot, onInitilized);
function onInitilized(err) {
try {
switch (err.result) {
case global.DEFAULT_OK_RESPONSE.result: {
logger.write("[INFO] getContextVariables -> getThisProcessReport -> onInitilized -> Execution finished well. :-)");
contextVariables.lastCandleFile = new Date(statusReportModule.file.lastFile);
/*
Here we assume that the last day written might contain incomplete information. This actually happens every time the head of the market is reached.
For that reason we go back one day, the partial information is discarded and added again with whatever new info is available.
*/
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS);
let customOk = {
result: global.CUSTOM_OK_RESPONSE.result,
message: "Status Report did exist."
};
callBackFunction(customOk);
return;
}
case global.CUSTOM_FAIL_RESPONSE.result: { // We need to see if we can handle this.
logger.write("[ERROR] getContextVariables -> getThisProcessReport -> onInitilized -> err.message = " + err.message);
if (err.message === "Status Report was never created.") {
contextVariables.lastCandleFile = new Date(contextVariables.firstTradeFile.getUTCFullYear() + "-" + (contextVariables.firstTradeFile.getUTCMonth() + 1) + "-" + contextVariables.firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well.
let customOk = {
result: global.CUSTOM_OK_RESPONSE.result,
message: "Status Report did not exist."
};
callBackFunction(customOk);
return;
} else {
callBackFunction(err); // we cant handle this here.
}
return;
}
default:
{
logger.write("[ERROR] getContextVariables -> getThisProcessReport -> onInitilized -> Operation Failed.");
callBackFunction(err);
return;
}
}
}
catch (err) {
logger.write("[ERROR] getContextVariables -> getThisProcessReport -> onInitilized -> err = " + err.message);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
}
catch (err) {
logger.write("[ERROR] getContextVariables -> err = " + err.message);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
};
@@ -1,4 +1,7 @@
exports.newInterval = function newInterval(BOT, UTILITIES, AZURE_FILE_STORAGE, DEBUG_MODULE, MARKETS_MODULE, POLONIEX_CLIENT_MODULE) {
exports.newUserBot = function newUserBot(BOT, COMMONS, UTILITIES, DEBUG_MODULE, FILE_STORAGE, STATUS_REPORT) {
const FULL_LOG = true;
const LOG_FILE_CONTENT = false;
let bot = BOT;
@@ -6,11 +9,9 @@
const GMT_MILI_SECONDS = '.000 GMT+0000';
const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000;
const MODULE_NAME = "Interval";
const LOG_INFO = true;
const MODULE_NAME = "UserBot";
const EXCHANGE_NAME = "Poloniex";
const EXCHANGE_ID = 1;
const TRADES_FOLDER_NAME = "Trades";
@@ -20,27 +21,27 @@
const VOLUMES_FOLDER_NAME = "Volumes";
const VOLUMES_ONE_MIN = "One-Min";
const GO_RANDOM = false;
const FORCE_MARKET = 2; // This allows to debug the execution of an specific market. Not intended for production.
const logger = DEBUG_MODULE.newDebugLog();
logger.fileName = MODULE_NAME;
logger.bot = bot;
interval = {
const commons = COMMONS.newCommons(bot, DEBUG_MODULE);
thisObject = {
initialize: initialize,
start: start
};
let markets;
let charlyAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
let bruceAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
let oliviaAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
let charlyFileStorage = FILE_STORAGE.newAzureFileStorage(bot);
let bruceFileStorage = FILE_STORAGE.newAzureFileStorage(bot);
let oliviaFileStorage = FILE_STORAGE.newAzureFileStorage(bot);
let utilities = UTILITIES.newUtilities(bot);
return interval;
let statusReport;
let statusReportModule;
return thisObject;
function initialize(yearAssigend, monthAssigned, callBackFunction) {
@@ -48,30 +49,72 @@
/* IMPORTANT NOTE:
We are ignoring in this Interval the received Year and Month. This interval is not depending on Year Month since it procecess the whole market at once.
We are ignoring in this UserBot the received Year and Month. thisObject is not depending on Year Month since it procecess the whole market at once.
*/
logger.fileName = MODULE_NAME;
const logText = "[INFO] initialize - Entering function 'initialize' ";
console.log(logText);
logger.write(logText);
if (FULL_LOG === true) { logger.write("[INFO] initialize -> Entering function."); }
if (FULL_LOG === true) { logger.write("[INFO] initialize -> yearAssigend = " + yearAssigend); }
if (FULL_LOG === true) { logger.write("[INFO] initialize -> monthAssigned = " + monthAssigned); }
charlyAzureFileStorage.initialize("Charly");
bruceAzureFileStorage.initialize("Bruce");
oliviaAzureFileStorage.initialize("Olivia");
initializeCharlyStorage();
markets = MARKETS_MODULE.newMarkets(bot);
markets.initialize(callBackFunction);
function initializeCharlyStorage() {
charlyFileStorage.initialize("AACharly", onCharlyInizialized);
function onCharlyInizialized(err) {
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
initializeBruceStorage();
} else {
logger.write("[ERROR] initialize -> initializeCharlyStorage -> onCharlyInizialized -> err = " + err.message);
callBackFunction(err);
}
}
}
function initializeBruceStorage() {
bruceFileStorage.initialize("AABruce", onBruceInizialized);
function onBruceInizialized(err) {
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
initializeOliviaStorage();
} else {
logger.write("[ERROR] initialize -> initializeBruceStorage -> onBruceInizialized -> err = " + err.message);
callBackFunction(err);
}
}
}
function initializeOliviaStorage() {
oliviaFileStorage.initialize("AAOlivia", onOliviaInizialized);
function onOliviaInizialized(err) {
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
callBackFunction(global.DEFAULT_OK_RESPONSE);
} else {
logger.write("[ERROR] initialize -> initializeOliviaStorage -> onOliviaInizialized -> err = " + err.message);
callBackFunction(err);
}
}
}
} catch (err) {
const logText = "[ERROR] initialize - ' ERROR : " + err.message;
console.log(logText);
logger.write(logText);
logger.write("[ERROR] initialize -> err = " + err.message);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
@@ -79,7 +122,7 @@
This process is going to do the following:
Read the candles and volumes from Bruce and produce a single Index File for each market with daily candles and volumes.
Read the candles and volumes from Bruce and produce a file for each day and for each period with candles and volumes.
*/
@@ -87,171 +130,18 @@ Read the candles and volumes from Bruce and produce a single Index File for each
try {
if (LOG_INFO === true) {
logger.write("[INFO] Entering function 'start'");
}
if (FULL_LOG === true) { logger.write("[INFO] start -> Entering function."); }
let nextIntervalExecution = false; // This tell weather the Interval module will be executed again or not. By default it will not unless some hole have been found in the current execution.
let nextIntervalLapse; // With this we can request the next execution wait time.
let market = global.MARKET;
let marketQueue; // This is the queue of all markets to be procesesd at each interval.
let market = { // This is the current market being processed after removing it from the queue.
id: 0,
assetA: "",
assetB: ""
};
let lastCandleFile; // Datetime of the last file included on the Index File.
let firstTradeFile; // Datetime of the first trade file in the whole market history.
let maxCandleFile; // Datetime of the last file available to be included in the Index File.
let periods =
'[' +
'[' + 45 * 60 * 1000 + ',' + '"45-min"' + ']' + ',' +
'[' + 40 * 60 * 1000 + ',' + '"40-min"' + ']' + ',' +
'[' + 30 * 60 * 1000 + ',' + '"30-min"' + ']' + ',' +
'[' + 20 * 60 * 1000 + ',' + '"20-min"' + ']' + ',' +
'[' + 15 * 60 * 1000 + ',' + '"15-min"' + ']' + ',' +
'[' + 10 * 60 * 1000 + ',' + '"10-min"' + ']' + ',' +
'[' + 05 * 60 * 1000 + ',' + '"05-min"' + ']' + ',' +
'[' + 04 * 60 * 1000 + ',' + '"04-min"' + ']' + ',' +
'[' + 03 * 60 * 1000 + ',' + '"03-min"' + ']' + ',' +
'[' + 02 * 60 * 1000 + ',' + '"02-min"' + ']' + ',' +
'[' + 01 * 60 * 1000 + ',' + '"01-min"' + ']' + ']';
const outputPeriods = JSON.parse(periods);
marketsLoop();
/*
At every run, the process needs to loop through all the markets at this exchange.
The following functions marketsLoop(), openMarket(), closeMarket() and closeAndOpenMarket() controls the serialization of this processing.
*/
function marketsLoop() {
try {
if (LOG_INFO === true) {
logger.write("[INFO] Entering function 'marketsLoop'");
}
markets.getMarketsByExchange(EXCHANGE_ID, onMarketsReady);
function onMarketsReady(marketsArray) {
marketQueue = JSON.parse(marketsArray);
openMarket(); // First execution and entering into the real loop.
}
}
catch (err) {
const logText = "[ERROR] 'marketsLoop' - ERROR : " + err.message;
logger.write(logText);
}
}
function openMarket() {
// To open a Market means picking a new market from the queue.
try {
if (LOG_INFO === true) {
logger.write("[INFO] Entering function 'openMarket'");
}
if (marketQueue.length === 0) {
if (LOG_INFO === true) {
logger.write("[INFO] 'openMarket' - marketQueue.length === 0");
}
const logText = "[WARN] We processed all the markets.";
logger.write(logText);
callBackFunction(nextIntervalExecution, nextIntervalLapse);
return;
}
if (GO_RANDOM === true) {
const index = parseInt(Math.random() * (marketQueue.length - 1));
market.id = marketQueue[index][0];
market.assetA = marketQueue[index][1];
market.assetB = marketQueue[index][2];
market.status = marketQueue[index][3];
marketQueue.splice(index, 1);
} else {
let marketRecord = marketQueue.shift();
market.id = marketRecord[0];
market.assetA = marketRecord[1];
market.assetB = marketRecord[2];
market.status = marketRecord[3];
if (FORCE_MARKET > 0) {
if (FORCE_MARKET !== market.id) {
closeAndOpenMarket();
return;
}
}
}
if (LOG_INFO === true) {
logger.write("[INFO] 'openMarket' - marketQueue.length = " + marketQueue.length);
logger.write("[INFO] 'openMarket' - market sucessfully opened : " + market.assetA + "_" + market.assetB);
}
if (market.status === markets.ENABLED) {
getStatusReport();
} else {
logger.write("[INFO] 'openMarket' - market " + market.assetA + "_" + market.assetB + " skipped because its status is not valid. Status = " + market.status);
closeAndOpenMarket();
return;
}
}
catch (err) {
const logText = "[ERROR] 'openMarket' - ERROR : " + err.message;
logger.write(logText);
closeMarket();
}
}
function closeMarket() {
if (LOG_INFO === true) {
logger.write("[INFO] Entering function 'closeMarket'");
}
}
function closeAndOpenMarket() {
if (LOG_INFO === true) {
logger.write("[INFO] Entering function 'closeAndOpenMarket'");
}
openMarket();
}
/*
The following code executes for each market.
*/
function getStatusReport() {
@@ -272,7 +162,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each
reportFilePath = EXCHANGE_NAME + "/Processes/" + "Poloniex-Historic-Trades";
charlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
charlyFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
function onStatusReportReceived(text) {
@@ -306,7 +196,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each
reportFilePath = EXCHANGE_NAME + "/Processes/" + "One-Min-Daily-Candles-Volumes" + "/" + currentYear + "/" + currentMonth;
bruceAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
bruceFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
function onStatusReportReceived(text) {
@@ -342,7 +232,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each
reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process;
oliviaAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
oliviaFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
function onStatusReportReceived(text) {
@@ -404,7 +294,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each
if (lastCandleFile.valueOf() > maxCandleFile.valueOf()) {
nextIntervalExecution = true; // we request a new interval execution.
nextIntervalExecution = true; // we request a new thisObject execution.
const logText = "[INFO] 'buildCandles' - Head of the market found @ " + lastCandleFile.getUTCFullYear() + "/" + (lastCandleFile.getUTCMonth() + 1) + "/" + lastCandleFile.getUTCDate() + ".";
logger.write(logText);
@@ -463,7 +353,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each
let fileName = market.assetA + '_' + market.assetB + ".json"
let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
bruceFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
function onFileReceived(text) {
@@ -480,7 +370,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each
closeAndOpenMarket();
nextIntervalExecution = true; // we request a new interval execution.
nextIntervalExecution = true; // we request a new thisObject execution.
nextIntervalLapse = 30000;
return;
@@ -567,7 +457,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each
let fileName = market.assetA + '_' + market.assetB + ".json"
let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath;
bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
bruceFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
function onFileReceived(text) {
@@ -705,11 +595,11 @@ Read the candles and volumes from Bruce and produce a single Index File for each
let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + "/" + bot.process + "/" + folderName + "/" + dateForPath;
utilities.createFolderIfNeeded(filePath, oliviaAzureFileStorage, onFolderCreated);
utilities.createFolderIfNeeded(filePath, oliviaFileStorage, onFolderCreated);
function onFolderCreated() {
oliviaAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated);
oliviaFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated);
function onFileCreated() {
@@ -750,11 +640,11 @@ Read the candles and volumes from Bruce and produce a single Index File for each
let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + "/" + bot.process + "/" + folderName + "/" + dateForPath;
utilities.createFolderIfNeeded(filePath, oliviaAzureFileStorage, onFolderCreated);
utilities.createFolderIfNeeded(filePath, oliviaFileStorage, onFolderCreated);
function onFolderCreated() {
oliviaAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated);
oliviaFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated);
function onFileCreated() {
@@ -785,7 +675,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each
let reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process;
utilities.createFolderIfNeeded(reportFilePath, oliviaAzureFileStorage, onFolderCreated);
utilities.createFolderIfNeeded(reportFilePath, oliviaFileStorage, onFolderCreated);
function onFolderCreated() {
@@ -803,7 +693,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each
let fileContent = JSON.stringify(report);
oliviaAzureFileStorage.createTextFile(reportFilePath, fileName, fileContent + '\n', onFileCreated);
oliviaFileStorage.createTextFile(reportFilePath, fileName, fileContent + '\n', onFileCreated);
function onFileCreated() {
+55 -230
View File
@@ -1,4 +1,4 @@
exports.newUserBot = function newUserBot(BOT, UTILITIES, DEBUG_MODULE, FILE_STORAGE, STATUS_REPORT) {
exports.newUserBot = function newUserBot(BOT, COMMONS, UTILITIES, DEBUG_MODULE, FILE_STORAGE, STATUS_REPORT) {
const FULL_LOG = true;
const LOG_FILE_CONTENT = false;
@@ -12,7 +12,6 @@
const MODULE_NAME = "UserBot";
const EXCHANGE_NAME = "Poloniex";
const EXCHANGE_ID = 1;
const TRADES_FOLDER_NAME = "Trades";
@@ -26,7 +25,9 @@
logger.fileName = MODULE_NAME;
logger.bot = bot;
interval = {
const commons = COMMONS.newCommons(bot, DEBUG_MODULE, UTILITIES);
thisObject = {
initialize: initialize,
start: start
};
@@ -37,10 +38,9 @@
let utilities = UTILITIES.newUtilities(bot);
let statusReport;
let statusReportModule;
let statusReportModule = STATUS_REPORT.newStatusReport(BOT, DEBUG_MODULE, FILE_STORAGE, UTILITIES);
return interval;
return thisObject;
function initialize(yearAssigend, monthAssigned, callBackFunction) {
@@ -48,7 +48,7 @@
/* IMPORTANT NOTE:
We are ignoring in this UserBot the received Year and Month. This interval is not depending on Year Month since it procecess the whole market at once.
We are ignoring in this UserBot the received Year and Month. thisObject is not depending on Year Month since it procecess the whole market at once.
*/
@@ -58,58 +58,7 @@
if (FULL_LOG === true) { logger.write("[INFO] initialize -> yearAssigend = " + yearAssigend); }
if (FULL_LOG === true) { logger.write("[INFO] initialize -> monthAssigned = " + monthAssigned); }
initializeCharlyStorage();
function initializeCharlyStorage() {
charlyFileStorage.initialize("AACharly", onCharlyInizialized);
function onCharlyInizialized(err) {
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
initializeBruceStorage();
} else {
logger.write("[ERROR] initialize -> initializeCharlyStorage -> onCharlyInizialized -> err = " + err.message);
callBackFunction(err);
}
}
}
function initializeBruceStorage() {
bruceFileStorage.initialize("AABruce", onBruceInizialized);
function onBruceInizialized(err) {
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
initializeOliviaStorage();
} else {
logger.write("[ERROR] initialize -> initializeBruceStorage -> onBruceInizialized -> err = " + err.message);
callBackFunction(err);
}
}
}
function initializeOliviaStorage() {
oliviaFileStorage.initialize("AAOlivia", onOliviaInizialized);
function onOliviaInizialized(err) {
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
callBackFunction(global.DEFAULT_OK_RESPONSE);
} else {
logger.write("[ERROR] initialize -> initializeOliviaStorage -> onOliviaInizialized -> err = " + err.message);
callBackFunction(err);
}
}
}
commons.initializeStorage(charlyFileStorage, bruceFileStorage, oliviaFileStorage, callBackFunction);
} catch (err) {
logger.write("[ERROR] initialize -> err = " + err.message);
@@ -135,176 +84,52 @@ Read the candles and volumes from Bruce and produce a single Index File for Mark
/* Context Variables */
let lastCandleFile; // Datetime of the last file included on the Index Files.
let firstTradeFile; // Datetime of the first trade file in the whole market history.
let maxCandleFile; // Datetime of the last file available to be included in the Index File.
contextVariables = {
lastCandleFile: undefined, // Datetime of the last file included on the Index Files.
firstTradeFile: undefined, // Datetime of the first trade file in the whole market history.
maxCandleFile: undefined // Datetime of the last file available to be included in the Index File.
};
getContextVariables();
commons.getContextVariables(charlyFileStorage, bruceFileStorage, oliviaFileStorage, contextVariables, statusReportModule, onVariablesReady);
function getContextVariables() {
function onVariablesReady(err) {
try {
if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> Entering function."); }
if (FULL_LOG === true) { logger.write("[INFO] start -> onVariablesReady -> Entering function."); }
let reportFilePath;
let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json"
if (err.result === global.CUSTOM_OK_RESPONSE.result) {
if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> fileName = " + fileName); }
switch (err.message) {
case 'Status Report did exist.': {
logger.write("[INFO] start -> onVariablesReady -> Status Report did exist.");
getHistoricTrades();
function getHistoricTrades() {
if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> getHistoricTrades -> Entering function."); }
/*
We need to know where is the begining of the market, since that will help us know where the Index Files should start.
*/
reportFilePath = EXCHANGE_NAME + "/Processes/" + "Poloniex-Historic-Trades";
if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> getHistoricTrades -> reportFilePath = " + reportFilePath); }
charlyFileStorage.getTextFile(reportFilePath, fileName, onFileReceived);
function onFileReceived(err, text) {
if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> getHistoricTrades -> onFileReceived -> Entering function."); }
if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> getHistoricTrades -> onFileReceived -> text = " + text); }
let thisReport;
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
try {
thisReport = JSON.parse(text);
firstTradeFile = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS);
getOneMinDailyCandlesVolumes();
} catch (err) {
logger.write("[ERROR] start -> getContextVariables -> getHistoricTrades -> onFileReceived -> err = " + err.message);
logger.write("[ERROR] start -> getContextVariables -> getHistoricTrades -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
}
} else {
logger.write("[ERROR] start -> getContextVariables -> getHistoricTrades -> onFileReceived -> err = " + err.message);
callBackFunction(err);
findPreviousContent();
return;
}
case 'Status Report did not exist.': {
logger.write("[ERROR] start -> onVariablesReady -> Status Report did not exist.");
buildCandles();
return;
}
default:
{
logger.write("[ERROR] start -> onVariablesReady -> Operation Failed.");
callBackFunction(err);
return;
}
}
} else {
logger.write("[ERROR] start -> onVariablesReady -> Operation Failed.");
callBackFunction(err);
return;
}
function getOneMinDailyCandlesVolumes() {
if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> getOneMinDailyCandlesVolumes -> Entering function."); }
/* We need to discover the maxCandle file, which is the last file with candles we can use as input. */
let date = new Date();
let currentYear = date.getUTCFullYear();
let currentMonth = utilities.pad(date.getUTCMonth() + 1,2);
reportFilePath = EXCHANGE_NAME + "/Processes/" + "One-Min-Daily-Candles-Volumes" + "/" + currentYear + "/" + currentMonth;
if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> getOneMinDailyCandlesVolumes -> reportFilePath = " + reportFilePath); }
bruceFileStorage.getTextFile(reportFilePath, fileName, onFileReceived);
function onFileReceived(err, text) {
if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> getOneMinDailyCandlesVolumes -> onFileReceived -> Entering function."); }
if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> getOneMinDailyCandlesVolumes -> onFileReceived -> text = " + text); }
let thisReport;
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
try {
thisReport = JSON.parse(text);
maxCandleFile = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + "00:00" + GMT_SECONDS);
getThisProcessReport();
} catch (err) {
logger.write("[ERROR] start -> getContextVariables -> getOneMinDailyCandlesVolumes -> onFileReceived -> err = " + err.message);
logger.write("[ERROR] start -> getContextVariables -> getOneMinDailyCandlesVolumes -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
}
} else {
logger.write("[ERROR] start -> getContextVariables -> getHistoricTrades -> onFileReceived -> err = " + err.message);
callBackFunction(err);
}
}
}
function getThisProcessReport() {
if (FULL_LOG === true) { logger.write("[INFO] start -> getContextVariables -> getThisProcessReport -> Entering function."); }
statusReportModule = STATUS_REPORT.newStatusReport(BOT, DEBUG_MODULE, FILE_STORAGE, UTILITIES);
statusReportModule.initialize(bot, onInitilized);
function onInitilized(err) {
try {
switch (err.result) {
case global.DEFAULT_OK_RESPONSE.result: {
logger.write("[INFO] start -> getContextVariables -> getThisProcessReport -> onInitilized -> Execution finished well. :-)");
statusReport = statusReportModule.file;
lastCandleFile = new Date(statusReport.lastFile);
/*
Here we assume that the last day written might contain incomplete information. This actually happens every time the head of the market is reached.
For that reason we go back one day, the partial information is discarded and added again with whatever new info is available.
*/
lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS);
findPreviousContent();
return;
}
case global.CUSTOM_FAIL_RESPONSE.result: { // We need to see if we can handle this.
logger.write("[ERROR] start -> getContextVariables -> getThisProcessReport -> onInitilized -> err.message = " + err.message);
if (err.message === "Status Report was never created.") {
statusReport = statusReportModule.file;
lastCandleFile = new Date(firstTradeFile.getUTCFullYear() + "-" + (firstTradeFile.getUTCMonth() + 1) + "-" + firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well.
buildCandles();
} else {
callBackFunction(err); // we cant handle this here.
}
return;
}
default:
{
logger.write("[ERROR] start -> getContextVariables -> getThisProcessReport -> onInitilized -> Operation Failed.");
callBackFunction(err);
return;
}
}
}
catch (err) {
logger.write("[ERROR] start -> getContextVariables -> getThisProcessReport -> onInitilized -> err = " + err.message);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
}
catch (err) {
logger.write("[ERROR] start -> getContextVariables -> err = " + err.message);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
} catch (err) {
logger.write("[ERROR] start -> onVariablesReady -> err = " + err.message);
callBack(global.DEFAULT_FAIL_RESPONSE);
}
}
@@ -471,15 +296,15 @@ Read the candles and volumes from Bruce and produce a single Index File for Mark
if (FULL_LOG === true) { logger.write("[INFO] start -> buildCandles -> advanceTime -> Entering function."); }
lastCandleFile = new Date(lastCandleFile.valueOf() + ONE_DAY_IN_MILISECONDS);
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() + ONE_DAY_IN_MILISECONDS);
if (FULL_LOG === true) { logger.write("[INFO] start -> buildCandles -> advanceTime -> New processing time @ " + lastCandleFile.getUTCFullYear() + "/" + (lastCandleFile.getUTCMonth() + 1) + "/" + lastCandleFile.getUTCDate() + "."); }
if (FULL_LOG === true) { logger.write("[INFO] start -> buildCandles -> advanceTime -> New processing time @ " + contextVariables.lastCandleFile.getUTCFullYear() + "/" + (contextVariables.lastCandleFile.getUTCMonth() + 1) + "/" + contextVariables.lastCandleFile.getUTCDate() + "."); }
/* Validation that we are not going past the head of the market. */
if (lastCandleFile.valueOf() > maxCandleFile.valueOf()) {
if (contextVariables.lastCandleFile.valueOf() > contextVariables.maxCandleFile.valueOf()) {
if (FULL_LOG === true) { logger.write("[INFO] start -> buildCandles -> advanceTime -> Head of the market found @ " + lastCandleFile.getUTCFullYear() + "/" + (lastCandleFile.getUTCMonth() + 1) + "/" + lastCandleFile.getUTCDate() + "."); }
if (FULL_LOG === true) { logger.write("[INFO] start -> buildCandles -> advanceTime -> Head of the market found @ " + contextVariables.lastCandleFile.getUTCFullYear() + "/" + (contextVariables.lastCandleFile.getUTCMonth() + 1) + "/" + contextVariables.lastCandleFile.getUTCDate() + "."); }
callBackFunction(global.DEFAULT_OK_RESPONSE); // Here is where we finish processing and wait for the platform to run this module again.
return;
@@ -535,7 +360,7 @@ Read the candles and volumes from Bruce and produce a single Index File for Mark
end: previousCandles[i][5]
};
if (candle.end < lastCandleFile.valueOf()) {
if (candle.end < contextVariables.lastCandleFile.valueOf()) {
outputCandles[n].push(candle);
@@ -557,7 +382,7 @@ Read the candles and volumes from Bruce and produce a single Index File for Mark
sell: previousVolumes[i][1]
};
if (volume.end < lastCandleFile.valueOf()) {
if (volume.end < contextVariables.lastCandleFile.valueOf()) {
outputVolumes[n].push(volume);
@@ -574,7 +399,7 @@ Read the candles and volumes from Bruce and produce a single Index File for Mark
if (FULL_LOG === true) { logger.write("[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> Entering function."); }
let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2);
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
let fileName = market.assetA + '_' + market.assetB + ".json"
let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
@@ -610,7 +435,7 @@ Read the candles and volumes from Bruce and produce a single Index File for Mark
const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs
let totalOutputCandles = inputFilePeriod / outputPeriod; // this should be 2 in this case.
let beginingOutputTime = lastCandleFile.valueOf();
let beginingOutputTime = contextVariables.lastCandleFile.valueOf();
for (let i = 0; i < totalOutputCandles; i++) {
@@ -690,7 +515,7 @@ Read the candles and volumes from Bruce and produce a single Index File for Mark
if (FULL_LOG === true) { logger.write("[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> Entering function."); }
let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2);
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
let fileName = market.assetA + '_' + market.assetB + ".json"
let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath;
@@ -724,7 +549,7 @@ Read the candles and volumes from Bruce and produce a single Index File for Mark
const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs
let totalOutputVolumes = inputFilePeriod / outputPeriod; // this should be 2 in this case.
let beginingOutputTime = lastCandleFile.valueOf();
let beginingOutputTime = contextVariables.lastCandleFile.valueOf();
for (let i = 0; i < totalOutputVolumes; i++) {
@@ -788,7 +613,7 @@ Read the candles and volumes from Bruce and produce a single Index File for Mark
} else {
writeStatusReport(lastCandleFile, advanceTime);
writeStatusReport(contextVariables.lastCandleFile, advanceTime);
}
}
@@ -954,8 +779,8 @@ Read the candles and volumes from Bruce and produce a single Index File for Mark
try {
statusReport.lastExecution = global.processDatetime;
statusReport.lastFile = lastFileDate;
statusReportModule.file.lastExecution = global.processDatetime;
statusReportModule.file.lastFile = lastFileDate;
statusReportModule.save(callBack);
}
+4 -1
View File
@@ -21,7 +21,10 @@
<Content Include="package-lock.json" />
<Content Include="package.json" />
<Content Include="README.md" />
<Compile Include="Multi-Period-Daily\Interval.js" />
<Compile Include="Commons.js">
<SubType>Code</SubType>
</Compile>
<Compile Include="Multi-Period-Daily\User.bot.js" />
<Compile Include="Multi-Period-Market\User.bot.js" />
<Content Include="logs\Dummy File.txt" />
<Content Include="obj\Debug\AABruce.njsproj.FileListAbsolute.txt" />
+4 -4
View File
@@ -28,8 +28,8 @@
"run": "true"
}
},
"intervalLapse": 3600000,
"intervalMaxLoops": -1
"executionWaitTime": 3600000,
"retryWaitTime": 10000
},
{
"name": "Multi-Period-Daily",
@@ -49,8 +49,8 @@
"run": "true"
}
},
"intervalLapse": 60000,
"intervalMaxLoops": -1
"executionWaitTime": 60000,
"retryWaitTime": 10000
}
],
"products": [