fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #47 from AAMasters/develop

Browse Source 
Develop last changes
This commit is contained in:
benme authored and GitHub committed 2019-03-20 12:50:18 +01:00
2 parents dc04e1f349 bc74aef9c7
commit 1ea41786ec
3 files changed
+189 -79
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

IndicatorBotProcessMainLoop.js
+16 -2
View File
@@ -296,8 +296,12 @@
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> onInizialized -> Execution finished well."); }
/* If the process is configured to run inside a framework, we continue there, otherwise we run the bot directly. */
if (processConfig.framework === undefined) {
initializeUserBot();
return;
}
switch (processConfig.framework) {
switch (processConfig.framework.name) {
case 'Multi-Period-Market': {
processFramework = MULTI_PERIOD_MARKET.newMultiPeriodMarket(bot, logger, COMMONS_MODULE, UTILITIES, BLOB_STORAGE, USER_BOT_MODULE, COMMONS_MODULE);
intitializeProcessFramework();
@@ -308,7 +312,17 @@
intitializeProcessFramework();
break;
}
default: { initializeUserBot();}
default: {
logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Process Framework not Supported.");
logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Process Framework Name = " + processConfig.framework.name);
logger.persist();
clearInterval(fixedTimeLoopIntervalHandle);
clearTimeout(nextLoopTimeoutHandle);
clearTimeout(checkLoopHealthHandle);
bot.enableCheckLoopHealth = false;
callBackFunction(err);
return;
}
}
return;
}
MultiPeriodDaily.js
+140 -76
View File
@@ -23,7 +23,7 @@
let usertBot;
let currentBotStorage = BLOB_STORAGE.newBlobStorage(bot, logger);
let pProcessConfig;
let processConfig;
return thisObject;
@@ -91,12 +91,14 @@
let contextVariables = {
lastFile: undefined, // Datetime of the last file files sucessfully produced by this process.
dateBeginingOfMarket: undefined, // Datetime of the first trade file in the whole market history.
dateBeginOfMarket: undefined, // Datetime of the first trade file in the whole market history.
dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process.
};
let previousDay; // Holds the date of the previous day relative to the processing date.
let currentDay; // Holds the processing date.
let currentDay; // Holds the processing date.
let interExecutionMemoryArray;
getContextVariables();
@@ -110,84 +112,103 @@
let reportKey;
let statusReport;
/*
We look first for the bot who knows the begining of the marke in order to get when the market starts.
*/
if (processConfig.framework.startDate.fixedDate !== undefined) {
let botWhoKnowsTheBeginingOfTheMarket = statusDependencies.config[processConfig.statusDependenciesIndexBegingOfMarket];
let botWhoKnowsTheEndOfTheMarket = statusDependencies.config[processConfig.statusDependenciesIndexEndOfMarket];
/* The starting date is fixed, we will start from there. */
reportKey = botWhoKnowsTheBeginingOfTheMarket.devTeam + "-" + botWhoKnowsTheBeginingOfTheMarket.bot + "-" + botWhoKnowsTheBeginingOfTheMarket.process + "-" + "dataSet.V1";
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
contextVariables.dateBeginOfMarket = new Date(processConfig.framework.startDate.fixedDate);
statusReport = statusDependencies.statusReports.get(reportKey);
} else {
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
/*
We look first for the bot who knows the begining of the marke in order to get when the market starts.
*/
if (statusReport.status === "Status Report is corrupt.") {
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
let botWhoKnowsTheBeginingOfTheMarket = statusDependencies.config[processConfig.framework.startDate.takeItFromStatusDependency];
thisReport = statusDependencies.statusReports.get(reportKey).file;
reportKey = botWhoKnowsTheBeginingOfTheMarket.devTeam + "-" + botWhoKnowsTheBeginingOfTheMarket.bot + "-" + botWhoKnowsTheBeginingOfTheMarket.process + "-" + "dataSet.V1";
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
if (thisReport.lastFile === undefined) {
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet.");
statusReport = statusDependencies.statusReports.get(reportKey);
let customOK = {
result: global.CUSTOM_OK_RESPONSE.result,
message: "Dependency does not exist."
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
callBackFunction(customOK);
return;
}
contextVariables.dateBeginingOfMarket = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS);
//contextVariables.dateBeginingOfMarket = new Date("2018-01-01"); // TODO: Remove this temporaty patch.
/*
Here we get the status report from the bot who knows which is the end of the market.
*/
reportKey = botWhoKnowsTheEndOfTheMarket.devTeam + "-" + botWhoKnowsTheEndOfTheMarket.bot + "-" + botWhoKnowsTheEndOfTheMarket.process + "-" + "dataSet.V1";
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
statusReport = statusDependencies.statusReports.get(reportKey);
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
if (statusReport.status === "Status Report is corrupt.") {
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
thisReport = statusDependencies.statusReports.get(reportKey).file;
if (thisReport.lastFile === undefined) {
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
let customOK = {
result: global.CUSTOM_OK_RESPONSE.result,
message: "Dependency not ready."
if (statusReport.status === "Status Report is corrupt.") {
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
callBackFunction(customOK);
return;
thisReport = statusDependencies.statusReports.get(reportKey).file;
if (thisReport.lastFile === undefined) {
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet.");
let customOK = {
result: global.CUSTOM_OK_RESPONSE.result,
message: "Dependency does not exist."
}
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
callBackFunction(customOK);
return;
}
contextVariables.dateBeginOfMarket = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS);
}
contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf());
if (processConfig.framework.endDate.fixedDate !== undefined) {
/* The ending date is fixed, we will end there. */
contextVariables.dateEndOfMarket = new Date(processConfig.framework.endDate.fixedDate);
} else {
/*
Here we get the status report from the bot who knows which is the end of the market.
*/
let botWhoKnowsTheEndOfTheMarket = statusDependencies.config[processConfig.framework.endDate.takeItFromStatusDependency];
reportKey = botWhoKnowsTheEndOfTheMarket.devTeam + "-" + botWhoKnowsTheEndOfTheMarket.bot + "-" + botWhoKnowsTheEndOfTheMarket.process + "-" + "dataSet.V1";
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
statusReport = statusDependencies.statusReports.get(reportKey);
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
if (statusReport.status === "Status Report is corrupt.") {
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
thisReport = statusDependencies.statusReports.get(reportKey).file;
if (thisReport.lastFile === undefined) {
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
let customOK = {
result: global.CUSTOM_OK_RESPONSE.result,
message: "Dependency not ready."
}
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
callBackFunction(customOK);
return;
}
contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf());
}
/* Finally we get our own Status Report. */
@@ -214,7 +235,7 @@
contextVariables.lastFile = new Date(thisReport.lastFile);
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> thisReport.lastFile !== undefined"); }
interExecutionMemoryArray = thisReport.interExecutionMemoryArray;
processTimePeriods();
return;
@@ -222,16 +243,28 @@
} else {
/*
In the case when there is no status report, we take the date of the file with the first trades as the begining of the market. Then we will
go one day further in time, so that the previous day does fine a file at the begining of the market.
In the case when there is no status report, we assume like the last processed file is the one on the date of Begining of Market.
*/
contextVariables.lastFile = new Date(contextVariables.dateBeginingOfMarket.getUTCFullYear() + "-" + (contextVariables.dateBeginingOfMarket.getUTCMonth() + 1) + "-" + contextVariables.dateBeginingOfMarket.getUTCDate() + " " + "00:00" + GMT_SECONDS);
contextVariables.lastFile = new Date(contextVariables.lastFile.valueOf() + ONE_DAY_IN_MILISECONDS);
contextVariables.lastFile = new Date(contextVariables.dateBeginOfMarket.getUTCFullYear() + "-" + (contextVariables.dateBeginOfMarket.getUTCMonth() + 1) + "-" + contextVariables.dateBeginOfMarket.getUTCDate() + " " + "00:00" + GMT_SECONDS);
//contextVariables.lastFile = new Date(contextVariables.lastFile.valueOf() + ONE_DAY_IN_MILISECONDS);
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> thisReport.lastFile === undefined"); }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> contextVariables.lastFile = " + contextVariables.lastFile); }
/*
The first time the process is running is the right time to create the data structure that is going to be shared across different executions.
This data structure has one object per each timePeriod.
*/
interExecutionMemoryArray = [];
for (let i = 0; i < global.dailyFilePeriods.length; i++)
{
let emptyObject = {};
interExecutionMemoryArray.push(emptyObject);
}
processTimePeriods();
return;
}
@@ -342,7 +375,12 @@
let previousFile;
let currentFile;
getPreviousFile();
if (currentDay.valueOf() > contextVariables.dateBeginOfMarket.valueOf()) {
getPreviousFile();
} else {
previousFile = [];
getCurrentFile()
}
function getPreviousFile() {
@@ -378,6 +416,14 @@
return;
}
if (err.result === "Fail Because" && err.message === "File does not exist.") {
logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.message);
logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> filePath = " + filePath);
logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> fileName = " + fileName);
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
@@ -424,6 +470,15 @@
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); }
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); }
if (err.result === "Fail Because" && err.message === "File does not exist.") {
logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.message);
logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> filePath = " + filePath);
logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> fileName = " + fileName);
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.message);
@@ -492,7 +547,15 @@
const timePeriod = global.dailyFilePeriods[n][0];
const outputPeriodLabel = global.dailyFilePeriods[n][1];
usertBot.start(dataFiles, timePeriod, outputPeriodLabel, currentDay, onBotFinished);
usertBot.start(
dataFiles,
timePeriod,
outputPeriodLabel,
currentDay,
contextVariables.dateBeginOfMarket,
contextVariables.dateEndOfMarket,
interExecutionMemoryArray[n],
onBotFinished);
function onBotFinished(err) {
@@ -592,7 +655,7 @@
let folderName = bot.products[productIndex].codeName;
writeDataRange(contextVariables.dateBeginingOfMarket, currentDay, folderName, controlLoop);
writeDataRange(contextVariables.dateBeginOfMarket, currentDay, folderName, controlLoop);
}
function controlLoop() {
@@ -669,6 +732,7 @@
thisReport.file.lastExecution = bot.currentDaytime;
thisReport.file.lastFile = lastFileDate;
thisReport.file.interExecutionMemoryArray = interExecutionMemoryArray;
thisReport.save(callBack);
}
MultiPeriodMarket.js
+33 -1
View File
@@ -23,6 +23,8 @@
let usertBot;
let processConfig;
return thisObject;
function initialize(pProcessConfig, pStatusDependencies, pDataDependencies, callBackFunction) {
@@ -36,6 +38,7 @@
statusDependencies = pStatusDependencies;
dataDependencies = pDataDependencies;
processConfig = pProcessConfig;
for (let i = 0; i < dataDependencies.config.length; i++) {
@@ -72,6 +75,29 @@
let market = global.MARKET;
/* Context Variables */
let contextVariables = {
dateBeginOfMarket: undefined, // Datetime of the first trade file in the whole market history.
dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process.
};
if (processConfig.framework.startDate.fixedDate !== undefined) {
/* The starting date is fixed, we will start from there. */
contextVariables.dateBeginOfMarket = new Date(processConfig.framework.startDate.fixedDate);
}
if (processConfig.framework.endDate.fixedDate !== undefined) {
/* The ending date is fixed, we will end there. */
contextVariables.dateEndOfMarket = new Date(processConfig.framework.endDate.fixedDate);
}
processTimePeriods();
function processTimePeriods() {
@@ -218,7 +244,13 @@
const timePeriod = global.marketFilesPeriods[n][0];
const outputPeriodLabel = global.marketFilesPeriods[n][1];
usertBot.start(dataFiles, timePeriod, outputPeriodLabel, onBotFinished);
usertBot.start(
dataFiles,
timePeriod,
outputPeriodLabel,
contextVariables.dateBeginOfMarket,
contextVariables.dateEndOfMarket,
onBotFinished);
function onBotFinished(err) {