fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Beta6 update: Time Frame Files

Browse Source
This commit is contained in:
Luis Fernando Molina committed 2020-07-28 13:28:14 +02:00
1 parent 6ec5580cd4
commit e90ccebe8f
31 files changed
+2632 -1382
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

TaskServer/LowFrequencyTrading/TradingProcess.js
+59 -11
View File
@@ -255,11 +255,11 @@
}
}
/*
We do market files first since if the simulation is run on daily files, there will be a loop to getch each daus files and we do not need
that loop to reload market files.
*/
function processMarketFiles() {
/*
We do market files first since if the simulation is run on daily files, there will be a loop to getch each daus files and we do not need
that loop to reload market files.
*/
let n;
timeFramesLoop();
@@ -603,22 +603,25 @@
bot.FIRST_EXECUTION = false // From here on, all other loops executions wont be the first execution.
if (currentTimeFrame > global.dailyFilePeriods[0][0]) {
writeMarketStatusReport(onMarketStatusReport)
} else {
writeDataRanges(currentTimeFrameLabel, onWritten);
writeTimeFramesFiles(currentTimeFrame, currentTimeFrameLabel, onTimeFrameFilesWritten)
function onTimeFrameFilesWritten() {
writeDataRanges(currentTimeFrameLabel, onDataRangesWritten);
}
function onWritten(err) {
function onDataRangesWritten(err) {
try {
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> buildCharts -> onOutputGenerated -> onWritten -> err = " + err.stack);
callBackFunction(err);
return;
}
writeDailyStatusReport(bot.tradingProcessDate, onDailyStatusReport);
if (currentTimeFrame > global.dailyFilePeriods[0][0]) {
writeMarketStatusReport(onMarketStatusReport)
} else {
writeDailyStatusReport(bot.tradingProcessDate, onDailyStatusReport);
}
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processDailyFiles -> buildCharts -> onOutputGenerated -> onWritten -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
@@ -727,6 +730,51 @@
thisReport.save(callBack);
}
function writeTimeFramesFiles(currentTimeFrame, currentTimeFrameLabel, callBack) {
let outputDatasetIndex = -1;
controlLoop()
function productLoopBody() {
let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName;
writeTimeFramesFile(currentTimeFrame, currentTimeFrameLabel, productCodeName, 'Multi-Period-Daily', dailyFileWritten)
function dailyFileWritten() {
writeTimeFramesFile(currentTimeFrame, currentTimeFrameLabel, productCodeName, 'Multi-Period-Market', controlLoop)
}
}
function controlLoop() {
outputDatasetIndex++
if (outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length) {
productLoopBody()
} else {
callBack()
}
}
}
function writeTimeFramesFile(currentTimeFrame, currentTimeFrameLabel, productCodeName, processType, callBack) {
let timeFramesArray = []
timeFramesArray.push(currentTimeFrameLabel)
let fileContent = JSON.stringify(timeFramesArray)
let fileName = '/Time.Frames.json';
let filePath = bot.filePathRoot + "/Output/" + bot.SESSION.folderName + "/" + productCodeName + "/" + processType + fileName;
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated)
function onFileCreated(err) {
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] start -> writeTimeFramesFile -> onFileCreated -> err = " + err.stack)
callBack(err)
return
}
callBack()
}
}
function trimDataFile(dataFile, recordDefinition) {
/*
Here we will discard all the records in a file that are outside of the current time range.
TaskServer/LowFrequencyTrading/TradingSystem.js
+2 -4
View File
@@ -272,8 +272,7 @@ exports.newTradingSystem = function newTradingSystem(bot, logger, tradingEngineM
*/
value = false
if (code.indexOf('previous') > -1 && err.message.indexOf('of undefined') > -1 ||
code.indexOf('chart') > -1 && err.message.indexOf('of undefined') > -1
if (code.indexOf('previous') > -1 && err.message.indexOf('of undefined') > -1
) {
/*
We are not going to set an error for the casess we are using previous and the error is that the indicator is undefined.
@@ -313,8 +312,7 @@ exports.newTradingSystem = function newTradingSystem(bot, logger, tradingEngineM
*/
value = 0
if (node.code.indexOf('previous') > -1 && err.message.indexOf('of undefined') > -1 ||
node.code.indexOf('chart') > -1 && err.message.indexOf('of undefined') > -1
if (node.code.indexOf('previous') > -1 && err.message.indexOf('of undefined') > -1
) {
/*
We are not going to set an error for the casess we are using previous and the error is that the indicator is undefined.
TaskServer/MultiPeriodDaily.js
+365 -600
View File
File diff suppressed because it is too large. Load diff
TaskServer/MultiPeriodMarket.js
+171 -278
View File
@@ -1,52 +1,38 @@
exports.newMultiPeriodMarket = function newMultiPeriodMarket(bot, logger, UTILITIES, FILE_STORAGE) {
const FULL_LOG = true;
const LOG_FILE_CONTENT = false;
const MODULE_NAME = "Multi Period Market";
thisObject = {
initialize: initialize,
finalize: finalize,
start: start
};
let utilities = UTILITIES.newCloudUtilities(logger);
let utilities = UTILITIES.newCloudUtilities(logger)
let fileStorage = FILE_STORAGE.newFileStorage(logger)
let statusDependencies;
let dataDependenciesModule;
let dataFiles = new Map();
let dataFiles = new Map()
let indicatorOutputModule;
let processConfig;
return thisObject;
function initialize(pProcessConfig, pStatusDependencies, pDataDependencies, callBackFunction) {
logger.fileName = MODULE_NAME;
logger.initialize()
try {
statusDependencies = pStatusDependencies;
dataDependenciesModule = pDataDependencies;
processConfig = pProcessConfig;
logger.fileName = MODULE_NAME;
logger.initialize();
let INDICATOR_OUTPUT_MODULE = require("./IndicatorOutput")
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
statusDependencies = pStatusDependencies;
dataDependenciesModule = pDataDependencies;
processConfig = pProcessConfig;
let INDICATOR_OUTPUT_MODULE = require("./IndicatorOutput")
indicatorOutputModule = INDICATOR_OUTPUT_MODULE.newIndicatorOutput(bot, logger, UTILITIES, FILE_STORAGE);
indicatorOutputModule.initialize(callBackFunction);
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
indicatorOutputModule = INDICATOR_OUTPUT_MODULE.newIndicatorOutput(bot, logger, UTILITIES, FILE_STORAGE)
indicatorOutputModule.initialize(callBackFunction)
}
function finalize() {
fileStorage = undefined
dataFiles = undefined
statusDependencies = undefined
dataDependenciesModule = undefined
@@ -56,272 +42,179 @@
}
function start(callBackFunction) {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); }
let market = bot.market;
/* Context Variables */
let contextVariables = {
dateBeginOfMarket: undefined, // Datetime of the first trade file in the whole market history.
dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process.
};
processTimeFrames();
processTimeFrames()
function processTimeFrames() {
let n;
timeFramesLoop()
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> Entering function."); }
try {
let n;
periodsLoop();
function periodsLoop() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoop -> Entering function."); }
/*
We will iterate through all posible periods.
*/
n = 0 // loop Variable representing each possible period as defined at the periods array.
periodsLoopBody();
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> periodsLoop -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function periodsLoopBody() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> Entering function."); }
const timeFrame = global.marketFilesPeriods[n][0];
const timeFrameLabel = global.marketFilesPeriods[n][1];
let dependencyIndex = 0;
dataFiles = new Map;
dependencyLoopBody();
function dependencyLoopBody() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> Entering function."); }
let dependency = dataDependenciesModule.nodeArray[dependencyIndex];
let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex];
getFile();
function getFile() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> getFile -> Entering function."); }
let dateForPath = bot.processDatetime.getUTCFullYear() + '/' + utilities.pad(bot.processDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.processDatetime.getUTCDate(), 2);
let fileName = "Data.json";
let filePath
if (dependency.referenceParent.config.codeName === "Multi-Period-Market") {
filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel;
} else {
filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + dateForPath;
}
datasetModule.getTextFile(filePath, fileName, onFileReceived);
function onFileReceived(err, text) {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); }
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); }
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack);
callBackFunction(err);
return;
}
let dataFile = JSON.parse(text);
dataFiles.set(dependency.id, dataFile);
dependencyControlLoop();
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> periodsLoopBody -> dependencyLoopBody -> getFile -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> periodsLoop -> dependencyLoopBody -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function dependencyControlLoop() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> dependencyControlLoop -> Entering function."); }
dependencyIndex++;
if (dependencyIndex < dataDependenciesModule.nodeArray.length) {
dependencyLoopBody();
} else {
generateOutput();
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> dependencyControlLoop -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function generateOutput() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> generateOutput -> Entering function."); }
const timeFrame = global.marketFilesPeriods[n][0];
const timeFrameLabel = global.marketFilesPeriods[n][1];
indicatorOutputModule.start(
dataFiles,
timeFrame,
timeFrameLabel,
undefined,
undefined,
onOutputGenerated);
function onOutputGenerated(err) {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsLoopBody -> generateOutput -> onOutputGenerated -> Entering function."); }
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
callBackFunction(err);
return;
}
periodsControlLoop();
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> periodsLoopBody -> generateOutput -> onOutputGenerated -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> periodsLoopBody -> generateOutput -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> periodsLoopBody -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function periodsControlLoop() {
try {
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> periodsControlLoop -> Entering function."); }
n++;
if (n < global.marketFilesPeriods.length) {
periodsLoopBody();
} else {
writeStatusReport(callBackFunction);
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> periodsControlLoop -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function timeFramesLoop() {
/* We will iterate through all posible timeFrames.*/
n = 0 // loop Variable representing each possible period as defined at the timeFrames array.
timeFramesLoopBody()
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
function timeFramesLoopBody() {
const timeFrame = global.marketFilesPeriods[n][0]
const timeFrameLabel = global.marketFilesPeriods[n][1]
/* Check Time Frames Filter */
if (bot.marketTimeFrames !== undefined) {
if (bot.marketTimeFrames.includes(timeFrameLabel) === false) {
/* We are not going to process this Time Frame */
timeFramesControlLoop()
return
}
}
let dependencyIndex = 0;
dataFiles = new Map;
dependencyLoopBody()
function dependencyLoopBody() {
let dependency = dataDependenciesModule.nodeArray[dependencyIndex]
let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]
getFile()
function getFile() {
let fileName = "Data.json";
let filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel;
datasetModule.getTextFile(filePath, fileName, onFileReceived)
function onFileReceived(err, text) {
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] start -> processTimeFrames -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack)
callBackFunction(err)
return;
}
let dataFile = JSON.parse(text)
dataFiles.set(dependency.id, dataFile)
dependencyControlLoop()
}
}
}
function dependencyControlLoop() {
dependencyIndex++;
if (dependencyIndex < dataDependenciesModule.nodeArray.length) {
dependencyLoopBody()
} else {
generateOutput()
}
function generateOutput() {
indicatorOutputModule.start(
dataFiles,
timeFrame,
timeFrameLabel,
undefined,
undefined,
onOutputGenerated)
function onOutputGenerated(err) {
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
callBackFunction(err)
return;
}
timeFramesControlLoop()
}
}
}
}
function timeFramesControlLoop() {
n++;
if (n < global.marketFilesPeriods.length) {
timeFramesLoopBody()
} else {
writeTimeFramesFiles(onTimeFrameFilesWritten)
function onTimeFrameFilesWritten() {
writeStatusReport(callBackFunction)
}
}
}
}
function writeTimeFramesFiles(callBack) {
let outputDatasetIndex = -1;
controlLoop()
function productLoopBody() {
let productCodeName = bot.processNode.referenceParent.processOutput.outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName;
writeTimeFramesFile(productCodeName, controlLoop)
}
function controlLoop() {
outputDatasetIndex++
if (outputDatasetIndex < bot.processNode.referenceParent.processOutput.outputDatasets.length) {
productLoopBody()
} else {
callBack()
}
}
}
function writeTimeFramesFile(productCodeName, callBack) {
let timeFramesArray = []
for (let n = 0; n < global.marketFilesPeriods.length; n++) {
let timeFrameLabel = global.marketFilesPeriods[n][1]
/* Check Time Frames Filter */
if (bot.marketTimeFrames !== undefined) {
if (bot.marketTimeFrames.includes(timeFrameLabel) === true) {
timeFramesArray.push(timeFrameLabel)
}
} else {
timeFramesArray.push(timeFrameLabel)
}
}
let fileContent = JSON.stringify(timeFramesArray)
let fileName = '/Time.Frames.json';
let filePath = bot.filePathRoot + "/Output/" + productCodeName + "/" + bot.process + fileName;
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated)
function onFileCreated(err) {
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] start -> writeTimeFramesFile -> onFileCreated -> err = " + err.stack)
callBack(err)
return
}
callBack()
}
}
function writeStatusReport(callBack) {
let reportKey = bot.dataMine + "-" + bot.codeName + "-" + "Multi-Period-Market"
let thisReport = statusDependencies.statusReports.get(reportKey)
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> Entering function."); }
try {
let reportKey = bot.dataMine + "-" + bot.codeName + "-" + "Multi-Period-Market"
let thisReport = statusDependencies.statusReports.get(reportKey);
thisReport.file.lastExecution = bot.processDatetime;
thisReport.save(callBack);
if (global.areEqualDates(bot.processDatetime, new Date()) === false) {
logger.newInternalLoop(bot.codeName, bot.process, bot.processDatetime);
}
/* Telling the world we are alive and doing well */
let currentDateString = bot.processDatetime.getUTCFullYear() + '-' + utilities.pad(bot.processDatetime.getUTCMonth() + 1, 2) + '-' + utilities.pad(bot.processDatetime.getUTCDate(), 2);
let currentDate = new Date(bot.processDatetime)
let lastDate = new Date()
bot.processHeartBeat(currentDateString, global.getPercentage(currentDate, currentDate, lastDate))
thisReport.file.lastExecution = bot.processDatetime;
if (bot.marketTimeFrames !== undefined) {
thisReport.file.timeFrames = bot.marketTimeFrames
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
thisReport.save(callBack)
if (global.areEqualDates(bot.processDatetime, new Date()) === false) {
logger.newInternalLoop(bot.codeName, bot.process, bot.processDatetime)
}
/* Telling the world we are alive and doing well */
let currentDateString = bot.processDatetime.getUTCFullYear() + '-' + utilities.pad(bot.processDatetime.getUTCMonth() + 1, 2) + '-' + utilities.pad(bot.processDatetime.getUTCDate(), 2)
let currentDate = new Date(bot.processDatetime)
let lastDate = new Date()
bot.processHeartBeat(currentDateString, global.getPercentage(currentDate, currentDate, lastDate))
}
}
catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.stack)
callBackFunction(global.DEFAULT_FAIL_RESPONSE)
}
}
};
}
TaskServer/ProcessOutput.js
+3 -2
View File
@@ -14,7 +14,7 @@
return thisObject;
function raiseEvents(lastFile, callBackFunction) {
function raiseEvents(lastFile, timeFrames, callBackFunction) {
try {
if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] raiseEvents -> Entering function."); }
@@ -83,7 +83,8 @@
key = dataMine + "-" + botName + "-" + product + "-" + dataset + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset
let event = {
lastFile: lastFile
lastFile: lastFile,
timeFrames: timeFrames
}
global.EVENT_SERVER_CLIENT.createEventHandler(key, 'Dataset Updated')
global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Dataset Updated', event)
TaskServer/Root.js
+6
View File
@@ -225,6 +225,12 @@
/* Bot Type */
botConfig.type = global.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.type
/* Time Frame Filter */
if (global.TASK_NODE.bot.timeFramesFilter !== undefined) {
botConfig.dailyTimeFrames = global.TASK_NODE.bot.timeFramesFilter.config.dailyTimeFrames
botConfig.marketTimeFrames = global.TASK_NODE.bot.timeFramesFilter.config.marketTimeFrames
}
if (botConfig.processNode) {
if (botConfig.processNode.marketReference) {
if (botConfig.processNode.marketReference.keyReference !== undefined) {
TaskServer/StatusReport.js
+1 -1
View File
@@ -334,7 +334,7 @@
let processOutput = PROCESS_OUTPUT.newProcessOutput(bot, logger)
processOutput.raiseEvents(thisObject.file.lastFile, callBackFunction);
processOutput.raiseEvents(thisObject.file.lastFile, thisObject.file.timeFrames, callBackFunction);
return;
}