fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Output Management Functions Implemented plus bug fixes

Browse Source
This commit is contained in:
Luis Fernando Molina committed 2021-02-05 12:02:22 +01:00
1 parent 12702289b0
commit 10698a49a7
14 files changed
+258 -197
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Projects/ProjectsSchema.json
+6
View File
@@ -454,6 +454,12 @@
"propertyName": "processFilesFunctions",
"functionName": "newSuperalgosFunctionLibrariesProcessFilesFunctions",
"fileName": "ProcessFilesFunctions.js"
},
{
"name": "Output Management Functions",
"propertyName": "outputManagementFunctions",
"functionName": "newSuperalgosFunctionLibrariesOutputManagementFunctions",
"fileName": "OutputManagementFunctions.js"
}
],
"utilities": [
Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Bollinger-Bands/BollingerBandsMultiPeriodDaily.js
+1 -1
View File
@@ -258,7 +258,7 @@
let n;
processDate = new Date(contextVariables.lastBandFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop.
let fromDate = new Date(processDate.valueOf())
let lastDate = new Date()
let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date())
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> buildBands -> processDate = " + processDate)
Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Bollinger-Bands/BollingerBandsMultiPeriodMarket.js
+1 -1
View File
@@ -409,7 +409,7 @@
/* Telling the world we are alive and doing well */
let currentDateString = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCDate(), 2);
let currentDate = new Date(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME)
let lastDate = new Date()
let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date())
TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, currentDateString, TS.projects.superalgos.utilities.dateTimeFunctions.getPercentage(currentDate, currentDate, lastDate))
}
catch (err) {
Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Candles-Volumes/CandlesVolumesMultiPeriodDaily.js
+1 -1
View File
@@ -228,7 +228,7 @@
function buildCandles() {
try {
let fromDate = new Date(contextVariables.lastCandleFile.valueOf())
let lastDate = new Date()
let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date())
let outputCandles
let outputVolumes
Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/Candles-Volumes/CandlesVolumesMultiPeriodMarket.js
+1 -1
View File
@@ -389,7 +389,7 @@
try {
let fromDate = new Date(contextVariables.lastCandleFile.valueOf())
let lastDate = new Date()
let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date())
/*
Firstly we prepere the arrays that will accumulate all the information for each output file.
*/
Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/MultiPeriodDaily.js
+1 -1
View File
@@ -234,7 +234,7 @@
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).DAILY_FILES_PROCESS_DATETIME =
new Date(contextVariables.lastFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) // Go back one day to start well when we advance time at the begining of the loop.
let fromDate = new Date(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).DAILY_FILES_PROCESS_DATETIME.valueOf())
let lastDate = new Date()
let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date())
advanceTime()
Projects/Superalgos/TS/Bot-Modules/Indicator-Bot/MultiPeriodMarket.js
+15 -12
View File
@@ -16,7 +16,7 @@
return thisObject;
function initialize(pStatusDependencies, pDataDependencies, callBackFunction) {
statusDependencies = pStatusDependencies;
dataDependenciesModule = pDataDependencies;
@@ -80,7 +80,7 @@
function onFileReceived(err, text) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> processTimeFrames -> timeFramesLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack)
callBackFunction(err)
return;
@@ -136,8 +136,8 @@
}
function writeTimeFramesFiles(callBack) {
let outputDatasets =
TS.projects.superalgos.utilities.nodeFunctions.nodeBranchToArray(TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.processOutput, 'Output Dataset')
let outputDatasets =
TS.projects.superalgos.utilities.nodeFunctions.nodeBranchToArray(TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.processOutput, 'Output Dataset')
let outputDatasetIndex = -1;
controlLoop()
@@ -178,14 +178,14 @@
let fileContent = JSON.stringify(timeFramesArray)
let fileName = '/Time.Frames.json';
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + productCodeName + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.codeName + fileName;
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + productCodeName + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.codeName + fileName;
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated)
function onFileCreated(err) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> writeTimeFramesFile -> onFileCreated -> err = " + err.stack)
callBack(err)
return
@@ -195,8 +195,8 @@
}
function writeStatusReport(callBack) {
let reportKey = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName + "-" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.config.codeName + "-" + "Multi-Period-Market"
let reportKey = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName + "-" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.config.codeName + "-" + "Multi-Period-Market"
let thisReport = statusDependencies.statusReports.get(reportKey)
thisReport.file.lastExecution = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME;
@@ -214,13 +214,16 @@
/* Telling the world we are alive and doing well */
let currentDateString = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME.getUTCDate(), 2)
let currentDate = new Date(TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME)
let lastDate = new Date()
TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, currentDateString, TS.projects.superalgos.utilities.dateTimeFunctions.getPercentage(currentDate, currentDate, lastDate))
TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(
processIndex,
currentDateString,
TS.projects.superalgos.utilities.dateTimeFunctions.getPercentage(currentDate, currentDate, currentDate)
)
}
}
catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> err = " + err.stack)
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE)
}
Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingOutput.js
+18 -170
View File
@@ -34,187 +34,35 @@ exports.newSuperalgosBotModulesTradingOutput = function (processIndex) {
let outputDatasets = TS.projects.superalgos.utilities.nodeFunctions.nodeBranchToArray(TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.processOutput, 'Output Dataset')
let outputDatasetsMap = new Map()
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES === true) {
/*
For Daily Files, we are going to initialize the outputs at the first execution
if we are not resuming, but also at every following execution except when
we are at the head of the market, because at the head of the market we need to
append more data to the same output files that already exists.
*/
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_FIRST_LOOP === true && TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_RESUMING === false) {
await initializeOutputs()
} else {
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.tradingEngine.current.episode.headOfTheMarket.value === true) {
await readFiles()
} else {
await initializeOutputs()
}
}
} else {
/*
Form Market Files, we are only going to initialize the outputs at the first run
and if we are not resuming execution. Subsequent runs means that we are at the
head of the market. Remember that backtests finishes the session once the reach
the final datetime.
*/
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_FIRST_LOOP === true && TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_RESUMING === false) {
await initializeOutputs()
} else {
await readFiles()
}
}
await TS.projects.superalgos.functionLibraries.outputManagementFunctions.readFiles(
processIndex,
outputDatasets,
outputDatasetsMap,
timeFrameLabel,
tradingProcessDate,
fileStorage
)
await tradingSimulationModuleObject.runSimulation(
chart,
market,
exchange,
outputDatasetsMap,
writeFiles
writeOutputFiles
)
tradingSimulationModuleObject.finalize()
return
async function initializeOutputs() {
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) {
await initializeDailyFiles()
} else {
await initializeMarketFiles()
}
}
async function initializeDailyFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Daily Files') {
outputDatasetsMap.set(dataset.parentNode.config.codeName, [])
}
}
}
async function initializeMarketFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Market Files') {
outputDatasetsMap.set(dataset.parentNode.config.codeName, [])
}
}
}
async function readFiles() {
/*
This bot have an output of files that it generates. At every call to the bot, it needs to read the previously generated
files in order to later append more information after the execution is over. Here in this function we are going to
read those output files and get them ready for appending content during the simulation.
*/
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) {
await readDailyFiles()
} else {
await readMarketFiles()
}
}
async function readMarketFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Market Files') {
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel
await readOutputFile(fileName, filePath, dataset.parentNode.config.codeName)
}
}
}
async function readDailyFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Daily Files') {
let dateForPath = tradingProcessDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCDate(), 2);
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath
await readOutputFile(fileName, filePath, dataset.parentNode.config.codeName)
}
}
}
async function readOutputFile(fileName, filePath, productName) {
filePath += '/' + fileName
let response = await fileStorage.asyncGetTextFile(filePath, true)
if (response.err.message === 'File does not exist.') {
outputDatasetsMap.set(productName, [])
return
}
if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
throw (response.err)
}
outputDatasetsMap.set(productName, JSON.parse(response.text))
}
async function writeFiles() {
/*
The output of files which were appended with information during the simulation execution, now needs to be saved.
*/
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) {
await writeDailyFiles()
} else {
await writeMarketFiles()
}
}
async function writeMarketFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Market Files') {
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel
await writeOutputFile(fileName, filePath, dataset.parentNode.config.codeName)
}
}
}
async function writeDailyFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Daily Files') {
let dateForPath = tradingProcessDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCDate(), 2);
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath
await writeOutputFile(fileName, filePath, dataset.parentNode.config.codeName)
}
}
}
async function writeOutputFile(fileName, filePath, productName) {
filePath += '/' + fileName
let fileContent = JSON.stringify(outputDatasetsMap.get(productName))
let response = await fileStorage.asyncCreateTextFile(filePath, fileContent)
if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
throw (response.err)
}
function writeOutputFiles() {
TS.projects.superalgos.functionLibraries.outputManagementFunctions.writeFiles(
processIndex,
outputDatasets,
outputDatasetsMap,
timeFrameLabel,
tradingProcessDate,
fileStorage
)
}
} catch (err) {
Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingProcess.js
+3 -3
View File
@@ -131,7 +131,7 @@
if (checkThereAreCandles() === true) {
TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Running')
await generateOutput()
await outputManagement()
TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Saving')
await TS.projects.superalgos.functionLibraries.processFilesFunctions.writeProcessFiles(processIndex, contextVariables, currentTimeFrame, tradingProcessDate, statusDependencies)
TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Sleeping')
@@ -190,7 +190,7 @@
*/
if (checkThereAreCandles() === true) {
TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Running')
await generateOutput()
await outputManagement()
TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Saving')
await TS.projects.superalgos.functionLibraries.processFilesFunctions.writeProcessFiles(processIndex, contextVariables, currentTimeFrame, tradingProcessDate, statusDependencies)
TS.projects.superalgos.functionLibraries.sessionFunctions.sessionHeartBeat(processIndex, undefined, undefined, 'Sleeping')
@@ -259,7 +259,7 @@
}
}
async function generateOutput() {
async function outputManagement() {
await tradingOutputModuleObject.start(
chart,
market,
Projects/Superalgos/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/TradingSimulation.js
-1
View File
@@ -310,7 +310,6 @@ exports.newSuperalgosBotModulesTradingSimulation = function (processIndex) {
let currentDateString = heartBeatDate.getUTCFullYear() + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(heartBeatDate.getUTCMonth() + 1, 2) + '-' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(heartBeatDate.getUTCDate(), 2)
let currentDate = new Date(heartBeatDate)
let percentage = TS.projects.superalgos.utilities.dateTimeFunctions.getPercentage(fromDate, currentDate, lastDate)
/*
Theere are a few tasks that we need to do only when the date changes,
otherwise it would be suboptimal.
Projects/Superalgos/TS/Function-Libraries/OutputManagementFunctions.js
+203
View File
@@ -0,0 +1,203 @@
exports.newSuperalgosFunctionLibrariesOutputManagementFunctions = function () {
/*
This module helps a bot process to manage it's output files, since it load
the current state of it's output and allows the bot process to append info
to it. It also allows the bot process to save all updated output.
*/
const MODULE_NAME = "Output Management Functions"
let thisObject = {
readFiles: readFiles,
writeFiles: writeFiles
}
return thisObject
async function readFiles(
processIndex,
outputDatasets,
outputDatasetsMap,
timeFrameLabel,
tradingProcessDate,
fileStorage
) {
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES === true) {
/*
For Daily Files, we are going to initialize the outputs at the first execution
if we are not resuming, but also at every following execution except when
we are at the head of the market, because at the head of the market we need to
append more data to the same output files that already exists.
*/
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_FIRST_LOOP === true && TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_RESUMING === false) {
await initializeOutputs()
} else {
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.tradingEngine.current.episode.headOfTheMarket.value === true) {
await loadFiles()
} else {
await initializeOutputs()
}
}
} else {
/*
Form Market Files, we are only going to initialize the outputs at the first run
and if we are not resuming execution. Subsequent runs means that we are at the
head of the market. Remember that backtests finishes the session once the reach
the final datetime.
*/
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_FIRST_LOOP === true && TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_RESUMING === false) {
await initializeOutputs()
} else {
await loadFiles()
}
}
async function initializeOutputs() {
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) {
await initializeDailyFiles()
} else {
await initializeMarketFiles()
}
async function initializeDailyFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Daily Files') {
outputDatasetsMap.set(dataset.parentNode.config.codeName, [])
}
}
}
async function initializeMarketFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Market Files') {
outputDatasetsMap.set(dataset.parentNode.config.codeName, [])
}
}
}
}
async function loadFiles() {
/*
This bot have an output of files that it generates. At every call to the bot, it needs to read the previously generated
files in order to later append more information after the execution is over. Here in this function we are going to
read those output files and get them ready for appending content during the simulation.
*/
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) {
await readDailyFiles()
} else {
await readMarketFiles()
}
}
async function readMarketFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Market Files') {
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel
await readOutputFile(fileName, filePath, dataset.parentNode.config.codeName)
}
}
}
async function readDailyFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Daily Files') {
let dateForPath = tradingProcessDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCDate(), 2);
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath
await readOutputFile(fileName, filePath, dataset.parentNode.config.codeName)
}
}
}
async function readOutputFile(fileName, filePath, productName) {
filePath += '/' + fileName
let response = await fileStorage.asyncGetTextFile(filePath, true)
if (response.err.message === 'File does not exist.') {
outputDatasetsMap.set(productName, [])
return
}
if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
throw (response.err)
}
outputDatasetsMap.set(productName, JSON.parse(response.text))
}
}
async function writeFiles(
processIndex,
outputDatasets,
outputDatasetsMap,
timeFrameLabel,
tradingProcessDate,
fileStorage
) {
/*
The output of files which were appended with information during the simulation execution, now needs to be saved.
*/
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) {
await writeDailyFiles()
} else {
await writeMarketFiles()
}
async function writeMarketFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Market Files') {
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel
await writeOutputFile(fileName, filePath, dataset.parentNode.config.codeName)
}
}
}
async function writeDailyFiles() {
for (let i = 0; i < outputDatasets.length; i++) {
let outputDatasetNode = outputDatasets[i]
let dataset = outputDatasetNode.referenceParent
if (dataset.config.type === 'Daily Files') {
let dateForPath = tradingProcessDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCDate(), 2);
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath
await writeOutputFile(fileName, filePath, dataset.parentNode.config.codeName)
}
}
}
async function writeOutputFile(fileName, filePath, productName) {
filePath += '/' + fileName
let fileContent = JSON.stringify(outputDatasetsMap.get(productName))
let response = await fileStorage.asyncCreateTextFile(filePath, fileContent)
if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
throw (response.err)
}
}
}
}
Projects/Superalgos/TS/Function-Libraries/ProcessFilesFunctions.js
+2 -2
View File
@@ -4,12 +4,13 @@ exports.newSuperalgosFunctionLibrariesProcessFilesFunctions = function () {
process context variables before you do your work, and to save the
process files once you are done.
*/
const MODULE_NAME = "Process Files Function"
const MODULE_NAME = "Process Files Functions"
let thisObject = {
getContextVariables: getContextVariables,
writeProcessFiles: writeProcessFiles
}
return thisObject
function getContextVariables(processIndex, contextVariables, statusDependencies, initialDatetime, callBackFunction) {
try {
@@ -222,5 +223,4 @@ exports.newSuperalgosFunctionLibrariesProcessFilesFunctions = function () {
await thisReport.asyncSave()
}
}
return thisObject
}
Projects/Superalgos/TS/Process-Modules/DataDependencies.js
+3 -1
View File
@@ -113,8 +113,10 @@
we need to add the default exchange and market to these maps in order
for the following sections to work.
*/
thisObject.filters.market.list.set(thisObject.defaultMarket, true)
thisObject.filters.exchange.list.set(thisObject.defaultExchange, true)
thisObject.filters.exchange.markets.set(thisObject.defaultExchange + '-' + thisObject.defautMarket, true)
thisObject.filters.exchange.markets.set(thisObject.defaultExchange + '-' + thisObject.defaultMarket, true)
}
/*
For each dependency declared at the curatedDependencyNodeArray, we will initialize a
Projects/Superalgos/TS/Utilities/DateTimeFunctions.js
+3 -3
View File
@@ -14,9 +14,9 @@ exports.newSuperalgosUtilitiesDateTimeFunctions = function () {
}
function getPercentage(fromDate, currentDate, lastDate) {
let fromDays = Math.trunc(fromDate.valueOf() / TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS)
let currentDays = Math.trunc(currentDate.valueOf() / TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS)
let lastDays = Math.trunc((lastDate.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) / TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS)
let fromDays = fromDate.valueOf()
let currentDays = currentDate.valueOf()
let lastDays = lastDate.valueOf()
let percentage = (currentDays - fromDays) * 100 / (lastDays - fromDays)
if ((lastDays - fromDays) === 0) {
percentage = 100