Merge pull request #959 from Luis-Fernando-Molina/develop

Contribution: API Data Fetcher Daily Files being processed
This commit is contained in:
Luis Fernando Molina authored and GitHub committed 2021-05-21 15:01:28 -04:00
commit e3a2e60209
26 files changed
+289 -808

No files matched your search

+1 -1
View File
@@ -655,7 +655,7 @@
{
"name": "From One Min To Multi Period Daily",
"propertyName": "fromOneMinToMultiPeriodDaily",
"functionName": "newSuperalgosBotModulesFromOneMinToMultiPeriodMarket",
"functionName": "newSuperalgosBotModulesFromOneMinToMultiPeriodDaily",
"fileName": "FromOneMinToMultiPeriodDaily.js",
"folderName": "Indicator-Bot"
},
@@ -192,6 +192,12 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
}
}
}
if (endpointNode === undefined) {
// TODO ERROR HANDLING.
console.log("API Endpoint counld not be found. Check that the at least one Record Property has an API Response Field Reference child, and that this one is referencing an API Response Field.")
throw("Cant continue")
}
}
function calculateRecordPropertiesNodePathMap() {
@@ -328,7 +328,7 @@
function loopBody() {
try {
const timeFrame = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1];
const timeFrame = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[n][1];
let candles = []; // Here we will put all the candles of the 2 files read.
@@ -725,7 +725,7 @@
function controlLoop() {
try {
n++
if (n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length) {
if (n < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length) {
loopBody()
} else {
n = 0
@@ -57,8 +57,8 @@
function loopBody() {
try {
const outputPeriod = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][0];
const timeFrame = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1];
const outputPeriod = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][0];
const timeFrame = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][1];
nextCandleFile()
@@ -372,7 +372,7 @@
function controlLoop() {
try {
n++
if (n < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length) {
if (n < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length) {
loopBody()
} else {
writeStatusReport(callBackFunction)
@@ -269,7 +269,7 @@
outputCandles = []
outputVolumes = []
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; n++) {
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length; n++) {
const emptyArray1 = []
const emptyArray2 = []
outputCandles.push(emptyArray1)
@@ -286,8 +286,8 @@
loopBody();
function loopBody() {
const outputPeriod = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][0]
const timeFrame = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1]
const outputPeriod = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[n][0]
const timeFrame = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[n][1]
nextCandleFile();
@@ -512,7 +512,7 @@
function controlLoop() {
n++;
if (n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length) {
if (n < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length) {
loopBody()
} else {
writeDataRanges(onWritten);
@@ -276,7 +276,7 @@
loopBody()
function loopBody() {
let timeFrame = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1];
let timeFrame = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][1];
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> findPreviousContent -> loopBody -> timeFrame = " + timeFrame)
@@ -378,7 +378,7 @@
function controlLoop() {
n++
if (n < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length) {
if (n < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length) {
loopBody()
} else {
buildCandles(allPreviousCandles, allPreviousVolumes);
@@ -404,7 +404,7 @@
let outputCandles = [];
let outputVolumes = [];
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; n++) {
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length; n++) {
const emptyArray1 = [];
const emptyArray2 = [];
outputCandles.push(emptyArray1);
@@ -467,8 +467,8 @@
previousVolumes = allPreviousVolumes[n];
}
const outputPeriod = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][0];
const timeFrame = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1];
const outputPeriod = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][0];
const timeFrame = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][1];
/*
Here we are inside a Loop that is going to advance 1 day at the time,
at each pass, we will read one of Exchange Raw Data's daily files and
@@ -760,7 +760,7 @@
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> buildCandles -> timeframesLoop -> controlLoop -> Entering function.")
n++
if (n < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length) {
if (n < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length) {
loopBody()
} else {
writeStatusReport(contextVariables.datetimeLastProducedFile, advanceTime);
@@ -1,762 +1,237 @@
exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (processIndex) {
exports.newSuperalgosBotModulesFromOneMinToMultiPeriodDaily = function (processIndex) {
/*
This module is about converting a One-Min Daily typeo of data set into a Multi Period Daily type.
const MODULE_NAME = "Candles Volumes Multi Period Daily";
const CANDLES_FOLDER_NAME = "Candles";
const CANDLES_ONE_MIN = "One-Min";
const VOLUMES_FOLDER_NAME = "Volumes";
const VOLUMES_ONE_MIN = "One-Min";
The process to do so involves:
Reading the elements (elements, volumens, bolllinger bands, news, asset metrics, etc.) from a
One-Min Dataset (a dataset that is organized with elements spanning one min, like one min begin-end-elements,
or elements with a timestamp captured approximatelly one minute from each other)
organized in Daily Files.
It is going to output a Daily Files dataset for every Daily Time Frame, aggregating the
input information into elements with a begin and end property.
Everytime this process runs, it will be able to resume its job and process everything pending until
reaching the head of the market.
*/
const MODULE_NAME = "From One Min To Multi Period Market"
let thisObject = {
initialize: initialize,
finalize: finalize,
start: start
}
let fileStorage = TS.projects.superalgos.taskModules.fileStorage.newFileStorage(processIndex)
let statusDependencies;
let beginingOfMarket
let statusDependenciesModule
let dataDependenciesModule
let node = {} // Usefull nodes for this module will be stored here.
return thisObject;
function initialize(pStatusDependencies, callBackFunction) {
try {
statusDependencies = pStatusDependencies
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE)
function initialize(pStatusDependencies, pDataDependencies, callBackFunction) {
} catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] initialize -> err = " + err.stack)
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE)
}
statusDependenciesModule = pStatusDependencies
dataDependenciesModule = pDataDependencies
TS.projects.superalgos.functionLibraries.dataAggregationFunctions.checkForKnownConstraints(
dataDependenciesModule,
node,
processIndex,
callBackFunction
)
}
/*
This process is going to do the following:
Read the candles and volumes from Exchange Raw Data and produce a
file for each day and for each period with candles and volumes.
*/
function finalize() {
fileStorage = undefined
statusDependenciesModule = undefined
dataDependenciesModule = undefined
thisObject = undefined
node = undefined
}
function start(callBackFunction) {
try {
/* Context Variables */
let contextVariables = {
datetimeLastProducedFile: undefined, // Datetime of the last file files successfully produced by this process.
datetimeBeginingOfMarketFile: undefined, // Datetime of the first trade file in the whole market history.
datetimeLastAvailableDependencyFile: undefined // Datetime of the last file available to be used as an input of this process.
};
getContextVariables();
function getContextVariables() {
try {
let thisReport;
let reportKey;
let statusReport;
/* We look first for Exchange Raw Data in order to get when the market starts. */
reportKey = "Masters" + "-" + "Exchange-Raw-Data" + "-" + "Historic-OHLCVs"
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> getContextVariables -> reportKey = " + reportKey)
statusReport = statusDependencies.statusReports.get(reportKey);
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE);
return;
}
if (statusReport.status === "Status Report is corrupt.") {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE);
return;
}
thisReport = statusDependencies.statusReports.get(reportKey).file;
if (thisReport.beginingOfMarket === undefined) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet.");
let customOK = {
result: TS.projects.superalgos.globals.standardResponses.CUSTOM_OK_RESPONSE.result,
message: "Dependency does not exist."
}
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> getContextVariables -> customOK = " + customOK.message)
callBackFunction(customOK)
return
}
contextVariables.datetimeBeginingOfMarketFile = new Date(
thisReport.beginingOfMarket.year + "-" +
thisReport.beginingOfMarket.month + "-" +
thisReport.beginingOfMarket.days + " " +
thisReport.beginingOfMarket.hours + ":" +
thisReport.beginingOfMarket.minutes +
TS.projects.superalgos.globals.timeConstants.GMT_SECONDS)
/* Second, we get the report from Exchange Raw Data, to know when the marted ends. */
reportKey = "Masters" + "-" + "Exchange-Raw-Data" + "-" + "Historic-OHLCVs"
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> getContextVariables -> reportKey = " + reportKey)
statusReport = statusDependencies.statusReports.get(reportKey);
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE);
return
}
if (statusReport.status === "Status Report is corrupt.") {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE);
return
}
thisReport = statusDependencies.statusReports.get(reportKey).file
if (thisReport.lastFile === undefined) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
let customOK = {
result: TS.projects.superalgos.globals.standardResponses.CUSTOM_OK_RESPONSE.result,
message: "Dependency not ready."
}
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> getContextVariables -> customOK = " + customOK.message)
callBackFunction(customOK)
return
}
contextVariables.datetimeLastAvailableDependencyFile = new Date(
thisReport.lastFile.year + "-" +
thisReport.lastFile.month + "-" +
thisReport.lastFile.days + " " + "00:00" +
TS.projects.superalgos.globals.timeConstants.GMT_SECONDS)
/* Finally we get our own Status Report. */
reportKey = "Masters" + "-" + "Candles-Volumes" + "-" + "Multi-Period-Daily"
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> getContextVariables -> reportKey = " + reportKey)
statusReport = statusDependencies.statusReports.get(reportKey)
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ")
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE)
return
}
if (statusReport.status === "Status Report is corrupt.") {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process.")
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE)
return
}
thisReport = statusDependencies.statusReports.get(reportKey).file;
if (thisReport.lastFile !== undefined) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> getContextVariables -> Process Running not for the very first time. -> reportKey = " + reportKey)
beginingOfMarket = new Date(thisReport.beginingOfMarket)
if (beginingOfMarket.valueOf() !== contextVariables.datetimeBeginingOfMarketFile.valueOf()) { // Reset Mechanism for Begining of the Market
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> getContextVariables -> Reset Mechanism for Begining of the Market Activated. -> reportKey = " + reportKey);
beginingOfMarket = new Date(contextVariables.datetimeBeginingOfMarketFile.getUTCFullYear() + "-" + (contextVariables.datetimeBeginingOfMarketFile.getUTCMonth() + 1) + "-" + contextVariables.datetimeBeginingOfMarketFile.getUTCDate() + " " + "00:00" + TS.projects.superalgos.globals.timeConstants.GMT_SECONDS);
contextVariables.datetimeLastProducedFile = new Date(
contextVariables.datetimeBeginingOfMarketFile.getUTCFullYear() + "-" +
(contextVariables.datetimeBeginingOfMarketFile.getUTCMonth() + 1) + "-" +
contextVariables.datetimeBeginingOfMarketFile.getUTCDate() + " " + "00:00" +
TS.projects.superalgos.globals.timeConstants.GMT_SECONDS)
contextVariables.datetimeLastProducedFile = new Date(
contextVariables.datetimeLastProducedFile.valueOf() -
TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) // Go back one day to start well.
buildCandles()
return
}
contextVariables.datetimeLastProducedFile = new Date(thisReport.lastFile);
/*
Here we assume that the last day written might contain incomplete information.
This actually happens every time the head of the market is reached.
For that reason we go back one day, the partial information is discarded
and added again with whatever new info is available.
*/
contextVariables.datetimeLastProducedFile = new Date(contextVariables.datetimeLastProducedFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS)
buildCandles()
return
} else {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> getContextVariables -> Process Running for the very first time. -> reportKey = " + reportKey);
beginingOfMarket = new Date(
contextVariables.datetimeBeginingOfMarketFile.getUTCFullYear() + "-" +
(contextVariables.datetimeBeginingOfMarketFile.getUTCMonth() + 1) + "-" +
contextVariables.datetimeBeginingOfMarketFile.getUTCDate() + " " + "00:00" +
TS.projects.superalgos.globals.timeConstants.GMT_SECONDS)
contextVariables.datetimeLastProducedFile = new Date(
contextVariables.datetimeBeginingOfMarketFile.getUTCFullYear() + "-" +
(contextVariables.datetimeBeginingOfMarketFile.getUTCMonth() + 1) + "-" +
contextVariables.datetimeBeginingOfMarketFile.getUTCDate() + " " + "00:00" +
TS.projects.superalgos.globals.timeConstants.GMT_SECONDS)
contextVariables.datetimeLastProducedFile = new Date(contextVariables.datetimeLastProducedFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS); // Go back one day to start well.
buildCandles()
return
}
} catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> getContextVariables -> err = " + err.stack)
if (err.message === "Cannot read property 'file' of undefined") {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[HINT] start -> getContextVariables -> Check the bot configuration to see if all of its statusDependencies declarations are correct. ");
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys))
}
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE)
}
datetimeLastAvailableDependencyFile: undefined, // Datetime of the last file available to be used as an input of this process.
beginingOfMarket: undefined // Datetime of the begining of the market.
}
function buildCandles() {
TS.projects.superalgos.functionLibraries.dataAggregationFunctions.getContextVariables(
statusDependenciesModule,
contextVariables,
buildOutput,
buildOutput,
processIndex,
callBackFunction
)
function buildOutput(allPreviousElements) {
try {
let fromDate = new Date(contextVariables.datetimeLastProducedFile.valueOf())
let lastDate = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(new Date())
let outputCandles
let outputVolumes
/*
Firstly we prepere the arrays that will accumulate all the information for each output file.
*/
let outputElements = []
advanceTime()
for (let timeFrameArrayIndex = 0; timeFrameArrayIndex < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length; timeFrameArrayIndex++) {
const emptyArray = []
outputElements.push(emptyArray)
}
moveToNextDay()
function advanceTime() {
contextVariables.datetimeLastProducedFile = new Date(contextVariables.datetimeLastProducedFile.valueOf() + TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS);
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> buildCandles -> advanceTime -> New processing time @ " + contextVariables.datetimeLastProducedFile.getUTCFullYear() + "/" + (contextVariables.datetimeLastProducedFile.getUTCMonth() + 1) + "/" + contextVariables.datetimeLastProducedFile.getUTCDate() + ".")
/* Validation that we are not going past the head of the market. */
if (contextVariables.datetimeLastProducedFile.valueOf() > contextVariables.datetimeLastAvailableDependencyFile.valueOf()) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> buildCandles -> advanceTime -> Head of the market found @ " + contextVariables.datetimeLastProducedFile.getUTCFullYear() + "/" + (contextVariables.datetimeLastProducedFile.getUTCMonth() + 1) + "/" + contextVariables.datetimeLastProducedFile.getUTCDate() + ".")
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE) // Here is where we finish processing and wait for the platform to run this module again.
return
}
/* Telling the world we are alive and doing well */
let currentDateString =
contextVariables.datetimeLastProducedFile.getUTCFullYear() + '-' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCMonth() + 1, 2) + '-' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCDate(), 2)
let currentDate = new Date(contextVariables.datetimeLastProducedFile)
let percentage = TS.projects.superalgos.utilities.dateTimeFunctions.getPercentage(fromDate, currentDate, lastDate)
TS.projects.superalgos.functionLibraries.processFunctions.processHeartBeat(processIndex, currentDateString, percentage)
if (TS.projects.superalgos.utilities.dateTimeFunctions.areTheseDatesEqual(currentDate, new Date()) === false) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.newInternalLoop(currentDate, percentage);
}
/*
We prepere the arrays that will accumulate all the information for each output file.
*/
outputCandles = []
outputVolumes = []
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; n++) {
const emptyArray1 = []
const emptyArray2 = []
outputCandles.push(emptyArray1)
outputVolumes.push(emptyArray2)
}
timeframesLoop();
function moveToNextDay() {
TS.projects.superalgos.functionLibraries.dataAggregationFunctions.advanceTime(
fromDate,
lastDate,
contextVariables,
timeframesLoop,
processIndex,
callBackFunction
)
}
function timeframesLoop() {
/*
We will iterate through all posible time frames.
*/
let n = 0 // loop Variable representing each possible period as defined at the periods array.
loopBody();
let timeFrameArrayIndex = 0 // loop Variable representing each possible period as defined at the periods array.
loopBody()
function loopBody() {
const outputPeriod = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][0]
const timeFrame = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1]
nextCandleFile();
const TIME_FRAME_VALUE = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[timeFrameArrayIndex][0]
const TIME_FRAME_LABEL = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[timeFrameArrayIndex][1]
/*
Here we are inside a Loop that is going to advance 1 day at the time,
at each pass, we will read one of Exchange Raw Data's daily files and
add all its elements to our in memory arrays.
*/
TS.projects.superalgos.functionLibraries.dataAggregationFunctions.nextDependencyDailyFile(
contextVariables,
node,
aggregateAndWriteOutputFile,
fileStorage,
processIndex,
callBackFunction
)
function nextCandleFile() {
let dateForPath =
contextVariables.datetimeLastProducedFile.getUTCFullYear() + '/' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCMonth() + 1, 2) + '/' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCDate(), 2)
let fileName = "Data.json"
function aggregateAndWriteOutputFile(dependencyDailyFile) {
/*
Here we call the function that will aggregate all the information
at the dependency file into standarized begin-end-elements.
*/
TS.projects.superalgos.functionLibraries.dataAggregationFunctions.aggregateFileContent(
node,
contextVariables,
dependencyDailyFile,
outputElements[timeFrameArrayIndex],
TIME_FRAME_VALUE
)
let filePathRoot =
'Project/' +
TS.projects.superalgos.globals.taskConstants.PROJECT_DEFINITION_NODE.config.codeName + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.type.replace(' ', '-') + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName + "/" +
"Exchange-Raw-Data" + '/' +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.config.codeName + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "-" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName
let filePath = filePathRoot + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath
filePath += '/' + fileName
writeOutputFile(outputElements[timeFrameArrayIndex], TIME_FRAME_LABEL, controlLoop)
fileStorage.getTextFile(filePath, onFileReceived)
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> buildCandles -> timeframesLoop -> loopBody -> getting file at dateForPath = " + dateForPath)
function onFileReceived(err, text) {
function writeOutputFile(outputElementsCurrentTimeFrame, TIME_FRAME_LABEL, callBack) {
/*
Here we will write the contents of the file to disk.
*/
try {
let candlesFile;
let fileContent = TS.projects.superalgos.functionLibraries.dataAggregationFunctions.generateOutputFileContent(
node,
outputElementsCurrentTimeFrame
)
/*
Once the file content is generated, we need to reset this array.
*/
outputElementsCurrentTimeFrame = []
/*
Daily files are saved to disk inside a folder for each year / month / day
*/
let file = { date: new Date(contextVariables.datetimeLastProducedFile.valueOf()) }
file.year = file.date.getUTCFullYear()
file.month = file.date.getUTCMonth() + 1
file.day = file.date.getUTCDate()
let dateForPath = file.year + '/' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(file.month, 2) + '/' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(file.day, 2)
if (err.result === TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
try {
candlesFile = JSON.parse(text);
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT +
"/Output/" +
node.outputDataset.referenceParent.parentNode.config.codeName + "/" +
node.outputDataset.referenceParent.config.codeName + "/" +
TIME_FRAME_LABEL + "/" +
dateForPath
} catch (err) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> Error Parsing JSON -> err = " + err.stack)
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE)
return;
}
} else {
filePath += '/' + fileName
if (err.message === 'File does not exist.' || err.code === 'The specified key does not exist.') {
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated)
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> Dependency Not Ready -> err = " + err.stack)
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE)
return
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> writeOutputFile -> creating file at filePath = " + filePath)
} else {
function onFileCreated(err) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> Error Received -> err = " + err.stack)
callBackFunction(err);
"[ERROR] start -> writeOutputFile -> onFileCreated -> err = " + err.stack)
callBackFunction(err)
return
}
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> writeOutputFile -> onFileCreated -> Finished with File @ " +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "_" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName + ", " +
" filePath = " + filePath + "/" + fileName)
callBack()
}
const inputCandlesPerdiod = 60 * 1000; // 1 min
const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs
let totalOutputCandles = inputFilePeriod / outputPeriod;
let beginingOutputTime = contextVariables.datetimeLastProducedFile.valueOf();
for (let i = 0; i < totalOutputCandles; i++) {
let outputCandle = {
open: 0,
close: 0,
min: 0,
max: 0,
begin: 0,
end: 0
}
let saveCandle = false;
outputCandle.begin = beginingOutputTime + i * outputPeriod;
outputCandle.end = beginingOutputTime + (i + 1) * outputPeriod - 1;
for (let j = 0; j < candlesFile.length; j++) {
let candle = {
open: candlesFile[j][2],
close: candlesFile[j][3],
min: candlesFile[j][0],
max: candlesFile[j][1],
begin: candlesFile[j][4],
end: candlesFile[j][5]
};
/* Here we discard all the candles out of range. */
if (candle.begin >= outputCandle.begin && candle.end <= outputCandle.end) {
if (saveCandle === false) { // this will set the value only once.
outputCandle.open = candle.open;
outputCandle.min = candle.min
outputCandle.max = candle.max
}
saveCandle = true;
outputCandle.close = candle.close; // only the last one will be saved
if (candle.min < outputCandle.min) {
outputCandle.min = candle.min
}
if (candle.max > outputCandle.max) {
outputCandle.max = candle.max
}
}
}
if (saveCandle === true) { // then we have a valid candle, otherwise it means there were no candles to fill this one in its time range.
outputCandles[n].push(outputCandle)
}
}
nextVolumeFile();
} catch (err) {
}
catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextCandleFile -> onFileReceived -> err = " + err.stack);
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
"[ERROR] start -> writeOutputFile -> err = " + err.stack)
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE)
}
}
}
function nextVolumeFile() {
try {
let dateForPath = contextVariables.datetimeLastProducedFile.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCDate(), 2);
let fileName = "Data.json"
let filePathRoot =
'Project/' +
TS.projects.superalgos.globals.taskConstants.PROJECT_DEFINITION_NODE.config.codeName + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.type.replace(' ', '-') + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName + "/" +
"Exchange-Raw-Data" + '/' + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.config.codeName + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "-" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName
let filePath =
filePathRoot +
"/Output/" +
VOLUMES_FOLDER_NAME + '/' +
VOLUMES_ONE_MIN + '/' +
dateForPath
filePath += '/' + fileName
fileStorage.getTextFile(filePath, onFileReceived);
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> buildCandles -> timeframesLoop -> loopBody -> nextVolumeFile -> getting file at dateForPath = " + dateForPath);
function onFileReceived(err, text) {
let volumesFile
if (err.result === TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
try {
volumesFile = JSON.parse(text);
} catch (err) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Error Parsing JSON -> err = " + err.stack);
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.")
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_RETRY_RESPONSE)
return
}
} else {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Error Received -> err = " + err.stack)
callBackFunction(err)
return
}
const inputVolumesPerdiod = 60 * 1000; // 1 min
const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs
let totalOutputVolumes = inputFilePeriod / outputPeriod; // this should be 2 in this case.
let beginingOutputTime = contextVariables.datetimeLastProducedFile.valueOf();
for (let i = 0; i < totalOutputVolumes; i++) {
let outputVolume = {
buy: 0,
sell: 0,
begin: 0,
end: 0
}
let saveVolume = false;
outputVolume.begin = beginingOutputTime + i * outputPeriod;
outputVolume.end = beginingOutputTime + (i + 1) * outputPeriod - 1;
for (let j = 0; j < volumesFile.length; j++) {
let volume = {
buy: volumesFile[j][0],
sell: volumesFile[j][1],
begin: volumesFile[j][2],
end: volumesFile[j][3]
}
/* Here we discard all the Volumes out of range. */
if (volume.begin >= outputVolume.begin && volume.end <= outputVolume.end) {
saveVolume = true
outputVolume.buy = outputVolume.buy + volume.buy
outputVolume.sell = outputVolume.sell + volume.sell
}
}
if (saveVolume === true) {
outputVolumes[n].push(outputVolume);
}
}
writeFiles(outputCandles[n], outputVolumes[n], timeFrame, controlLoop)
}
} catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> buildCandles -> timeframesLoop -> loopBody -> nextVolumeFile -> onFileReceived -> err = " + err.stack);
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
}
}
}
function controlLoop() {
n++;
if (n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length) {
timeFrameArrayIndex++
if (timeFrameArrayIndex < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length) {
loopBody()
} else {
writeDataRanges(onWritten);
function onWritten(err) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] writeDataRanges -> writeDataRanges -> onCandlesDataRangeWritten -> err = " + err.stack)
callBack(err)
return
}
writeStatusReport(contextVariables.datetimeLastProducedFile, advanceTime);
}
TS.projects.superalgos.functionLibraries.dataAggregationFunctions.writeStatusReport(
statusDependenciesModule,
contextVariables,
contextVariables.datetimeLastProducedFile,
moveToNextDay,
processIndex,
callBackFunction
)
}
}
}
}
catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> buildCandles -> err = " + err.stack);
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
"[ERROR] start -> buildOutput -> err = " + err.stack)
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE)
}
}
function writeFiles(candles, volumes, timeFrame, callBack) {
/*
Here we will write the contents of the Candles and Volumens files.
*/
try {
writeCandles()
function writeCandles() {
let separator = ""
let fileRecordCounter = 0
let fileContent = ""
for (let i = 0; i < candles.length; i++) {
let candle = candles[i]
fileContent = fileContent + separator + '[' + candles[i].min + "," + candles[i].max + "," + candles[i].open + "," + candles[i].close + "," + candles[i].begin + "," + candles[i].end + "]";
if (separator === "") { separator = ","; }
fileRecordCounter++
}
fileContent = "[" + fileContent + "]";
let fileName = 'Data.json';
let dateForPath =
contextVariables.datetimeLastProducedFile.getUTCFullYear() + '/' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCMonth() + 1, 2) + '/' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCDate(), 2)
let filePath =
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT +
"/Output/" +
CANDLES_FOLDER_NAME + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.codeName + "/" +
timeFrame + "/" +
dateForPath
filePath += '/' + fileName
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated)
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> writeFiles -> writeCandles -> creating file at filePath = " + filePath);
function onFileCreated(err) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> writeFiles -> writeCandles -> onFileCreated -> err = " + err.stack);
callBackFunction(err);
return;
}
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> writeFiles -> writeCandles -> onFileCreated -> Finished with File @ " + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "_" + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName);
writeVolumes();
}
}
function writeVolumes() {
let separator = "";
let fileRecordCounter = 0;
let fileContent = "";
for (let i = 0; i < volumes.length; i++) {
let candle = volumes[i];
fileContent = fileContent + separator + '[' + volumes[i].buy + "," + volumes[i].sell + "," + volumes[i].begin + "," + volumes[i].end + "]";
if (separator === "") { separator = ","; }
fileRecordCounter++
}
fileContent = "[" + fileContent + "]";
let fileName = 'Data.json';
let dateForPath =
contextVariables.datetimeLastProducedFile.getUTCFullYear() + '/' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCMonth() + 1, 2) + '/' +
TS.projects.superalgos.utilities.miscellaneousFunctions.pad(contextVariables.datetimeLastProducedFile.getUTCDate(), 2)
let filePath =
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT +
"/Output/" +
VOLUMES_FOLDER_NAME + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.codeName + "/" +
timeFrame + "/" + dateForPath
filePath += '/' + fileName
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated);
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> writeFiles -> writeVolumes -> creating file at filePath = " + filePath);
function onFileCreated(err) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> writeFiles -> writeVolumes -> onFileCreated -> err = " + err.stack)
callBackFunction(err)
return
}
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> writeFiles -> writeVolumes -> onFileCreated -> Finished with File @ " + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "_" + TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName);
callBack()
}
}
}
catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> writeFiles -> err = " + err.stack);
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
}
}
function writeDataRanges(callBack) {
try {
writeDataRange(contextVariables.datetimeBeginingOfMarketFile, contextVariables.datetimeLastProducedFile, CANDLES_FOLDER_NAME, onCandlesDataRangeWritten);
function onCandlesDataRangeWritten(err) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] writeDataRanges -> writeDataRanges -> onCandlesDataRangeWritten -> err = " + err.stack)
callBack(err)
return
}
writeDataRange(contextVariables.datetimeBeginingOfMarketFile, contextVariables.datetimeLastProducedFile, VOLUMES_FOLDER_NAME, onVolumeDataRangeWritten);
function onVolumeDataRangeWritten(err) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] writeDataRanges -> writeDataRanges -> onVolumeDataRangeWritten -> err = " + err.stack)
callBack(err)
return
}
callBack(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE);
}
}
}
catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> writeDataRanges -> err = " + err.stack);
callBack(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
}
}
function writeDataRange(pBegin, pEnd, pProductFolder, callBack) {
try {
let dataRange = {
begin: pBegin.valueOf(),
end: pEnd.valueOf() + TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS
}
let fileContent = JSON.stringify(dataRange);
let fileName = 'Data.Range.json';
let filePath =
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + pProductFolder + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.codeName
filePath += '/' + fileName
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated);
function onFileCreated(err) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> writeDataRange -> onFileCreated -> err = " + err.stack)
callBack(err)
return
}
let key =
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.parentNode.config.codeName + "-" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.parentNode.config.codeName + "-" +
pProductFolder + "-" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.config.codeName + "-" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + '/' +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName
let event = {
dateRange: dataRange
}
TS.projects.superalgos.globals.taskConstants.EVENT_SERVER_CLIENT_MODULE_OBJECT.raiseEvent(key, 'Data Range Updated', event)
callBack(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE);
}
}
catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> writeDataRange -> err = " + err.stack)
callBack(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE)
}
}
function writeStatusReport(lastFileDate, callBack) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> writeStatusReport -> lastFileDate = " + lastFileDate)
try {
let reportKey = "Masters" + "-" + "Candles-Volumes" + "-" + "Multi-Period-Daily"
let thisReport = statusDependencies.statusReports.get(reportKey)
thisReport.file.lastExecution = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).PROCESS_DATETIME;
thisReport.file.lastFile = lastFileDate;
thisReport.file.beginingOfMarket = beginingOfMarket.toUTCString()
thisReport.save(callBack);
}
catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> writeStatusReport -> err = " + err.stack);
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] 'Start' - ERROR : " + err.stack);
"[ERROR] start -> err = " + err.stack)
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE)
}
}
@@ -1,9 +1,11 @@
exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (processIndex) {
/*
This process is going to do the following:
This module is about converting a One-Min Daily typeo of data set into a Multi Period Market type.
The process to do so involves:
Read the elements (elements, volumens, bolllinger bands, news, asset metrics, etc.) from a
One-Min dataset (a dataset that is organized with elements spanning one min, like one min begin-end-elements,
Reading the elements (elements, volumens, bolllinger bands, news, asset metrics, etc.) from a
One-Min Dataset (a dataset that is organized with elements spanning one min, like one min begin-end-elements,
or elements with a timestamp captured approximatelly one minute from each other)
organized in Daily Files.
@@ -18,10 +20,8 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process
2. Then it will append to these arrays the new information it gets from the data dependency file.
3. It knows from it's status report which was the last DAY it processed. Since that day might not have been
completed (maybe it was at the head of the market). The process will have to be carefull not to append elements
that are already belonging to the first day to process. To take care of that, it will discard all elements
of the last processed day, and then it will process that full day again adding all the elements found at the current run.
3. The already saved elements belonging to the start processing day will be discarded and replaced by the
dependency elements of the same day, potentially completing missing elements.
*/
const MODULE_NAME = "From One Min To Multi Period Market"
@@ -92,7 +92,7 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process
loopBody()
function loopBody() {
const TIME_FRAME_LABEL = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[timeFrameArrayIndex][1];
const TIME_FRAME_LABEL = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[timeFrameArrayIndex][1];
let previousElements
@@ -143,7 +143,7 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process
function controlLoop() {
timeFrameArrayIndex++
if (timeFrameArrayIndex < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length) {
if (timeFrameArrayIndex < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length) {
loopBody()
} else {
buildOutput(allPreviousElements)
@@ -168,7 +168,7 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process
*/
let outputElements = []
for (let timeFrameArrayIndex = 0; timeFrameArrayIndex < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; timeFrameArrayIndex++) {
for (let timeFrameArrayIndex = 0; timeFrameArrayIndex < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length; timeFrameArrayIndex++) {
const emptyArray = []
outputElements.push(emptyArray)
}
@@ -200,8 +200,8 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process
previousElements = allPreviousElements[timeFrameArrayIndex];
}
const TIME_FRAME_VALUE = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[timeFrameArrayIndex][0]
const TIME_FRAME_LABEL = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[timeFrameArrayIndex][1]
const TIME_FRAME_VALUE = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[timeFrameArrayIndex][0]
const TIME_FRAME_LABEL = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[timeFrameArrayIndex][1]
/*
Here we are inside a Loop that is going to advance 1 day at the time,
at each pass, we will read one of Exchange Raw Data's daily files and
@@ -259,17 +259,17 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process
writeOutputFile(outputElements[timeFrameArrayIndex], TIME_FRAME_LABEL, controlLoop)
function writeOutputFile(outputElements, TIME_FRAME_LABEL, callBack) {
function writeOutputFile(outputElementsCurrentTimeFrame, TIME_FRAME_LABEL, callBack) {
/*
Here we will write the contents of the file to disk.
*/
try {
let fileContent = TS.projects.superalgos.functionLibraries.dataAggregationFunctions.generateOutputFileContent(
node,
outputElements
outputElementsCurrentTimeFrame
)
let fileName = 'Data.json';
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT +
"/Output/" +
node.outputDataset.referenceParent.parentNode.config.codeName + "/" +
@@ -311,7 +311,7 @@ exports.newSuperalgosBotModulesFromOneMinToMultiPeriodMarket = function (process
function controlLoop() {
timeFrameArrayIndex++
if (timeFrameArrayIndex < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length) {
if (timeFrameArrayIndex < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length) {
loopBody()
} else {
TS.projects.superalgos.functionLibraries.dataAggregationFunctions.writeStatusReport(
@@ -208,7 +208,7 @@
*/
bootstrappingTheProcess = true
for (let i = 0; i < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; i++) {
for (let i = 0; i < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length; i++) {
let emptyObject = {};
interExecutionMemoryArray.push(emptyObject)
}
@@ -298,8 +298,8 @@
function timeFramesLoopBody() {
const timeFrame = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][0]
const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1]
const timeFrame = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[n][0]
const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[n][1]
/* Check Time Frames Filter */
if (TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.timeFramesFilter !== undefined) {
@@ -486,7 +486,7 @@
function timeFramesControlLoop() {
n++
if (n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length) {
if (n < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length) {
timeFramesLoopBody()
} else {
n = 0;
@@ -567,8 +567,8 @@
function writeTimeFramesFile(productCodeName, callBack) {
let timeFramesArray = []
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; n++) {
let timeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1]
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length; n++) {
let timeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[n][1]
/* Check Time Frames Filter */
if (TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.timeFramesFilter !== undefined) {
@@ -52,8 +52,8 @@
}
function timeFramesLoopBody() {
const timeFrame = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][0]
const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1]
const timeFrame = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][0]
const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][1]
/* Check Time Frames Filter */
if (TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.timeFramesFilter !== undefined) {
@@ -138,7 +138,7 @@
function timeFramesControlLoop() {
n++;
if (n < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length) {
if (n < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length) {
timeFramesLoopBody()
} else {
writeTimeFramesFiles(onTimeFrameFilesWritten)
@@ -173,8 +173,8 @@
function writeTimeFramesFile(productCodeName, callBack) {
let timeFramesArray = []
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; n++) {
let timeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1]
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length; n++) {
let timeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][1]
/* Check Time Frames Filter */
if (TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.timeFramesFilter !== undefined) {
@@ -401,18 +401,18 @@
function getTimeFrameFromLabel(timeFrameLabel) {
for (let i = 0; i < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; i++) {
let value = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[i][0]
let label = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[i][1]
for (let i = 0; i < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length; i++) {
let value = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[i][0]
let label = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[i][1]
if (timeFrameLabel === label) {
return value
}
}
for (let i = 0; i < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; i++) {
let value = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[i][0]
let label = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[i][1]
for (let i = 0; i < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length; i++) {
let value = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[i][0]
let label = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[i][1]
if (timeFrameLabel === label) {
return value
@@ -22,7 +22,7 @@ exports.newSuperalgosBotModulesLearningOutput = function (processIndex) {
try {
let fileStorage = TS.projects.superalgos.taskModules.fileStorage.newFileStorage(processIndex)
if (timeFrame > TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[0][0]) {
if (timeFrame > TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[0][0]) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES = false
} else {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES = true
@@ -407,8 +407,8 @@ exports.newSuperalgosBotModulesLearningSimulation = function (processIndex) {
/* Finding the Current Element on Market Files */
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) {
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; j++) {
let mapKey = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[j][1]
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length; j++) {
let mapKey = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[j][1]
let propertyName = 'at' + mapKey.replace('-', '')
let thisChart = chart[propertyName]
@@ -430,8 +430,8 @@ exports.newSuperalgosBotModulesLearningSimulation = function (processIndex) {
}
/* Finding the Current Element on Market Files */
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; j++) {
let mapKey = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[j][1]
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length; j++) {
let mapKey = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[j][1]
let propertyName = 'at' + mapKey.replace('-', '')
let thisChart = chart[propertyName]
@@ -22,7 +22,7 @@ exports.newSuperalgosBotModulesTradingOutput = function (processIndex) {
try {
let fileStorage = TS.projects.superalgos.taskModules.fileStorage.newFileStorage(processIndex)
if (timeFrame > TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[0][0]) {
if (timeFrame > TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[0][0]) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES = false
} else {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES = true
@@ -376,8 +376,8 @@ exports.newSuperalgosBotModulesTradingSimulation = function (processIndex) {
/* Finding the Current Element on Daily Files */
if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) {
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; j++) {
let mapKey = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[j][1]
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length; j++) {
let mapKey = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[j][1]
let propertyName = 'at' + mapKey.replace('-', '')
let thisChart = chart[propertyName]
@@ -399,8 +399,8 @@ exports.newSuperalgosBotModulesTradingSimulation = function (processIndex) {
}
/* Finding the Current Element on Market Files */
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; j++) {
let mapKey = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[j][1]
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length; j++) {
let mapKey = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[j][1]
let propertyName = 'at' + mapKey.replace('-', '')
let thisChart = chart[propertyName]
@@ -613,18 +613,18 @@
function getTimeFrameFromLabel(timeFrameLabel) {
for (let i = 0; i < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; i++) {
let value = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[i][0]
let label = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[i][1]
for (let i = 0; i < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length; i++) {
let value = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[i][0]
let label = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[i][1]
if (timeFrameLabel === label) {
return value
}
}
for (let i = 0; i < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; i++) {
let value = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[i][0]
let label = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[i][1]
for (let i = 0; i < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length; i++) {
let value = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[i][0]
let label = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[i][1]
if (timeFrameLabel === label) {
return value
@@ -63,8 +63,8 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () {
function getContextVariables(
statusDependenciesModule,
contextVariables,
loadExistingFiles,
buildOutput,
callbackNotFirstRun,
callbackFirstRun,
processIndex,
callBackFunction
) {
@@ -205,7 +205,7 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () {
contextVariables.datetimeLastProducedFile.valueOf() -
TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) // Go back one day to start well.
buildOutput()
callbackFirstRun()
return
}
@@ -218,7 +218,7 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () {
*/
contextVariables.datetimeLastProducedFile = new Date(contextVariables.datetimeLastProducedFile.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS)
loadExistingFiles()
callbackNotFirstRun()
return
} else {
/*
@@ -239,7 +239,7 @@ exports.newSuperalgosFunctionLibrariesDataAggregationFunctions = function () {
contextVariables.datetimeLastProducedFile.valueOf() -
TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS) // Go back one day to start well.
buildOutput()
callbackFirstRun()
return
}
@@ -118,16 +118,16 @@ exports.newSuperalgosFunctionLibrariesDataDependenciesFunctions = function () {
/*
We will iterate through all posible timeFrames.
*/
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; n++) {
const timeFrame = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][0]
const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1]
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length; n++) {
const timeFrame = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][0]
const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][1]
dataFiles = new Map()
/* Current Time Frame detection */
if (userDefinedTimeFrame === timeFrameLabel) {
currentTimeFrame.value = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][0]
currentTimeFrame.label = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1]
currentTimeFrame.value = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][0]
currentTimeFrame.label = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][1]
}
/* Loop across the list of curated dependencies */
@@ -223,7 +223,7 @@ exports.newSuperalgosFunctionLibrariesDataDependenciesFunctions = function () {
}
}
let mapKey = currentExchange + '-' + currentMarket + '-' + TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[n][1]
let mapKey = currentExchange + '-' + currentMarket + '-' + TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[n][1]
multiTimeFrameDataFiles.set(mapKey, dataFiles)
}
}
@@ -257,8 +257,8 @@ exports.newSuperalgosFunctionLibrariesDataDependenciesFunctions = function () {
/*
We will iterate through all posible timeFrames.
*/
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; n++) {
const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1]
for (let n = 0; n < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length; n++) {
const timeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[n][1]
if (TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent.config.framework.validtimeFrames !== undefined) {
let validPeriod = false
@@ -272,8 +272,8 @@ exports.newSuperalgosFunctionLibrariesDataDependenciesFunctions = function () {
}
if (userDefinedTimeFrame === timeFrameLabel) {
currentTimeFrame.value = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][0]
currentTimeFrame.label = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1]
currentTimeFrame.value = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[n][0]
currentTimeFrame.label = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[n][1]
}
dataFiles = new Map()
@@ -350,7 +350,7 @@ exports.newSuperalgosFunctionLibrariesDataDependenciesFunctions = function () {
return JSON.parse(response.text)
}
}
let mapKey = currentExchange + '-' + currentMarket + '-' + TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1]
let mapKey = currentExchange + '-' + currentMarket + '-' + TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[n][1]
multiTimeFrameDataFiles.set(mapKey, dataFiles)
}
@@ -420,8 +420,8 @@ exports.newSuperalgosFunctionLibrariesDataDependenciesFunctions = function () {
let quotedAsset = splittedMarket[1]
/* Market Files */
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.marketFilesPeriods().length; j++) {
let timeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketFilesPeriods()[j][1]
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray().length; j++) {
let timeFrameLabel = TS.projects.superalgos.globals.timeFrames.marketTimeFramesArray()[j][1]
let dataFiles = multiTimeFrameDataFiles.get(currentExchange + '-' + currentMarket + '-' + timeFrameLabel)
let products = {}
@@ -434,8 +434,8 @@ exports.newSuperalgosFunctionLibrariesDataDependenciesFunctions = function () {
}
/* Daily Files */
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.dailyFilePeriods().length; j++) {
let timeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[j][1]
for (let j = 0; j < TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray().length; j++) {
let timeFrameLabel = TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[j][1]
let dataFiles = multiTimeFrameDataFiles.get(currentExchange + '-' + currentMarket + '-' + timeFrameLabel)
let products = {}
@@ -128,7 +128,7 @@ exports.newSuperalgosFunctionLibrariesProcessFilesFunctions = function () {
await writeTimeFramesFiles()
await writeDataRanges()
if (currentTimeFrame.value > TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[0][0]) {
if (currentTimeFrame.value > TS.projects.superalgos.globals.timeFrames.dailyTimeFramesArray()[0][0]) {
await writeMarketStatusReport()
} else {
await writeDailyStatusReport()
+8 -8
View File
@@ -1,15 +1,15 @@
exports.newSuperalgosGlobalsTimeFrames = function () {
let thisObject = {
marketFilesPeriods: marketFilesPeriods,
dailyFilePeriods: dailyFilePeriods
marketTimeFramesArray: marketTimeFramesArray,
dailyTimeFramesArray: dailyTimeFramesArray
}
return thisObject
function marketFilesPeriods() {
function marketTimeFramesArray() {
let marketFilesPeriods =
let marketTimeFramesArray =
'[' +
'[' + 24 * 60 * 60 * 1000 + ',' + '"24-hs"' + ']' + ',' +
'[' + 12 * 60 * 60 * 1000 + ',' + '"12-hs"' + ']' + ',' +
@@ -20,12 +20,12 @@ exports.newSuperalgosGlobalsTimeFrames = function () {
'[' + 2 * 60 * 60 * 1000 + ',' + '"02-hs"' + ']' + ',' +
'[' + 1 * 60 * 60 * 1000 + ',' + '"01-hs"' + ']' + ']'
return JSON.parse(marketFilesPeriods)
return JSON.parse(marketTimeFramesArray)
}
function dailyFilePeriods() {
function dailyTimeFramesArray() {
let dailyFilePeriods =
let dailyTimeFramesArray =
'[' +
'[' + 45 * 60 * 1000 + ',' + '"45-min"' + ']' + ',' +
'[' + 40 * 60 * 1000 + ',' + '"40-min"' + ']' + ',' +
@@ -39,6 +39,6 @@ exports.newSuperalgosGlobalsTimeFrames = function () {
'[' + 02 * 60 * 1000 + ',' + '"02-min"' + ']' + ',' +
'[' + 01 * 60 * 1000 + ',' + '"01-min"' + ']' + ']'
return JSON.parse(dailyFilePeriods)
return JSON.parse(dailyTimeFramesArray)
}
}
@@ -28,7 +28,7 @@ function newTimeFrameScale() {
let filePeriodIndex = FILES_PERIOD_DEFAULT_VALUE
let timeFrameIndex = TIME_PERIOD_DEFAULT_VALUE
let timeFramesMasterArray = [marketFilesPeriods, dailyFilePeriods]
let timeFramesMasterArray = [marketTimeFramesArray, dailyTimeFramesArray]
let timeFrameLabel = '24-hs'
let onMouseWheelEventSubscriptionId
+6 -6
View File
@@ -150,9 +150,9 @@ function newDailyFiles() {
function createFileCursors() {
try {
/* Now we will get the daily files */
for (let i = 0; i < dailyFilePeriods.length; i++) {
let periodTime = dailyFilePeriods[i][0]
let periodName = dailyFilePeriods[i][1]
for (let i = 0; i < dailyTimeFramesArray.length; i++) {
let periodTime = dailyTimeFramesArray[i][0]
let periodName = dailyTimeFramesArray[i][1]
if (pDataset.config.validTimeFrames.includes(periodName) === false) {
continue
@@ -198,9 +198,9 @@ function newDailyFiles() {
function loadThemAll() {
try {
for (let i = 0; i < dailyFilePeriods.length; i++) {
let periodTime = dailyFilePeriods[i][0]
let periodName = dailyFilePeriods[i][1]
for (let i = 0; i < dailyTimeFramesArray.length; i++) {
let periodTime = dailyTimeFramesArray[i][0]
let periodName = dailyTimeFramesArray[i][1]
if (pDataset.config.validTimeFrames.includes(periodName) === true) {
let fileCursor = fileCursors.get(periodTime)
+2 -2
View File
@@ -207,8 +207,8 @@ function newFileCursor() {
let positionA
for (let i = 0; i < dailyFilePeriods.length; i++) {
let period = dailyFilePeriods[i]
for (let i = 0; i < dailyTimeFramesArray.length; i++) {
let period = dailyTimeFramesArray[i]
if (period[0] === pTimeFrame) {
positionA = i
+8 -8
View File
@@ -148,9 +148,9 @@ function newMarketFiles() {
function getMarketFiles() {
/* Now we will get the market files */
for (let i = 0; i < marketFilesPeriods.length; i++) {
let periodTime = marketFilesPeriods[i][0]
let periodName = marketFilesPeriods[i][1]
for (let i = 0; i < marketTimeFramesArray.length; i++) {
let periodTime = marketTimeFramesArray[i][0]
let periodName = marketTimeFramesArray[i][1]
if (dataset.config.validTimeFrames.includes(periodName) === false) {
filesNotLoaded++
@@ -176,7 +176,7 @@ function newMarketFiles() {
filesNotLoaded++
}
if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) {
if (filesLoaded + filesNotLoaded === marketTimeFramesArray.length) {
let key = mine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.config.codeName + '-' + market.baseAsset + '/' + market.quotedAsset
let callerId = key + '-' + periodName + newUniqueId()
eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles)
@@ -209,9 +209,9 @@ function newMarketFiles() {
/* Now we will get the market files */
for (let i = 0; i < marketFilesPeriods.length; i++) {
let periodTime = marketFilesPeriods[i][0]
let periodName = marketFilesPeriods[i][1]
for (let i = 0; i < marketTimeFramesArray.length; i++) {
let periodTime = marketTimeFramesArray[i][0]
let periodName = marketTimeFramesArray[i][1]
if (dataset.config.validTimeFrames.includes(periodName) !== true) { continue }
if (message.event !== undefined) {
@@ -245,7 +245,7 @@ function newMarketFiles() {
}
function getExpectedFiles() {
return marketFilesPeriods.length
return marketTimeFramesArray.length
}
function getFilesLoaded() {
+4 -4
View File
@@ -129,7 +129,7 @@ const UI_PANEL = {
};
let marketFilesPeriods =
let marketTimeFramesArray =
'[' +
'[' + 24 * 60 * 60 * 1000 + ',' + '"24-hs"' + ']' + ',' +
'[' + 12 * 60 * 60 * 1000 + ',' + '"12-hs"' + ']' + ',' +
@@ -140,9 +140,9 @@ let marketFilesPeriods =
'[' + 2 * 60 * 60 * 1000 + ',' + '"02-hs"' + ']' + ',' +
'[' + 1 * 60 * 60 * 1000 + ',' + '"01-hs"' + ']' + ']';
marketFilesPeriods = JSON.parse(marketFilesPeriods);
marketTimeFramesArray = JSON.parse(marketTimeFramesArray);
let dailyFilePeriods =
let dailyTimeFramesArray =
'[' +
'[' + 45 * 60 * 1000 + ',' + '"45-min"' + ']' + ',' +
'[' + 40 * 60 * 1000 + ',' + '"40-min"' + ']' + ',' +
@@ -156,7 +156,7 @@ let dailyFilePeriods =
'[' + 02 * 60 * 1000 + ',' + '"02-min"' + ']' + ',' +
'[' + 01 * 60 * 1000 + ',' + '"01-min"' + ']' + ']';
dailyFilePeriods = JSON.parse(dailyFilePeriods);
dailyTimeFramesArray = JSON.parse(dailyTimeFramesArray);
let SHOW_ANIMATION_PERFORMACE = false;
let MENU_ITEM_ON_FOCUS
+4 -4
View File
@@ -37,15 +37,15 @@ function dynamicDecimals(value, minDecimals) {
}
function convertTimeFrameToName(pTimeFrame) {
for (let i = 0; i < dailyFilePeriods.length; i++) {
let period = dailyFilePeriods[i]
for (let i = 0; i < dailyTimeFramesArray.length; i++) {
let period = dailyTimeFramesArray[i]
if (period[0] === pTimeFrame) {
return period[1]
}
}
for (let i = 0; i < marketFilesPeriods.length; i++) {
let period = marketFilesPeriods[i]
for (let i = 0; i < marketTimeFramesArray.length; i++) {
let period = marketTimeFramesArray[i]
if (period[0] === pTimeFrame) {
return period[1]
}