fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Beta 6 without Execution

Browse Source
This commit is contained in:
Luis Fernando Molina committed 2020-07-06 15:43:34 +02:00
1 parent 4033d4d07f
commit d40b985f5e
123 files changed
+8114 -12392
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

TaskServer/Commons.js
+35 -37
View File
@@ -2,8 +2,6 @@
const FULL_LOG = true;
const LOG_FILE_CONTENT = false;
const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000;
const ONE_MIN_IN_MILISECONDS = 60 * 1000;
const MODULE_NAME = "Commons";
@@ -12,7 +10,7 @@
validateOutputDatasets: validateOutputDatasets,
inflateDatafiles: inflateDatafiles,
dataBuildingProcedure: dataBuildingProcedure,
calculationsProcedure: calculationsProcedure,
calculationsProcedure: calculationsProcedure,
generateFileContent: generateFileContent,
writeFile: writeFile
};
@@ -25,15 +23,15 @@
function validateDataDependencies(dataDependencies, callBackFunction) {
for (let i = 0; i < dataDependencies.length; i++) {
let dataDependencyNode = dataDependencies[i]
let dataDependencyNode = dataDependencies[i]
/* Basic validations to see if we have everything we need. */
if (dataDependencyNode.referenceParent.parentNode.code.singularVariableName === undefined) {
if (dataDependencyNode.referenceParent.parentNode.config.singularVariableName === undefined) {
logger.write(MODULE_NAME, "[ERROR] start -> Product Definition without a Single Variable Name defined. Product Definition = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (dataDependencyNode.referenceParent.parentNode.code.pluralVariableName === undefined) {
if (dataDependencyNode.referenceParent.parentNode.config.pluralVariableName === undefined) {
logger.write(MODULE_NAME, "[ERROR] start -> Product Definition without a Plural Variable Name defined. Product Definition = " + JSON.stringify(dataDependencyNode.referenceParent.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
@@ -61,12 +59,12 @@
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (outputDatasetNode.referenceParent.parentNode.code.singularVariableName === undefined) {
if (outputDatasetNode.referenceParent.parentNode.config.singularVariableName === undefined) {
logger.write(MODULE_NAME, "[ERROR] start -> Product Definition without a Single Variable Name defined. Product Definition = " + JSON.stringify(outputDatasetNode.referenceParent.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
}
if (outputDatasetNode.referenceParent.parentNode.code.pluralVariableName === undefined) {
if (outputDatasetNode.referenceParent.parentNode.config.pluralVariableName === undefined) {
logger.write(MODULE_NAME, "[ERROR] start -> Product Definition without a Plural Variable Name defined. Product Definition = " + JSON.stringify(outputDatasetNode.referenceParent.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
@@ -79,7 +77,7 @@
if (outputDatasetNode.referenceParent.parentNode.dataBuilding === undefined) {
logger.write(MODULE_NAME, "[WARN] start -> Product Definition " + outputDatasetNode.referenceParent.parentNode.name + " without a Data Building Procedure. Product Definition = " + JSON.stringify(outputDatasetNode.referenceParent.parentNode));
}
if (outputDatasetNode.referenceParent.code.codeName === undefined) {
if (outputDatasetNode.referenceParent.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] start -> Dataset witn no codeName defined. Product Dataset = " + JSON.stringify(outputDatasetNode.referenceParent));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
@@ -91,7 +89,7 @@
return
}
if (outputDatasetNode.referenceParent.parentNode.code.codeName === undefined) {
if (outputDatasetNode.referenceParent.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] start -> Product Definition witn no codeName defined. Product Definition = " + JSON.stringify(outputDatasetNode.referenceParent.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
@@ -103,7 +101,7 @@
return
}
if (outputDatasetNode.referenceParent.parentNode.parentNode.code.codeName === undefined) {
if (outputDatasetNode.referenceParent.parentNode.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] start -> Bot witn no codeName defined. Bot = " + JSON.stringify(outputDatasetNode.referenceParent.parentNode.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
@@ -115,7 +113,7 @@
return
}
if (outputDatasetNode.referenceParent.parentNode.parentNode.parentNode.code.codeName === undefined) {
if (outputDatasetNode.referenceParent.parentNode.parentNode.parentNode.config.codeName === undefined) {
logger.write(MODULE_NAME, "[ERROR] start -> Data Mine witn no codeName defined. Data Mine = " + JSON.stringify(outputDatasetNode.referenceParent.parentNode.parentNode.parentNode));
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return
@@ -132,19 +130,19 @@
*/
for (let i = 0; i < dataDependencies.length; i++) {
let dataFile
let dataFile
let jsonData // Datafile converted into Json objects
let inputData // Includes calculated properties
let singularVariableName // name of the variable for this product
let recordDefinition
let dataDependencyNode = dataDependencies[i]
let dataDependencyNode = dataDependencies[i]
dataFile = dataFiles.get(dataDependencyNode.id)
if (dataFile === undefined) { continue } // When a datafile is not found it might be because we are processing market or daily and at the dependency array there are both types mixed up.
recordDefinition = dataDependencyNode.referenceParent.parentNode.record
singularVariableName = dataDependencyNode.referenceParent.parentNode.code.singularVariableName
singularVariableName = dataDependencyNode.referenceParent.parentNode.config.singularVariableName
/* Transform the raw data into JSON objects */
jsonData = jsonifyDataFile(dataFile, recordDefinition)
/* Add the calculated properties */
@@ -153,7 +151,7 @@
} else {
inputData = jsonData
}
products[dataDependencyNode.referenceParent.parentNode.code.pluralVariableName] = inputData
products[dataDependencyNode.referenceParent.parentNode.config.pluralVariableName] = inputData
/* The main dependency is defined as the first dependency processed. */
if (mainDependency.records === undefined) {
@@ -177,8 +175,8 @@
let record = {}
for (let j = 0; j < recordDefinition.properties.length; j++) {
let property = recordDefinition.properties[j]
if (property.code.isCalculated !== true) {
record[property.code.codeName] = dataFile[i][j]
if (property.config.isCalculated !== true) {
record[property.config.codeName] = dataFile[i][j]
}
}
@@ -199,7 +197,7 @@
let system = { // These are the available system variables to be used in User Code and Formulas
timeFrame: timeFrame,
ONE_DAY_IN_MILISECONDS: ONE_DAY_IN_MILISECONDS
ONE_DAY_IN_MILISECONDS: global.ONE_DAY_IN_MILISECONDS
}
let variable = {} // This is the structure where the user will define its own variables that will be shared across different code blocks and formulas.
let results = []
@@ -238,13 +236,13 @@
/* For each calculated property we apply its formula*/
for (let j = 0; j < recordDefinition.properties.length; j++) {
let property = recordDefinition.properties[j]
if (property.code.isCalculated === true) {
if (property.config.isCalculated === true) {
if (property.formula !== undefined) {
if (property.formula.code !== undefined) {
try {
let newValue = eval(property.formula.code)
let currentRecord = product[variableName]
currentRecord[property.code.codeName] = newValue
currentRecord[property.config.codeName] = newValue
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] calculationsProcedure -> loop -> formula -> Error executing User Code. Error = " + err.stack)
logger.write(MODULE_NAME, "[ERROR] calculationsProcedure -> loop -> formula -> Error executing User Code. product = " + JSON.stringify(product))
@@ -277,7 +275,7 @@
interExecutionMemory,
processingDailyFiles,
currentDay
) {
) {
/*
This function has as an input the products object, with all the information
@@ -286,12 +284,12 @@
a new set of information.
*/
let lastInstantOfTheDay
let lastInstantOfTheDay
let yesterday = {}
let system = { // These are the available system variables to be used in User Code and Formulas
timeFrame: timeFrame,
ONE_DAY_IN_MILISECONDS: ONE_DAY_IN_MILISECONDS,
ONE_MIN_IN_MILISECONDS: ONE_MIN_IN_MILISECONDS
ONE_DAY_IN_MILISECONDS: global.ONE_DAY_IN_MILISECONDS,
ONE_MIN_IN_MILISECONDS: global.ONE_MIN_IN_MILISECONDS
}
let variable = {} // This is the structure where the user will define its own variables that will be shared across different code blocks and formulas.
let results = []
@@ -313,7 +311,7 @@
let dataDependencyMap = new Map()
for (let j = 0; j < dataDependecyArray.length; j++) {
let record = dataDependecyArray[j]
let key = record.begin.toString() + '-' + record.end.toString()
let key = record.begin.toString() + '-' + record.end.toString()
dataDependencyMap.set(key, record)
}
dataDependencies[dataDependencyName] = dataDependencyMap
@@ -321,7 +319,7 @@
}
function getElement(dependencyName, currentRecordPrimaryDataDependency) {
let key = currentRecordPrimaryDataDependency.begin.toString() + '-' + currentRecordPrimaryDataDependency.end.toString()
let key = currentRecordPrimaryDataDependency.begin.toString() + '-' + currentRecordPrimaryDataDependency.end.toString()
return dataDependencies[dependencyName].get(key)
}
@@ -340,7 +338,7 @@
if (processingDailyFiles) {
/* Initialization of Last Instance */
lastInstantOfTheDay = currentDay.valueOf() + ONE_DAY_IN_MILISECONDS - 1;
lastInstantOfTheDay = currentDay.valueOf() + global.ONE_DAY_IN_MILISECONDS - 1;
if (interExecutionMemory[productName] === undefined) {
/* The first time the intialization variables goes to the Inter Execution Memory. */
@@ -349,7 +347,7 @@
}
else {
/* We override the initialization, since the valid stuff is already at the Inter Execution Memory */
variable = JSON.parse(JSON.stringify(interExecutionMemory[productName].variable))
variable = JSON.parse(JSON.stringify(interExecutionMemory[productName].variable))
}
}
@@ -387,13 +385,13 @@
/* For each non-calculated property we apply its formula*/
for (let j = 0; j < recordDefinition.properties.length; j++) {
let property = recordDefinition.properties[j]
if (property.code.isCalculated !== true) {
if (property.config.isCalculated !== true) {
if (property.formula !== undefined) {
if (property.formula.code !== undefined) {
try {
let newValue = eval(property.formula.code)
let currentRecord = product[variableName]
currentRecord[property.code.codeName] = newValue
currentRecord[property.config.codeName] = newValue
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] dataBuildingProcedure -> loop -> formula -> Error executing User Code. Error = " + err.stack)
logger.write(MODULE_NAME, "[ERROR] dataBuildingProcedure -> loop -> formula -> Error executing User Code. product = " + JSON.stringify(product))
@@ -411,7 +409,7 @@
results.push(product[variableName]);
}
/* While we are positioned at Yesterday, we keey updating this data structure. */
/* While we are positioned at Yesterday, we keep updating this data structure. */
if (processingDailyFiles) {
if (positionedAtYesterday) {
yesterday.variable = JSON.parse(JSON.stringify(variable))
@@ -457,7 +455,7 @@
at the next day, in whole, even if it starts in the previous day.
*/
let lastInstantOdDay = currentDay.valueOf() + ONE_DAY_IN_MILISECONDS - 1;
let lastInstantOdDay = currentDay.valueOf() + global.ONE_DAY_IN_MILISECONDS - 1;
if (record.end < currentDay.valueOf() - 1) { continue; }
if (record.end === lastInstantOdDay) { continue; }
@@ -474,13 +472,13 @@
let propertySeparator = ""
for (let j = 0; j < recordDefinition.properties.length; j++) {
let property = recordDefinition.properties[j]
if (property.code.isCalculated !== true) {
if (property.config.isCalculated !== true) {
fileContent = fileContent + propertySeparator
if (property.code.isString === true) {
if (property.config.isString === true) {
fileContent = fileContent + '"'
}
fileContent = fileContent + record[property.code.codeName]
if (property.code.isString === true) {
fileContent = fileContent + record[property.config.codeName]
if (property.config.isString === true) {
fileContent = fileContent + '"'
}
if (propertySeparator === "") { propertySeparator = ","; }