fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stuck at possible bug on Lignting Path Function

Browse Source
This commit is contained in:
Luis Fernando Molina committed 2021-04-24 21:30:25 +02:00
1 parent 5d1804853d
commit ece6b383e5
5 files changed
+114 -45
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Projects/Superalgos/Plugins/Data-Mines/Masters.json
+1 -1
View File
@@ -1 +1 @@
{"type":"Data Mine","name":"Masters","config":"{\n\"codeName\": \"Masters\"\n}","project":"Superalgos","sensorBots":[{"type":"Sensor Bot","name":"Exchange Raw Data","config":"{\n \"codeName\": \"Exchange-Raw-Data\"\n}","project":"Superalgos","processes":[{"type":"Process Definition","name":"Historic-OHLCVs","config":"{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical maket data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}","project":"Superalgos","processOutput":{"type":"Process Output","name":"New Process Output","project":"Superalgos","outputDatasets":[],"outputDatasetFolders":[],"id":"dfe2556e-a765-4e38-9d30-7f91090c0069","savedPayload":{"position":{"x":19705.67390477555,"y":18305.314530096774},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","project":"Superalgos","statusDependencies":[{"type":"Status Dependency","name":"Exchange Raw Data Historic-OHLCVs","config":"{\n \"mainUtility\": \"Self Reference\"\n}","project":"Superalgos","id":"ea39c98b-18e2-44ce-9f7f-86a8a76a688a","savedPayload":{"position":{"x":20031.77290451477,"y":18126.735984334173},"targetPosition":{"x":19849.144475812154,"y":18364.741986421544},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":1,"distanceToParent":3,"arrangementStyle":0},"uiObject":{},"referenceParent":{"type":"Status Report","name":"New Status Report","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e"}}}],"dataDependencies":[],"dataMineDataDependencies":[],"id":"5c7fae5a-6da6-4864-a167-f2f21d578c72","savedPayload":{"position":{"x":19849.144475812154,"y":18364.741986421544},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"statusReport":{"type":"Status Report","name":"New Status Report","project":"Superalgos","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e","savedPayload":{"position":{"x":19943.679906862923,"y":18487.94295879939},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionStartedEvent":{"type":"Execution Started Event","name":"","project":"Superalgos","id":"957e7e3a-b167-40be-93b5-9ba4ceff56e5","savedPayload":{"position":{"x":19963.94950552168,"y":18641.90584617493},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionFinishedEvent":{"type":"Execution Finished Event","name":"New Execution Finished Event","project":"Superalgos","id":"fe40a546-43ee-49df-908a-7e72f598f0cd","savedPayload":{"position":{"x":19904.52204919691,"y":18785.376417211533},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc","savedPayload":{"position":{"x":19666.516047109537,"y":18602.747988508916},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}}],"products":[{"type":"Product Definition","name":"Candles","config":"{\n\"codeName\": \"Candles\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-Min\"\n}","project":"Superalgos","id":"1c3696e4-c28e-4744-9100-706d6dce0c3c","savedPayload":{"position":{"x":19839.35218735615,"y":18717.55301821844},"targetPosition":{"x":19689.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":1,"distanceToParent":2,"arrangementStyle":0},"uiObject":{}}}],"id":"2443883b-6959-42a3-ae3b-ef3102ffcdcc","savedPayload":{"position":{"x":19689.35218735615,"y":18717.55301821844},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Product Definition","name":"Volumes","config":"{\n\"codeName\": \"Volumes\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-MiLine truncated
{"type":"Data Mine","name":"Masters","config":"{\n\"codeName\": \"Masters\"\n}","project":"Superalgos","sensorBots":[{"type":"Sensor Bot","name":"Exchange Raw Data","config":"{\n \"codeName\": \"Exchange-Raw-Data\"\n}","project":"Superalgos","processes":[{"type":"Process Definition","name":"Historic-OHLCVs","config":"{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical maket data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}","project":"Superalgos","processOutput":{"type":"Process Output","name":"New Process Output","project":"Superalgos","outputDatasets":[],"outputDatasetFolders":[],"id":"dfe2556e-a765-4e38-9d30-7f91090c0069","savedPayload":{"position":{"x":19705.67390477555,"y":18305.314530096774},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","project":"Superalgos","statusDependencies":[{"type":"Status Dependency","name":"Exchange Raw Data Historic-OHLCVs","config":"{\n \"mainUtility\": \"Self Reference\"\n}","project":"Superalgos","id":"ea39c98b-18e2-44ce-9f7f-86a8a76a688a","savedPayload":{"position":{"x":20031.77290451477,"y":18126.735984334173},"targetPosition":{"x":19849.144475812154,"y":18364.741986421544},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":1,"distanceToParent":3,"arrangementStyle":0},"uiObject":{},"referenceParent":{"type":"Status Report","name":"New Status Report","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e"}}}],"dataDependencies":[],"dataMineDataDependencies":[],"id":"5c7fae5a-6da6-4864-a167-f2f21d578c72","savedPayload":{"position":{"x":19849.144475812154,"y":18364.741986421544},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"statusReport":{"type":"Status Report","name":"New Status Report","project":"Superalgos","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e","savedPayload":{"position":{"x":19943.679906862923,"y":18487.94295879939},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionStartedEvent":{"type":"Execution Started Event","name":"","project":"Superalgos","id":"957e7e3a-b167-40be-93b5-9ba4ceff56e5","savedPayload":{"position":{"x":19963.94950552168,"y":18641.90584617493},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionFinishedEvent":{"type":"Execution Finished Event","name":"New Execution Finished Event","project":"Superalgos","id":"fe40a546-43ee-49df-908a-7e72f598f0cd","savedPayload":{"position":{"x":19904.52204919691,"y":18785.376417211533},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc","savedPayload":{"position":{"x":19666.516047109537,"y":18602.747988508916},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}}],"products":[{"type":"Product Definition","name":"Candles","config":"{\n\"codeName\": \"Candles\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-Min\"\n}","project":"Superalgos","id":"1c3696e4-c28e-4744-9100-706d6dce0c3c","savedPayload":{"position":{"x":19839.35218735615,"y":18717.55301821844},"targetPosition":{"x":19689.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":1,"distanceToParent":2,"arrangementStyle":0},"uiObject":{}}}],"id":"2443883b-6959-42a3-ae3b-ef3102ffcdcc","savedPayload":{"position":{"x":19689.35218735615,"y":18717.55301821844},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Product Definition","name":"Volumes","config":"{\n\"codeName\": \"Volumes\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-MiLine truncated
Projects/Superalgos/Schemas/App-Schema/api-query-parameters.json
+1 -1
View File
@@ -20,7 +20,7 @@
"addLeftIcons": true,
"level": 3,
"attachingRules": {
"compatibleTypes": "->API Endpoint->"
"compatibleTypes": "->API Endpoint->Product Definition->"
},
"propertyNameAtParent": "apiQueryParameters",
"childrenNodesProperties": [
Projects/Superalgos/Schemas/App-Schema/product-definition.json
+6
View File
@@ -76,6 +76,12 @@
"name": "dataBuilding",
"type": "node",
"childType": "Data Building Procedure"
},
{
"name": "apiQueryParameters",
"type": "node",
"childType": "API Query Parameters",
"autoAdd": false
}
],
"propertyNameAtParent": "products"
Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js
+103 -42
View File
@@ -73,7 +73,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
callBack()
}
}
} else {
} else {
/*
If there is no status report, we assume there is no previous file or that if there is we will override it.
*/
@@ -96,13 +96,15 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
function fetchData() {
try {
let processNode = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent
if (TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].apiMapReference === undefined) {
// TODO Error Handling
return
}
let apiMAP = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].apiMapReference.referenceParent
if (apiMap === undefined) {
if (apiMAP === undefined) {
// TODO Error Handling
return
}
@@ -129,64 +131,123 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
path = "/" + apiMAP.config.path
}
if (apiVersions.length === 0) {
if (apiMAP.apiVersions.length === 0) {
// TODO Error Handling
return
}
if (processNode.processOutput === undefined) {
// TODO Error Handling
return
}
let http = require('http');
let url = protocol + '://' +
hostName +
portNumber +
path
http.get(url, onResponse);
function onResponse(response) {
const chunks = []
response.on('data', onDataArrived)
response.on('end', onEnd)
function onDataArrived(chunk) {
chunks.push(chunk)
for (let i = 0; i < processNode.processOutput.outputDatasets.length; i++) {
let outputDataset = processNode.processOutput.outputDatasets[i]
if (outputDataset.referenceParent === undefined) {
continue
}
if (outputDataset.referenceParent.parentNode === undefined) {
continue
}
let productDefinition = outputDataset.referenceParent.parentNode
let endpoint
/*
To get the endpoint node, we will go through the references at the Record Definition.
We will check that all references are pointing to fields belonging to the same Endpoint node.
*/
for (let j = 0; j < productDefinition.record.properties.length; j++) {
let recordProperty = productDefinition.record.properties[j]
if (recordProperty.apiResponseFieldReference !== undefined) {
if (recordProperty.apiResponseFieldReference.referenceParent !== undefined) {
let apiResponseField = recordProperty.apiResponseFieldReference.referenceParent
let endpointFound = TS.projects.superalgos.utilities.nodeFunctions.findNodeInNodeMesh(apiResponseField, 'API Endpoint')
function onEnd() {
let dataReceived = Buffer.concat(chunks).toString('utf8')
if (endpointFound === undefined) {
// TODO Error Handling
return
}
if (fileContent !== undefined) {
// we are going to append the curernt dataReceived to the existing file.
let fileContentArray = JSON.parse(fileContent)
let dataReceivedArray = JSON.parse(dataReceived)
if (endpoint === undefined) {
/* The first enpoint found will be our endpoint */
endpoint = endpointFound
}
for (let i = 0; i < dataReceivedArray.length; i++) {
let message = dataReceivedArray[i]
fileContentArray.push(message)
if (endpointFound.id !== endpoint.id) {
// TODO Error Handling - we can not reference fields from different enpoints.
return
}
}
}
}
/*
Let's see if we need to place the API call with Parameters...
*/
let queryString = ""
let separator = ""
if (productDefinition.apiQueryParameters !== undefined) {
queryString = '?'
for (j = 0; j < productDefinition.apiQueryParameters.length; j++) {
let apiQueryParameter = productDefinition.apiQueryParameters[j]
queryString = queryString + separator + apiQueryParameter.config.codeName + '=' + apiQueryParameter.config.value
separator = "&"
}
}
/*
Now that we have the Endpoint, we can call the API.
*/
let http = require('http');
let url = protocol + '://' +
hostName +
portNumber +
path +
'/' + endpoint.config.codeName +
queryString
fileContent = JSON.stringify(fileContentArray)
} else {
// we are going to save the current dataReceived.
fileContent = dataReceived
http.get(url, onResponse);
function onResponse(response) {
const chunks = []
response.on('data', onDataArrived)
response.on('end', onEnd)
function onDataArrived(chunk) {
chunks.push(chunk)
}
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + FOLDER_NAME + "/" + 'Single-File'
fileStorage.createTextFile(filePath + '/' + fileName, fileContent + '\n', onFileCreated);
function onEnd() {
let dataReceived = Buffer.concat(chunks).toString('utf8')
function onFileCreated(err) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> fetchData -> onResponse -> onEnd -> onFileCreated -> Could not save file. -> filePath = " + filePath + "/" + fileName);
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
if (fileContent !== undefined) {
// we are going to append the curernt dataReceived to the existing file.
let fileContentArray = JSON.parse(fileContent)
let dataReceivedArray = JSON.parse(dataReceived)
for (let i = 0; i < dataReceivedArray.length; i++) {
let message = dataReceivedArray[i]
fileContentArray.push(message)
}
fileContent = JSON.stringify(fileContentArray)
} else {
writeStatusReport()
// we are going to save the current dataReceived.
fileContent = dataReceived
}
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + FOLDER_NAME + "/" + 'Single-File'
fileStorage.createTextFile(filePath + '/' + fileName, fileContent + '\n', onFileCreated);
function onFileCreated(err) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> fetchData -> onResponse -> onEnd -> onFileCreated -> Could not save file. -> filePath = " + filePath + "/" + fileName);
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
} else {
writeStatusReport()
}
}
}
}
}
} catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
Projects/Superalgos/UI/Function-Libraries/TaskFunctions.js
+3 -1
View File
@@ -135,7 +135,9 @@ function newSuperalgosFunctionLibraryTaskFunctions() {
'Data Dependency Folder->Data Dependency Folder->Data Dependency Folder->Data Dependency Folder->Data Dependency Folder->' +
'Data Dependency->Dataset Definition->Product Definition->' +
'Record Definition->Record Property->Record Formula->' +
'API Response Field Reference->API Response Field->' +
'API Response Field Reference->' +
'API Response Field->API Response Field->API Response Field->API Response Field->API Response Field->API Response Field->API Response Field->API Response Field->API Response Field->' +
'API Response Schema->API Query Response->API Query Responses->API Endpoint->' +
'Data Building Procedure->' +
'Procedure Initialization->Procedure Javascript Code->' +
'Procedure Loop->Procedure Javascript Code->' +