fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ready to code the fetching of data from the API

Browse Source
This commit is contained in:
Luis Fernando Molina committed 2021-04-21 18:55:46 +02:00
1 parent 62bf2d0e5f
commit 593258668b
6 files changed
+97 -17
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Projects/Superalgos/Plugins/Data-Mines/Masters.json
+1 -1
View File
@@ -1 +1 @@
{"type":"Data Mine","name":"Masters","config":"{\n\"codeName\": \"Masters\"\n}","project":"Superalgos","sensorBots":[{"type":"Sensor Bot","name":"Exchange Raw Data","config":"{\n \"codeName\": \"Exchange-Raw-Data\"\n}","project":"Superalgos","processes":[{"type":"Process Definition","name":"Historic-OHLCVs","config":"{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical maket data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}","project":"Superalgos","processOutput":{"type":"Process Output","name":"New Process Output","project":"Superalgos","outputDatasets":[],"outputDatasetFolders":[],"id":"dfe2556e-a765-4e38-9d30-7f91090c0069","savedPayload":{"position":{"x":19495.67390477555,"y":18276.314530096774},"targetPosition":{"x":19456.516047109537,"y":18573.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","project":"Superalgos","statusDependencies":[{"type":"Status Dependency","name":"Exchange Raw Data Historic-OHLCVs","config":"{\n \"mainUtility\": \"Self Reference\"\n}","project":"Superalgos","id":"ea39c98b-18e2-44ce-9f7f-86a8a76a688a","savedPayload":{"position":{"x":19821.77290451477,"y":18097.735984334173},"targetPosition":{"x":19639.144475812154,"y":18335.741986421544},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":1,"distanceToParent":3,"arrangementStyle":0},"uiObject":{},"referenceParent":{"type":"Status Report","name":"New Status Report","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e"}}}],"dataDependencies":[],"dataMineDataDependencies":[],"id":"5c7fae5a-6da6-4864-a167-f2f21d578c72","savedPayload":{"position":{"x":19639.144475812154,"y":18335.741986421544},"targetPosition":{"x":19456.516047109537,"y":18573.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"statusReport":{"type":"Status Report","name":"New Status Report","project":"Superalgos","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e","savedPayload":{"position":{"x":19733.679906862923,"y":18458.94295879939},"targetPosition":{"x":19456.516047109537,"y":18573.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionStartedEvent":{"type":"Execution Started Event","name":"","project":"Superalgos","id":"957e7e3a-b167-40be-93b5-9ba4ceff56e5","savedPayload":{"position":{"x":19753.94950552168,"y":18612.90584617493},"targetPosition":{"x":19456.516047109537,"y":18573.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionFinishedEvent":{"type":"Execution Finished Event","name":"New Execution Finished Event","project":"Superalgos","id":"fe40a546-43ee-49df-908a-7e72f598f0cd","savedPayload":{"position":{"x":19694.52204919691,"y":18756.376417211533},"targetPosition":{"x":19456.516047109537,"y":18573.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc","savedPayload":{"position":{"x":19456.516047109537,"y":18573.747988508916},"targetPosition":{"x":19179.35218735615,"y":18688.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}}],"products":[{"type":"Product Definition","name":"Candles","config":"{\n\"codeName\": \"Candles\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-Min\"\n}","project":"Superalgos","id":"1c3696e4-c28e-4744-9100-706d6dce0c3c","savedPayload":{"position":{"x":19629.35218735615,"y":18688.55301821844},"targetPosition":{"x":19479.35218735615,"y":18688.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":1,"distanceToParent":2,"arrangementStyle":0},"uiObject":{}}}],"id":"2443883b-6959-42a3-ae3b-ef3102ffcdcc","savedPayload":{"position":{"x":19479.35218735615,"y":18688.55301821844},"targetPosition":{"x":19179.35218735615,"y":18688.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Product Definition","name":"Volumes","config":"{\n\"codeName\": \"Volumes\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-MiLine truncated
{"type":"Data Mine","name":"Masters","config":"{\n\"codeName\": \"Masters\"\n}","project":"Superalgos","sensorBots":[{"type":"Sensor Bot","name":"Exchange Raw Data","config":"{\n \"codeName\": \"Exchange-Raw-Data\"\n}","project":"Superalgos","processes":[{"type":"Process Definition","name":"Historic-OHLCVs","config":"{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical maket data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}","project":"Superalgos","processOutput":{"type":"Process Output","name":"New Process Output","project":"Superalgos","outputDatasets":[],"outputDatasetFolders":[],"id":"dfe2556e-a765-4e38-9d30-7f91090c0069","savedPayload":{"position":{"x":19705.67390477555,"y":18305.314530096774},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","project":"Superalgos","statusDependencies":[{"type":"Status Dependency","name":"Exchange Raw Data Historic-OHLCVs","config":"{\n \"mainUtility\": \"Self Reference\"\n}","project":"Superalgos","id":"ea39c98b-18e2-44ce-9f7f-86a8a76a688a","savedPayload":{"position":{"x":20031.77290451477,"y":18126.735984334173},"targetPosition":{"x":19849.144475812154,"y":18364.741986421544},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":1,"distanceToParent":3,"arrangementStyle":0},"uiObject":{},"referenceParent":{"type":"Status Report","name":"New Status Report","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e"}}}],"dataDependencies":[],"dataMineDataDependencies":[],"id":"5c7fae5a-6da6-4864-a167-f2f21d578c72","savedPayload":{"position":{"x":19849.144475812154,"y":18364.741986421544},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"statusReport":{"type":"Status Report","name":"New Status Report","project":"Superalgos","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e","savedPayload":{"position":{"x":19943.679906862923,"y":18487.94295879939},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionStartedEvent":{"type":"Execution Started Event","name":"","project":"Superalgos","id":"957e7e3a-b167-40be-93b5-9ba4ceff56e5","savedPayload":{"position":{"x":19963.94950552168,"y":18641.90584617493},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionFinishedEvent":{"type":"Execution Finished Event","name":"New Execution Finished Event","project":"Superalgos","id":"fe40a546-43ee-49df-908a-7e72f598f0cd","savedPayload":{"position":{"x":19904.52204919691,"y":18785.376417211533},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc","savedPayload":{"position":{"x":19666.516047109537,"y":18602.747988508916},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}}],"products":[{"type":"Product Definition","name":"Candles","config":"{\n\"codeName\": \"Candles\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-Min\"\n}","project":"Superalgos","id":"1c3696e4-c28e-4744-9100-706d6dce0c3c","savedPayload":{"position":{"x":19839.35218735615,"y":18717.55301821844},"targetPosition":{"x":19689.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":1,"distanceToParent":2,"arrangementStyle":0},"uiObject":{}}}],"id":"2443883b-6959-42a3-ae3b-ef3102ffcdcc","savedPayload":{"position":{"x":19689.35218735615,"y":18717.55301821844},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Product Definition","name":"Volumes","config":"{\n\"codeName\": \"Volumes\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-MiLine truncated
Projects/Superalgos/Schemas/App-Schema/api-response-field-reference.json
+28
View File
@@ -0,0 +1,28 @@
{
"type": "API Response Field Reference",
"menuItems": [
{
"action": "Delete UI Object",
"askConfirmation": true,
"confirmationLabel": "Confirm to Delete",
"label": "Delete",
"iconPathOn": "delete-entity",
"iconPathOff": "delete-entity",
"actionFunction": "payload.executeAction"
}
],
"title": [
"Use Reference Parent"
],
"isTitleAllwaysVisible": true,
"addLeftIcons": true,
"level": 3,
"attachingRules": {
"compatibleTypes": "->Record Property->"
},
"referencingRules": {
"compatibleTypes": "->API Response Field->"
},
"propertyNameAtParent": "apiResponseFieldReference",
"icon": "bitcoin"
}
Projects/Superalgos/Schemas/App-Schema/record-property.json
+13
View File
@@ -17,6 +17,14 @@
"relatedUiObject": "Record Formula",
"actionFunction": "payload.executeAction"
},
{
"action": "Add UI Object",
"disableIfPropertyIsDefined": true,
"propertyToCheckFor": "apiResponseFieldReference",
"label": "Add Field Reference",
"relatedUiObject": "API Response Field Reference",
"actionFunction": "payload.executeAction"
},
{
"action": "Delete UI Object",
"askConfirmation": true,
@@ -44,6 +52,11 @@
"name": "recordFormula",
"type": "node",
"childType": "Record Formula"
},
{
"name": "apiResponseFieldReference",
"type": "node",
"childType": "API Response Field Reference"
}
],
"propertyNameAtParent": "properties"
Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js
+52 -14
View File
@@ -34,11 +34,12 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
}
let fileContent
getContextVariables(saveMessages)
getContextVariables(fetchData)
function getContextVariables(callBack) {
try {
let reportKey = "Masters" + "-" + "Webhooks" + "-" + "Check-Webhook"
let processNode = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent
let reportKey = processNode.parentNode.parentNode.config.codeName + "-" + processNode.parentNode.config.codeName + "-" + processNode.config.codeName
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[INFO] start -> getContextVariables -> reportKey = " + reportKey)
@@ -52,7 +53,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
thisReport = statusDependencies.statusReports.get(reportKey)
if (thisReport.file.lasrRun !== undefined) {
if (thisReport.file.lastRun !== undefined) {
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + FOLDER_NAME + "/" + 'Single-File'
@@ -87,16 +88,53 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
}
}
function saveMessages() {
function fetchData() {
try {
let http = require('http');
let url = 'http://' + global.env.WEB_SERVER_URL +
':' + global.env.HTTP_INTERFACE_PORT +
'/Webhook/Fetch-Messages/' +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.config.codeName + "/" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "-" +
TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName
if (TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].apiMapReference === undefined) {
// TODO Error Handling
return
}
let apiMAP = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].apiMapReference.referenceParent
if (apiMap === undefined) {
// TODO Error Handling
return
}
let hostName
let portNumber = ""
let path = ""
let protocol = "https" // default value
if (apiMAP.config.hostName === undefined || apiMAP.config.hostName === "") {
// TODO Error Handling
return
}
if (apiMAP.config.portNumber !== undefined && apiMAP.config.portNumber !== "") {
portNumber = ":" + apiMAP.config.portNumber
}
if (apiMAP.config.protocol !== undefined && apiMAP.config.protocol !== "") {
protocol = apiMAP.config.protocol
}
if (apiMAP.config.path !== undefined && apiMAP.config.path !== "") {
path = "/" + apiMAP.config.path
}
if (apiVersions.length === 0) {
// TODO Error Handling
return
}
let http = require('http');
let url = protocol + '://' +
hostName +
portNumber +
path
http.get(url, onResponse);
function onResponse(response) {
@@ -134,7 +172,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
function onFileCreated(err) {
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> saveMessages -> onResponse -> onEnd -> onFileCreated -> Could not save file. -> filePath = " + filePath + "/" + fileName);
"[ERROR] start -> fetchData -> onResponse -> onEnd -> onFileCreated -> Could not save file. -> filePath = " + filePath + "/" + fileName);
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
} else {
writeStatusReport()
@@ -146,7 +184,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
} catch (err) {
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[ERROR] start -> saveMessages -> err = " + err.stack);
"[ERROR] start -> fetchData -> err = " + err.stack);
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
abort = true
}
@@ -155,7 +193,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
function writeStatusReport() {
try {
thisReport.file = {
lasrRun: (new Date()).toISOString()
lastRun: (new Date()).toISOString()
};
thisReport.save(onSaved);
Projects/Superalgos/TS/Bot-Modules/Sensor-Bot/Webhooks/CheckWebhook.js
+2 -2
View File
@@ -52,7 +52,7 @@ exports.newSuperalgosBotModulesCheckWebhook = function (processIndex) {
thisReport = statusDependencies.statusReports.get(reportKey)
if (thisReport.file.lasrRun !== undefined) {
if (thisReport.file.lastRun !== undefined) {
let fileName = 'Data.json'
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + FOLDER_NAME + "/" + 'Single-File'
@@ -155,7 +155,7 @@ exports.newSuperalgosBotModulesCheckWebhook = function (processIndex) {
function writeStatusReport() {
try {
thisReport.file = {
lasrRun: (new Date()).toISOString()
lastRun: (new Date()).toISOString()
};
thisReport.save(onSaved);
Projects/Superalgos/UI/Function-Libraries/TaskFunctions.js
+1
View File
@@ -135,6 +135,7 @@ function newSuperalgosFunctionLibraryTaskFunctions() {
'Data Dependency Folder->Data Dependency Folder->Data Dependency Folder->Data Dependency Folder->Data Dependency Folder->' +
'Data Dependency->Dataset Definition->Product Definition->' +
'Record Definition->Record Property->Record Formula->' +
'API Response Field Reference->API Response Field->' +
'Data Building Procedure->' +
'Procedure Initialization->Procedure Javascript Code->' +
'Procedure Loop->Procedure Javascript Code->' +