Merge pull request #856 from Luis-Fernando-Molina/develop
Contribution: API Fetcher bot, basic implementation
This commit is contained in:
5 files changed
+521
-135
No files matched your search
@@ -1 +1 @@
|
||||
{"type":"Data Mine","name":"Masters","config":"{\n\"codeName\": \"Masters\"\n}","project":"Superalgos","sensorBots":[{"type":"Sensor Bot","name":"Exchange Raw Data","config":"{\n \"codeName\": \"Exchange-Raw-Data\"\n}","project":"Superalgos","processes":[{"type":"Process Definition","name":"Historic-OHLCVs","config":"{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical maket data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}","project":"Superalgos","processOutput":{"type":"Process Output","name":"New Process Output","project":"Superalgos","outputDatasets":[],"outputDatasetFolders":[],"id":"dfe2556e-a765-4e38-9d30-7f91090c0069","savedPayload":{"position":{"x":19705.67390477555,"y":18305.314530096774},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","project":"Superalgos","statusDependencies":[{"type":"Status Dependency","name":"Exchange Raw Data Historic-OHLCVs","config":"{\n \"mainUtility\": \"Self Reference\"\n}","project":"Superalgos","id":"ea39c98b-18e2-44ce-9f7f-86a8a76a688a","savedPayload":{"position":{"x":20031.77290451477,"y":18126.735984334173},"targetPosition":{"x":19849.144475812154,"y":18364.741986421544},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":1,"distanceToParent":3,"arrangementStyle":0},"uiObject":{},"referenceParent":{"type":"Status Report","name":"New Status Report","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e"}}}],"dataDependencies":[],"dataMineDataDependencies":[],"id":"5c7fae5a-6da6-4864-a167-f2f21d578c72","savedPayload":{"position":{"x":19849.144475812154,"y":18364.741986421544},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"statusReport":{"type":"Status Report","name":"New Status Report","project":"Superalgos","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e","savedPayload":{"position":{"x":19943.679906862923,"y":18487.94295879939},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionStartedEvent":{"type":"Execution Started Event","name":"","project":"Superalgos","id":"957e7e3a-b167-40be-93b5-9ba4ceff56e5","savedPayload":{"position":{"x":19963.94950552168,"y":18641.90584617493},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionFinishedEvent":{"type":"Execution Finished Event","name":"New Execution Finished Event","project":"Superalgos","id":"fe40a546-43ee-49df-908a-7e72f598f0cd","savedPayload":{"position":{"x":19904.52204919691,"y":18785.376417211533},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc","savedPayload":{"position":{"x":19666.516047109537,"y":18602.747988508916},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}}],"products":[{"type":"Product Definition","name":"Candles","config":"{\n\"codeName\": \"Candles\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-Min\"\n}","project":"Superalgos","id":"1c3696e4-c28e-4744-9100-706d6dce0c3c","savedPayload":{"position":{"x":19839.35218735615,"y":18717.55301821844},"targetPosition":{"x":19689.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":1,"distanceToParent":2,"arrangementStyle":0},"uiObject":{}}}],"id":"2443883b-6959-42a3-ae3b-ef3102ffcdcc","savedPayload":{"position":{"x":19689.35218735615,"y":18717.55301821844},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Product Definition","name":"Volumes","config":"{\n\"codeName\": \"Volumes\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-MiLine truncated
|
||||
{"type":"Data Mine","name":"Masters","config":"{\n\"codeName\": \"Masters\"\n}","project":"Superalgos","sensorBots":[{"type":"Sensor Bot","name":"Exchange Raw Data","config":"{\n \"codeName\": \"Exchange-Raw-Data\"\n}","project":"Superalgos","processes":[{"type":"Process Definition","name":"Historic-OHLCVs","config":"{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical maket data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}","project":"Superalgos","processOutput":{"type":"Process Output","name":"New Process Output","project":"Superalgos","outputDatasets":[],"outputDatasetFolders":[],"id":"dfe2556e-a765-4e38-9d30-7f91090c0069","savedPayload":{"position":{"x":19705.67390477555,"y":18305.314530096774},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","project":"Superalgos","statusDependencies":[{"type":"Status Dependency","name":"Exchange Raw Data Historic-OHLCVs","config":"{\n \"mainUtility\": \"Self Reference\"\n}","project":"Superalgos","id":"ea39c98b-18e2-44ce-9f7f-86a8a76a688a","savedPayload":{"position":{"x":20031.77290451477,"y":18126.735984334173},"targetPosition":{"x":19849.144475812154,"y":18364.741986421544},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":1,"distanceToParent":3,"arrangementStyle":0},"uiObject":{},"referenceParent":{"type":"Status Report","name":"New Status Report","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e"}}}],"dataDependencies":[],"dataMineDataDependencies":[],"id":"5c7fae5a-6da6-4864-a167-f2f21d578c72","savedPayload":{"position":{"x":19849.144475812154,"y":18364.741986421544},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"statusReport":{"type":"Status Report","name":"New Status Report","project":"Superalgos","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e","savedPayload":{"position":{"x":19943.679906862923,"y":18487.94295879939},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionStartedEvent":{"type":"Execution Started Event","name":"","project":"Superalgos","id":"957e7e3a-b167-40be-93b5-9ba4ceff56e5","savedPayload":{"position":{"x":19963.94950552168,"y":18641.90584617493},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionFinishedEvent":{"type":"Execution Finished Event","name":"New Execution Finished Event","project":"Superalgos","id":"fe40a546-43ee-49df-908a-7e72f598f0cd","savedPayload":{"position":{"x":19904.52204919691,"y":18785.376417211533},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc","savedPayload":{"position":{"x":19666.516047109537,"y":18602.747988508916},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}}],"products":[{"type":"Product Definition","name":"Candles","config":"{\n\"codeName\": \"Candles\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-Min\"\n}","project":"Superalgos","id":"1c3696e4-c28e-4744-9100-706d6dce0c3c","savedPayload":{"position":{"x":19839.35218735615,"y":18717.55301821844},"targetPosition":{"x":19689.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":1,"distanceToParent":2,"arrangementStyle":0},"uiObject":{}}}],"id":"2443883b-6959-42a3-ae3b-ef3102ffcdcc","savedPayload":{"position":{"x":19689.35218735615,"y":18717.55301821844},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Product Definition","name":"Volumes","config":"{\n\"codeName\": \"Volumes\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-MiLine truncated
|
||||
@@ -0,0 +1 @@
|
||||
{"type":"Workspace","name":"API Fetcher Demo","rootNodes":[{"type":"Network","name":"My Network","project":"Superalgos","networkNodes":[{"type":"Network Node","name":"My Computer","config":"{ \n\"host\": \"localhost\", \n\"webPort\": \"34248\", \n\"webSocketsPort\": \"18041\"\n}","project":"Superalgos","dataTasks":{"type":"Data Tasks","name":"New Data Tasks","project":"Superalgos","projectDataTasks":[{"type":"Project Data Tasks","name":"Superalgos","project":"Superalgos","exchangeDataTasks":[{"type":"Exchange Data Tasks","name":"Binance","project":"Superalgos","marketDataTasks":[{"type":"Market Data Tasks","name":"Binance BTC/USDT","project":"Superalgos","dataMineTasks":[{"type":"Data Mine Tasks","name":"Masters","project":"Superalgos","taskManagers":[{"type":"Task Manager","name":"Masters","project":"Superalgos","tasks":[{"type":"Task","name":"Exchange Raw Data","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Exchange Raw Data","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Exchange Raw Data Historic-OHLCVs","project":"Superalgos","id":"ecc383c5-3e31-4f80-b2c5-1c966b9d77f4","savedPayload":{"position":{"x":23470.023749611923,"y":29701.01019239557},"targetPosition":{"x":23759.801497498644,"y":29623.364478864813},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Historic-OHLCVs","id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc"}}}],"id":"624f61cb-a76a-4059-b46d-c81975d88a9b","savedPayload":{"position":{"x":23759.801497498644,"y":29623.364478864813},"targetPosition":{"x":24049.579245385365,"y":29545.718765334055},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"94b155fe-1012-4c24-9830-2e4adfe4a614","savedPayload":{"position":{"x":24049.579245385365,"y":29545.718765334055},"targetPosition":{"x":24339.356993272086,"y":29468.073051803298},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Webhooks","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Webhooks","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Webhooks Check-Webhook","project":"Superalgos","id":"cea8f612-7533-4bc8-9df9-2e14f2c6eabd","savedPayload":{"position":{"x":23439.356993272086,"y":29468.073051803298},"targetPosition":{"x":23739.356993272086,"y":29468.073051803298},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Check-Webhook","id":"69f37820-ab16-4e19-bfd8-386dd6e249a3"}}}],"id":"124d68e8-969b-4426-b736-d8a3f9fd730f","savedPayload":{"position":{"x":23739.356993272086,"y":29468.073051803298},"targetPosition":{"x":24039.356993272086,"y":29468.073051803298},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"77f3b15e-546d-4259-82bd-96a56781dd69","savedPayload":{"position":{"x":24039.356993272086,"y":29468.073051803298},"targetPosition":{"x":24339.356993272086,"y":29468.073051803298},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Messari","project":"Superalgos","bot":{"type":"API Data Fetcher Bot Instance","name":"Messari","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"API Data Fetcher Process Instance","name":"Messari Fetching Process","project":"Superalgos","apiMapReference":{"type":"API Map Reference","name":"Messari","project":"Superalgos","id":"99521f12-7042-4a19-a584-52726e07f3d9","savedPayload":{"position":{"x":23180.2460017252,"y":29157.49019768027},"targetPosition":{"x":23470.023749611923,"y":29235.135911211026},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{},"referenceParent":{"type":"API Map","name":"Messari","id":"0f9e2b07-1084-4489-927c-bab05bbd08d2"}}},"id":"34fa36bd-11aa-4a83-8c7f-05b23abc0f5d","savedPayload":{"position":{"x":23470.023749611923,"y":29235.135911211026},"targetPosition":{"x":23759.801497498644,"y":29312.781624741783},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Fetching Process"Line truncated
|
||||
@@ -1,8 +1,17 @@
|
||||
{
|
||||
"type": "API Response Schema",
|
||||
"menuItems": [
|
||||
{
|
||||
"action": "Configure",
|
||||
"actionFunction": "uiObject.configEditor.activate",
|
||||
"label": "Configure",
|
||||
"iconPathOn": "configuration",
|
||||
"iconPathOff": "configuration"
|
||||
},
|
||||
{
|
||||
"action": "Add UI Object",
|
||||
"disableIfPropertyIsDefined": true,
|
||||
"propertyToCheckFor": "apiResponseFields",
|
||||
"label": "Add API Response Field",
|
||||
"relatedUiObject": "API Response Field",
|
||||
"actionFunction": "payload.executeAction"
|
||||
@@ -17,6 +26,12 @@
|
||||
"actionFunction": "payload.executeAction"
|
||||
}
|
||||
],
|
||||
"editors": {
|
||||
"config": true
|
||||
},
|
||||
"initialValues": {
|
||||
"config": "{\"nodePath\": \"apiDataObject.data\"}"
|
||||
},
|
||||
"addLeftIcons": true,
|
||||
"level": 4,
|
||||
"attachingRules": {
|
||||
@@ -26,7 +41,7 @@
|
||||
"childrenNodesProperties": [
|
||||
{
|
||||
"name": "apiResponseFields",
|
||||
"type": "array",
|
||||
"type": "node",
|
||||
"childType": "API Response Field",
|
||||
"autoAdd": false
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
|
||||
const MODULE_NAME = "Fetching Process";
|
||||
const FOLDER_NAME = "API-Data";
|
||||
|
||||
thisObject = {
|
||||
initialize: initialize,
|
||||
@@ -33,12 +32,16 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
return
|
||||
}
|
||||
|
||||
let fileContent
|
||||
getContextVariables(fetchData)
|
||||
let thisReport // This holds the status report.
|
||||
let processNode // This hold the node Process Definition
|
||||
let lastPage = {} // This holds the last page fetched for each endpoint
|
||||
|
||||
function getContextVariables(callBack) {
|
||||
getContextVariables()
|
||||
startProcess()
|
||||
|
||||
function getContextVariables() {
|
||||
try {
|
||||
let processNode = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent
|
||||
processNode = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent
|
||||
let reportKey = processNode.parentNode.parentNode.config.codeName + "-" + processNode.parentNode.config.codeName + "-" + processNode.config.codeName
|
||||
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
@@ -53,32 +56,6 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
|
||||
thisReport = statusDependencies.statusReports.get(reportKey)
|
||||
|
||||
if (thisReport.file.lastRun !== undefined) {
|
||||
/*
|
||||
If this process already ran before, then we are going to load the data stored so as to append
|
||||
information to it later.
|
||||
*/
|
||||
let fileName = 'Data.json'
|
||||
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + FOLDER_NAME + "/" + 'Single-File'
|
||||
fileStorage.getTextFile(filePath + '/' + fileName, onFileReceived);
|
||||
|
||||
function onFileReceived(err, text) {
|
||||
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[ERROR] start -> getContextVariables -> onFileReceived -> Could read file. -> filePath = " + filePath + "/" + fileName);
|
||||
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
|
||||
} else {
|
||||
|
||||
fileContent = text
|
||||
callBack()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
/*
|
||||
If there is no status report, we assume there is no previous file or that if there is we will override it.
|
||||
*/
|
||||
callBack()
|
||||
}
|
||||
} catch (err) {
|
||||
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
@@ -90,21 +67,20 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
"[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys));
|
||||
}
|
||||
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
|
||||
abort = true
|
||||
}
|
||||
}
|
||||
|
||||
function fetchData() {
|
||||
async function startProcess() {
|
||||
try {
|
||||
let processNode = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent
|
||||
processNode = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].referenceParent
|
||||
|
||||
if (TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].apiMapReference === undefined) {
|
||||
// TODO Error Handling
|
||||
return
|
||||
}
|
||||
|
||||
let apiMAP = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].apiMapReference.referenceParent
|
||||
if (apiMAP === undefined) {
|
||||
let apiMap = TS.projects.superalgos.globals.taskConstants.TASK_NODE.bot.processes[processIndex].apiMapReference.referenceParent
|
||||
if (apiMap === undefined) {
|
||||
// TODO Error Handling
|
||||
return
|
||||
}
|
||||
@@ -114,26 +90,26 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
let path = ""
|
||||
let protocol = "https" // default value
|
||||
|
||||
if (apiMAP.config.hostName === undefined || apiMAP.config.hostName === "") {
|
||||
if (apiMap.config.hostName === undefined || apiMap.config.hostName === "") {
|
||||
// TODO Error Handling
|
||||
return
|
||||
} else {
|
||||
hostName = apiMAP.config.hostName
|
||||
}
|
||||
|
||||
if (apiMAP.config.portNumber !== undefined && apiMAP.config.portNumber !== "") {
|
||||
portNumber = ":" + apiMAP.config.portNumber
|
||||
hostName = apiMap.config.hostName
|
||||
}
|
||||
|
||||
if (apiMAP.config.protocol !== undefined && apiMAP.config.protocol !== "") {
|
||||
protocol = apiMAP.config.protocol
|
||||
if (apiMap.config.portNumber !== undefined && apiMap.config.portNumber !== "") {
|
||||
portNumber = ":" + apiMap.config.portNumber
|
||||
}
|
||||
|
||||
if (apiMAP.config.path !== undefined && apiMAP.config.path !== "") {
|
||||
path = "/" + apiMAP.config.path
|
||||
if (apiMap.config.protocol !== undefined && apiMap.config.protocol !== "") {
|
||||
protocol = apiMap.config.protocol
|
||||
}
|
||||
|
||||
if (apiMAP.apiVersions.length === 0) {
|
||||
if (apiMap.config.path !== undefined && apiMap.config.path !== "") {
|
||||
path = "/" + apiMap.config.path
|
||||
}
|
||||
|
||||
if (apiMap.apiVersions.length === 0) {
|
||||
// TODO Error Handling
|
||||
return
|
||||
}
|
||||
@@ -141,7 +117,11 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
// TODO Error Handling
|
||||
return
|
||||
}
|
||||
|
||||
/*
|
||||
Process Output Datasets determine which Datasets are going to be built by
|
||||
this process. We will loop though all of them and build a file for each.
|
||||
If the file already exists, we will add new information to it.
|
||||
*/
|
||||
for (let i = 0; i < processNode.processOutput.outputDatasets.length; i++) {
|
||||
let outputDataset = processNode.processOutput.outputDatasets[i]
|
||||
if (outputDataset.referenceParent === undefined) {
|
||||
@@ -150,119 +130,511 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
if (outputDataset.referenceParent.parentNode === undefined) {
|
||||
continue
|
||||
}
|
||||
let productDefinition = outputDataset.referenceParent.parentNode
|
||||
let endpoint
|
||||
/*
|
||||
To get the endpoint node, we will go through the references at the Record Definition.
|
||||
We will check that all references are pointing to fields belonging to the same Endpoint node.
|
||||
*/
|
||||
for (let j = 0; j < productDefinition.record.properties.length; j++) {
|
||||
let recordProperty = productDefinition.record.properties[j]
|
||||
if (recordProperty.apiResponseFieldReference !== undefined) {
|
||||
if (recordProperty.apiResponseFieldReference.referenceParent !== undefined) {
|
||||
let apiResponseField = recordProperty.apiResponseFieldReference.referenceParent
|
||||
let endpointFound = TS.projects.superalgos.utilities.nodeFunctions.findNodeInNodeMesh(apiResponseField, 'API Endpoint')
|
||||
let productDefinition = outputDataset.referenceParent.parentNode // This is the Product Definition Node related to the current Output Dataset.
|
||||
let endpointNode // This holds the Endpoint Node at the API Map
|
||||
let existingFileContent // This holds the data of the existing dataset file.
|
||||
let queryString = "" // This holds the query string to be sent at the API call.
|
||||
let apiDataReceived // This hold the data received from the latest API call.
|
||||
let apiData // This holds the specific object containing the data to be stored inside the structure received by the API call.
|
||||
let dataReceivedArray = [] // This hold the cumulative data received from all calls to the API (multiple pages of data).
|
||||
let pageNumberParameter // This holds the node that represents a Page Number parameter.
|
||||
let pageQueryString // This holds the node part of query sting that deals with page numbers
|
||||
let recordPropertiesNodePathMap = new Map() // This holds the calculated nodePath for each record property, in order to help find the property value on the data received.
|
||||
|
||||
if (endpointFound === undefined) {
|
||||
getApiEndpointAndSchema()
|
||||
calculateRecordPropertiesNodePathMap()
|
||||
getEndpointQueryParameters()
|
||||
await fetchAllPages()
|
||||
await readDatasetFile()
|
||||
appendToExistingDataset()
|
||||
await saveDatasetFile()
|
||||
|
||||
function getApiEndpointAndSchema() {
|
||||
/*
|
||||
We will assume that each Dataset is built fetching data from a single endpointNode at the target API.
|
||||
To get the endpointNode node, we will go through the references at the Record Definition.
|
||||
We will check that all references are pointing to fields belonging to the same endpointNode node.
|
||||
If they are not, we will consider the user did not define well the relationaships between nodes
|
||||
and abort the process.
|
||||
*/
|
||||
for (let j = 0; j < productDefinition.record.properties.length; j++) {
|
||||
let recordProperty = productDefinition.record.properties[j]
|
||||
if (recordProperty.apiResponseFieldReference !== undefined) {
|
||||
if (recordProperty.apiResponseFieldReference.referenceParent !== undefined) {
|
||||
let apiResponseField = recordProperty.apiResponseFieldReference.referenceParent
|
||||
let endpointNodeFound = TS.projects.superalgos.utilities.nodeFunctions.findNodeInNodeMesh(apiResponseField, 'API Endpoint')
|
||||
/*
|
||||
Every Record Property with an apiResponseFieldReference must be on the path
|
||||
to a certain enpoint.
|
||||
*/
|
||||
if (endpointNodeFound === undefined) {
|
||||
// TODO Error Handling
|
||||
return
|
||||
}
|
||||
|
||||
if (endpointNode === undefined) {
|
||||
/* The first endpointNode found will be our endpointNode */
|
||||
endpointNode = endpointNodeFound
|
||||
}
|
||||
|
||||
/*
|
||||
Here we check all enpoints are actually the same endpoiht.
|
||||
*/
|
||||
if (endpointNodeFound.id !== endpointNode.id) {
|
||||
// TODO Error Handling - we can not reference fields from different enpoints.
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function calculateRecordPropertiesNodePathMap() {
|
||||
/*
|
||||
Each record property references an API Response Field, which can be deep into the
|
||||
data structure received from the API call. What we need to do, is to find for each
|
||||
record proeprty the nodePath that will help us later retrieve the value of that
|
||||
particular proeprty.
|
||||
|
||||
To do so, we will have to recursivelly climb the API Map until reaching the API Response
|
||||
Schema node, which is the one from where we will get the root object of the response data
|
||||
structure.
|
||||
*/
|
||||
for (let j = 0; j < productDefinition.record.properties.length; j++) {
|
||||
let recordProperty = productDefinition.record.properties[j]
|
||||
if (recordProperty.apiResponseFieldReference !== undefined) {
|
||||
if (recordProperty.apiResponseFieldReference.referenceParent !== undefined) {
|
||||
let apiResponseField = recordProperty.apiResponseFieldReference.referenceParent
|
||||
let nodePath = getPath(apiResponseField)
|
||||
recordPropertiesNodePathMap.set(recordProperty.config.codeName, nodePath)
|
||||
|
||||
function getPath(node, path) {
|
||||
|
||||
if (node.parentNode === undefined || node.parentNode.type !== "API Response Field" || node.parentNode.config.fieldType === 'array') {
|
||||
if (path === undefined) {
|
||||
path = 'recordReceived' + '.' + node.config.codeName
|
||||
} else {
|
||||
path = 'recordReceived' + '.' + node.config.codeName + '.' + path
|
||||
}
|
||||
return path
|
||||
} else {
|
||||
if (path === undefined) {
|
||||
path = node.config.codeName
|
||||
} else {
|
||||
path = node.config.codeName + '.' + path
|
||||
}
|
||||
return getPath(node.parentNode, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getEndpointQueryParameters() {
|
||||
/*
|
||||
Parameters to tue query of the API are all optional. Here we will see if
|
||||
we need to place the API call with Parameters...
|
||||
*/
|
||||
let separator = ""
|
||||
if (productDefinition.apiQueryParameters !== undefined) {
|
||||
queryString = '?'
|
||||
for (j = 0; j < productDefinition.apiQueryParameters.apiQueryParameters.length; j++) {
|
||||
let apiQueryParameter = productDefinition.apiQueryParameters.apiQueryParameters[j]
|
||||
|
||||
/*
|
||||
There is a special parameter which has a flag to indicate is a Page Number.
|
||||
This will be treated differently since we will need to iterate to get each possible page.
|
||||
The page number will no be added here to the query string.
|
||||
*/
|
||||
if (apiQueryParameter.config.isPageNumber === true) {
|
||||
pageNumberParameter = apiQueryParameter
|
||||
} else {
|
||||
queryString = queryString + separator + apiQueryParameter.config.codeName + '=' + apiQueryParameter.config.value
|
||||
separator = "&"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchAllPages() {
|
||||
|
||||
if (pageNumberParameter === undefined) {
|
||||
/*
|
||||
There is no paging mechanism at this Endpoint, so we will just make an
|
||||
Async call to the API Server
|
||||
*/
|
||||
await sleep(apiMap.config.millisecondsBetweenCalls)
|
||||
await fetchAPIData()
|
||||
dataReceivedArray = JSON.parse(apiDataReceived)
|
||||
} else {
|
||||
/*
|
||||
There is a paging mechanism at this Endpoint, so we will iterate through all
|
||||
the available pages. If this process was ran before, we will have at the Status
|
||||
Report the latest page fetched. We will fetch that one again, since it might have
|
||||
been not full the last time we fetched it. We will assume there will be a huge amount
|
||||
of pages available to fetch, and we will also assume that once we get an empty array
|
||||
that will mean that we have requested already the last page with data.
|
||||
*/
|
||||
let initialPage = 1
|
||||
let finalPage = 1 // Number.MAX_SAFE_INTEGER
|
||||
if (thisReport.file.lastPage !== undefined) {
|
||||
initialPage = thisReport.file.lastPage[endpointNode.config.codeName]
|
||||
}
|
||||
|
||||
for (let page = initialPage; page <= finalPage; page++) {
|
||||
if (queryString === "") {
|
||||
pageQueryString = "?" + pageNumberParameter.config.codeName + "=" + page
|
||||
} else {
|
||||
pageQueryString = "&" + pageNumberParameter.config.codeName + "=" + page
|
||||
}
|
||||
lastPage[endpointNode.config.codeName] = page
|
||||
|
||||
await sleep(apiMap.config.millisecondsBetweenCalls)
|
||||
let fetchResult = await fetchAPIData()
|
||||
|
||||
switch (fetchResult) {
|
||||
case 'UNEXPECTED_API_RESPONSE': {
|
||||
/*
|
||||
Any unexpected response will abort this loop and allow the process to continue,
|
||||
possibly saving accumulated data.
|
||||
*/
|
||||
page = finalPage
|
||||
break
|
||||
}
|
||||
case 'ERROR_CODE_RECEIVED': {
|
||||
/*
|
||||
An error code at the response will abort this loop and allow the process to continue,
|
||||
possibly saving accumulated data.
|
||||
*/
|
||||
page = finalPage
|
||||
break
|
||||
}
|
||||
case 'NO_MORE_PAGES': {
|
||||
/*
|
||||
We will just abort this loop and continue.
|
||||
*/
|
||||
page = finalPage
|
||||
break
|
||||
}
|
||||
case 'PAGE_FETCHED': {
|
||||
/*
|
||||
Just stay at the current loop and try to fetch more pages.
|
||||
This is how we accumulate the data from multiple pages into a single array.
|
||||
*/
|
||||
dataReceivedArray = dataReceivedArray.concat(apiData)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchAPIData() {
|
||||
let promise = new Promise((resolve, reject) => {
|
||||
/*
|
||||
Now that we have the endpointNode, the parameters and all the information
|
||||
needed to place the call to the API.
|
||||
*/
|
||||
let httpClient = require('https');
|
||||
let url = protocol + '://' +
|
||||
hostName +
|
||||
portNumber +
|
||||
path +
|
||||
'/' + endpointNode.parentNode.config.codeName +
|
||||
'/' + endpointNode.config.codeName +
|
||||
queryString +
|
||||
pageQueryString
|
||||
|
||||
/*
|
||||
This is how we call the API.
|
||||
*/
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[INFO] start -> startProcess -> fetchAPIData -> httpClient.get -> url = " + url)
|
||||
|
||||
httpClient.get(url, onResponse)
|
||||
|
||||
function onResponse(response) {
|
||||
const chunks = []
|
||||
response.on('data', onDataArrived)
|
||||
response.on('end', onEnd)
|
||||
|
||||
let apiResponseSchemaNode
|
||||
let errorCodeReceived
|
||||
|
||||
getApiResponseSchema(response.statusCode)
|
||||
|
||||
if (apiResponseSchemaNode === undefined) {
|
||||
// TODO Error Handling
|
||||
return
|
||||
}
|
||||
|
||||
if (endpoint === undefined) {
|
||||
/* The first enpoint found will be our endpoint */
|
||||
endpoint = endpointFound
|
||||
function onDataArrived(chunk) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
if (endpointFound.id !== endpoint.id) {
|
||||
// TODO Error Handling - we can not reference fields from different enpoints.
|
||||
return
|
||||
function getApiResponseSchema(responseCode) {
|
||||
/*
|
||||
The API Response Schema we will depend on the response code received from the
|
||||
API server. Here we will locate the right Api Response Schema.
|
||||
*/
|
||||
for (let j = 0; j < apiMap.apiVersions.length; j++) {
|
||||
let version = apiMap.apiVersions[j]
|
||||
for (let k = 0; k < version.apiEndpoints.length; k++) {
|
||||
let endpoint = version.apiEndpoints[k]
|
||||
if (endpointNode.id === endpoint.id) {
|
||||
for (let i = 0; i < endpoint.apiQueryResponses.apiQueryResponses.length; i++) {
|
||||
let apiQueryResponse = endpoint.apiQueryResponses.apiQueryResponses[i]
|
||||
|
||||
if (Number(apiQueryResponse.config.codeName) === responseCode) {
|
||||
apiResponseSchemaNode = apiQueryResponse.apiResponseSchema
|
||||
|
||||
/*
|
||||
We will check that what we received is not an error code.
|
||||
*/
|
||||
if (apiQueryResponse.config.isError === true) {
|
||||
errorCodeReceived = true
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> getApiResponseSchema -> API Response is an Error -> responseCode = " + responseCode);
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function onEnd() {
|
||||
apiDataReceived = Buffer.concat(chunks).toString('utf8')
|
||||
/*
|
||||
If we received an errror code, we abort the processing at this point.
|
||||
*/
|
||||
if (errorCodeReceived === true) {
|
||||
resolve('ERROR_CODE_RECEIVED')
|
||||
return
|
||||
}
|
||||
/*
|
||||
Here we will validate that the overall format is what we are expecting.
|
||||
*/
|
||||
switch (apiResponseSchemaNode.apiResponseFields.config.fieldType) {
|
||||
case 'object': {
|
||||
/*
|
||||
If we did not received an object, that probably means something is not
|
||||
good, and we got an HTML with the reason inside.
|
||||
*/
|
||||
if (apiDataReceived.substring(0, 1) !== "{") {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiDataReceived = " + apiDataReceived);
|
||||
resolve('UNEXPECTED_API_RESPONSE')
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
case 'array': {
|
||||
/*
|
||||
If we did not received an array, that probably means something is not
|
||||
good, and we got an HTML with the reason inside.
|
||||
*/
|
||||
if (apiDataReceived.substring(0, 1) !== "[") {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiDataReceived = " + apiDataReceived);
|
||||
resolve('UNEXPECTED_API_RESPONSE')
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
/*
|
||||
The actual data we need could be anywhere within the data structure received.
|
||||
The exact place is configured at the apiResponseSchemaNode property nodePath.
|
||||
We will eval the nodePath property (this assumes that the apiDataObject is defined)
|
||||
*/
|
||||
let apiDataObject = JSON.parse(apiDataReceived)
|
||||
apiData = eval(apiResponseSchemaNode.config.nodePath)
|
||||
/*
|
||||
We will expect the apiData to be an Array. Depending if it has data or not we will return
|
||||
NO_MORE_PAGES or PAGE_FETCHED so that pagination procedure knows when to stop.
|
||||
*/
|
||||
if (apiData.length === 0) {
|
||||
resolve('NO_MORE_PAGES')
|
||||
} else {
|
||||
resolve('PAGE_FETCHED')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return promise
|
||||
}
|
||||
/*
|
||||
Let's see if we need to place the API call with Parameters...
|
||||
*/
|
||||
let queryString = ""
|
||||
let separator = ""
|
||||
if (productDefinition.apiQueryParameters !== undefined) {
|
||||
queryString = '?'
|
||||
for (j = 0; j < productDefinition.apiQueryParameters.apiQueryParameters.length; j++) {
|
||||
let apiQueryParameter = productDefinition.apiQueryParameters.apiQueryParameters[j]
|
||||
queryString = queryString + separator + apiQueryParameter.config.codeName + '=' + apiQueryParameter.config.value
|
||||
separator = "&"
|
||||
}
|
||||
}
|
||||
/*
|
||||
Now that we have the Endpoint, we can call the API.
|
||||
*/
|
||||
let http = require('http');
|
||||
let url = protocol + '://' +
|
||||
hostName +
|
||||
portNumber +
|
||||
path +
|
||||
'/' + endpoint.config.codeName +
|
||||
queryString
|
||||
|
||||
http.get(url, onResponse);
|
||||
|
||||
function onResponse(response) {
|
||||
const chunks = []
|
||||
response.on('data', onDataArrived)
|
||||
response.on('end', onEnd)
|
||||
|
||||
function onDataArrived(chunk) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
function onEnd() {
|
||||
let dataReceived = Buffer.concat(chunks).toString('utf8')
|
||||
|
||||
if (fileContent !== undefined) {
|
||||
// we are going to append the curernt dataReceived to the existing file.
|
||||
let fileContentArray = JSON.parse(fileContent)
|
||||
let dataReceivedArray = JSON.parse(dataReceived)
|
||||
|
||||
for (let i = 0; i < dataReceivedArray.length; i++) {
|
||||
let message = dataReceivedArray[i]
|
||||
fileContentArray.push(message)
|
||||
}
|
||||
|
||||
fileContent = JSON.stringify(fileContentArray)
|
||||
} else {
|
||||
// we are going to save the current dataReceived.
|
||||
fileContent = dataReceived
|
||||
}
|
||||
async function readDatasetFile() {
|
||||
/*
|
||||
If this process already ran before, then we are going to load the data stored so as to append
|
||||
information to it later.
|
||||
*/
|
||||
if (thisReport.file.lastRun !== undefined) {
|
||||
|
||||
let fileName = 'Data.json'
|
||||
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + FOLDER_NAME + "/" + 'Single-File'
|
||||
fileStorage.createTextFile(filePath + '/' + fileName, fileContent + '\n', onFileCreated);
|
||||
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + productDefinition.config.codeName + "/" + outputDataset.referenceParent.config.codeName
|
||||
let response = await fileStorage.asyncGetTextFile(filePath + '/' + fileName)
|
||||
let text = response.text
|
||||
|
||||
function onFileCreated(err) {
|
||||
if (err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[ERROR] start -> fetchData -> onResponse -> onEnd -> onFileCreated -> Could not save file. -> filePath = " + filePath + "/" + fileName);
|
||||
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
|
||||
} else {
|
||||
writeStatusReport()
|
||||
if (response.err !== undefined && response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[ERROR] start -> startProcess -> readDatasetFile -> Could read file. -> filePath = " + filePath + "/" + fileName);
|
||||
throw (response.err)
|
||||
} else {
|
||||
existingFileContent = text
|
||||
}
|
||||
|
||||
} else {
|
||||
/*
|
||||
If there is no status report, we assume there is no previous file or that if there is we will override it.
|
||||
*/
|
||||
existingFileContent = "[]"
|
||||
}
|
||||
}
|
||||
|
||||
function appendToExistingDataset() {
|
||||
/*
|
||||
We are going to append the curernt apiDataReceived to the existing file.
|
||||
*/
|
||||
let existingFileArray = JSON.parse(existingFileContent)
|
||||
/*
|
||||
We will create a map with all the existing record primary keys, so as to use it
|
||||
later to avoid appending records that already exists. The first step is to
|
||||
build an array with the primary keys of the file. After that we will populate
|
||||
the existingKeys map.
|
||||
*/
|
||||
let primaryKeys = []
|
||||
let existingKeys = new Map()
|
||||
/*
|
||||
Setup the primaryKeys array.
|
||||
*/
|
||||
for (let j = 0; j < productDefinition.record.properties.length; j++) {
|
||||
let recordProperty = productDefinition.record.properties[j]
|
||||
if (recordProperty.config.primaryKey === true) {
|
||||
primaryKeys.push(recordProperty.config.codeName)
|
||||
}
|
||||
}
|
||||
/*
|
||||
Setup the existingKeys map.
|
||||
*/
|
||||
for (let i = 0; i < existingFileArray.length; i++) {
|
||||
let record = {
|
||||
values: existingFileArray[i],
|
||||
map: new Map()
|
||||
}
|
||||
/*
|
||||
Building the record map...
|
||||
*/
|
||||
for (let j = 0; j < productDefinition.record.properties.length; j++) {
|
||||
let recordProperty = productDefinition.record.properties[j]
|
||||
record.map.set(recordProperty.config.codeName, record.values[j])
|
||||
}
|
||||
/*
|
||||
Building the current key...
|
||||
*/
|
||||
let key = ""
|
||||
for (j = 0; j < primaryKeys.length; j++) {
|
||||
let keyValue = record.map.get(primaryKeys[j])
|
||||
key = key + '->' + keyValue
|
||||
}
|
||||
existingKeys.set(key, record)
|
||||
}
|
||||
/*
|
||||
Scan the dataReceivedArray array and insert records into
|
||||
the existingFileArray array only when they are not going to
|
||||
duplicate a primary key.
|
||||
*/
|
||||
for (let i = 0; i < dataReceivedArray.length; i++) {
|
||||
let record = getRecord(dataReceivedArray[i])
|
||||
|
||||
let key = ""
|
||||
for (j = 0; j < primaryKeys.length; j++) {
|
||||
let keyValue = record.map.get(primaryKeys[j])
|
||||
key = key + '->' + keyValue
|
||||
}
|
||||
if (existingKeys.get(key) === undefined) {
|
||||
existingFileArray.push(record.values)
|
||||
existingKeys.set(key, record.values)
|
||||
}
|
||||
}
|
||||
|
||||
existingFileContent = JSON.stringify(existingFileArray)
|
||||
|
||||
function getRecord(recordReceived) {
|
||||
/*
|
||||
The record to be stored is not directly the record received. The record
|
||||
structure depends on the list of record properties and the data fields each one is referencing.
|
||||
*/
|
||||
let record = {
|
||||
values: [],
|
||||
headers: [],
|
||||
map: new Map()
|
||||
}
|
||||
for (let j = 0; j < productDefinition.record.properties.length; j++) {
|
||||
let recordProperty = productDefinition.record.properties[j]
|
||||
if (recordProperty.apiResponseFieldReference !== undefined) {
|
||||
if (recordProperty.apiResponseFieldReference.referenceParent !== undefined) {
|
||||
let apiResponseField = recordProperty.apiResponseFieldReference.referenceParent
|
||||
/*
|
||||
We will need to get the nodePath for this property, representing the path
|
||||
to the value on the received data.
|
||||
*/
|
||||
let nodePath = recordPropertiesNodePathMap.get(recordProperty.config.codeName)
|
||||
let value = eval(nodePath)
|
||||
record.values.push(value)
|
||||
record.headers.push(recordProperty.config.codeName)
|
||||
record.map.set(recordProperty.config.codeName, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return record
|
||||
}
|
||||
}
|
||||
|
||||
async function saveDatasetFile() {
|
||||
|
||||
let fileName = 'Data.json'
|
||||
let filePath = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + productDefinition.config.codeName + "/" + outputDataset.referenceParent.config.codeName
|
||||
let response = await fileStorage.asyncCreateTextFile(filePath + '/' + fileName, existingFileContent + '\n');
|
||||
|
||||
if (response.err !== undefined && response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[ERROR] start -> startProcess -> saveDatasetFile -> Could not save file. -> filePath = " + filePath + "/" + fileName);
|
||||
throw (response.err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
When all Output Datasets have been succesfully written, we can go and write
|
||||
the Status Report.
|
||||
*/
|
||||
writeStatusReport()
|
||||
|
||||
} catch (err) {
|
||||
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[ERROR] start -> fetchData -> err = " + err.stack);
|
||||
"[ERROR] start -> startProcess -> err = " + err.stack);
|
||||
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
|
||||
abort = true
|
||||
}
|
||||
}
|
||||
|
||||
function writeStatusReport() {
|
||||
try {
|
||||
thisReport.file = {
|
||||
lastRun: (new Date()).toISOString()
|
||||
lastRun: (new Date()).toISOString(),
|
||||
lastPage: lastPage
|
||||
};
|
||||
thisReport.save(onSaved);
|
||||
|
||||
|
||||
@@ -83,7 +83,6 @@ exports.newSuperalgosBotModulesCheckWebhook = function (processIndex) {
|
||||
"[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys));
|
||||
}
|
||||
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
|
||||
abort = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,7 +147,6 @@ exports.newSuperalgosBotModulesCheckWebhook = function (processIndex) {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[ERROR] start -> saveMessages -> err = " + err.stack);
|
||||
callBackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE);
|
||||
abort = true
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user