fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

API Data Fetcher, now it can fetch fields from any part of the response

Browse Source
This commit is contained in:
Luis Fernando Molina committed 2021-05-14 18:55:14 +02:00
1 parent fb0566afe3
commit 8d5fdf3ed4
1 file changed
+27 -13
Download Patch File Download Diff File Expand all files Collapse all files
Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js
+27 -13
View File
@@ -135,10 +135,11 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
let existingFileContent // This holds the data of the existing dataset file.
let queryString = "" // This holds the query string to be sent at the API call.
let pathString = "" // This holds the path string to be sent at the API call.
let apiDataReceived // This hold the data received from the latest API call.
let apiData // This holds the specific object containing the data to be stored inside the structure received by the API call.
let apiResponseReceivedText // This hold the whole text received from the latest API call.
let apiResponseReceivedObject // This is the whole response received turned into a JSON object.
let apiData // This holds the specific object containing the data -to be stored- inside the structure received by the API call.
let dataReceivedArray = [] // This hold the cumulative data received from all calls to the API (multiple pages of data).
let dataReceivedObject // This hold a single data object when there is no pagination.
let dataReceivedObject // This hold a single data object when there is no pagination. The Data object is a node within the whole response received from the API.
let pageNumberParameter // This holds the node that represents a Page Number parameter.
let pageQueryString = "" // This holds the node part of query sting that deals with page numbers
let recordPropertiesNodePathMap = new Map() // This holds the calculated nodePath for each record property, in order to help find the property value on the data received.
@@ -456,7 +457,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
}
function onEnd() {
apiDataReceived = Buffer.concat(chunks).toString('utf8')
apiResponseReceivedText = Buffer.concat(chunks).toString('utf8')
/*
If we received an errror code, we abort the processing at this point.
*/
@@ -473,9 +474,9 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
If we did not received an object, that probably means something is not
good, and we got an HTML with the reason inside.
*/
if (apiDataReceived.substring(0, 1) !== "{") {
if (apiResponseReceivedText.substring(0, 1) !== "{") {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiDataReceived = " + apiDataReceived);
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiResponseReceivedText = " + apiResponseReceivedText);
resolve('UNEXPECTED_API_RESPONSE')
return
}
@@ -486,9 +487,9 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
If we did not received an array, that probably means something is not
good, and we got an HTML with the reason inside.
*/
if (apiDataReceived.substring(0, 1) !== "[") {
if (apiResponseReceivedText.substring(0, 1) !== "[") {
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiDataReceived = " + apiDataReceived);
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiResponseReceivedText = " + apiResponseReceivedText);
resolve('UNEXPECTED_API_RESPONSE')
return
}
@@ -498,9 +499,9 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
/*
The actual data we need could be anywhere within the data structure received.
The exact place is configured at the apiResponseSchemaNode property nodePath.
We will eval the nodePath property (this assumes that the apiDataObject is defined)
We will eval the nodePath property (this assumes that the apiResponseReceivedObject is defined)
*/
let apiDataObject = JSON.parse(apiDataReceived)
apiResponseReceivedObject = JSON.parse(apiResponseReceivedText)
apiData = eval(apiResponseSchemaNode.config.nodePath)
/*
We will expect the apiData to be an Array. Depending if it has data or not we will return
@@ -549,7 +550,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
function appendToExistingDataset() {
/*
We are going to append the curernt apiDataReceived to the existing file.
We are going to append the curernt apiResponseReceivedText to the existing file.
*/
let existingFileArray = JSON.parse(existingFileContent)
/*
@@ -644,11 +645,24 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
let apiResponseField = recordProperty.apiResponseFieldReference.referenceParent
/*
We will need to get the nodePath for this property, representing the path
to the value on the received data.
to the value on the received data. There are 2 ways to get this nodePath.
1) It is explicitly declared at the node. If this exists then we will use it.
2) It was calculated automatically.
*/
let nodePath = recordPropertiesNodePathMap.get(recordProperty.config.codeName)
let nodePath = recordProperty.config.nodePath
if (nodePath === undefined) {
nodePath = recordPropertiesNodePathMap.get(recordProperty.config.codeName)
}
let value = eval(nodePath)
if (recordProperty.config.isDate === true) {
value = (new Date(value)).valueOf()
}
/*
Check that we do not accept values that will break the JSON format of the file.
*/