diff --git a/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js b/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js index 0d963edc3..d0eada4b8 100644 --- a/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js +++ b/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js @@ -135,10 +135,11 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { let existingFileContent // This holds the data of the existing dataset file. let queryString = "" // This holds the query string to be sent at the API call. let pathString = "" // This holds the path string to be sent at the API call. - let apiDataReceived // This hold the data received from the latest API call. - let apiData // This holds the specific object containing the data to be stored inside the structure received by the API call. + let apiResponseReceivedText // This hold the whole text received from the latest API call. + let apiResponseReceivedObject // This is the whole response received turned into a JSON object. + let apiData // This holds the specific object containing the data -to be stored- inside the structure received by the API call. let dataReceivedArray = [] // This hold the cumulative data received from all calls to the API (multiple pages of data). - let dataReceivedObject // This hold a single data object when there is no pagination. + let dataReceivedObject // This hold a single data object when there is no pagination. The Data object is a node within the whole response received from the API. let pageNumberParameter // This holds the node that represents a Page Number parameter. let pageQueryString = "" // This holds the node part of query sting that deals with page numbers let recordPropertiesNodePathMap = new Map() // This holds the calculated nodePath for each record property, in order to help find the property value on the data received. @@ -456,7 +457,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { } function onEnd() { - apiDataReceived = Buffer.concat(chunks).toString('utf8') + apiResponseReceivedText = Buffer.concat(chunks).toString('utf8') /* If we received an errror code, we abort the processing at this point. */ @@ -473,9 +474,9 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { If we did not received an object, that probably means something is not good, and we got an HTML with the reason inside. */ - if (apiDataReceived.substring(0, 1) !== "{") { + if (apiResponseReceivedText.substring(0, 1) !== "{") { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiDataReceived = " + apiDataReceived); + "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiResponseReceivedText = " + apiResponseReceivedText); resolve('UNEXPECTED_API_RESPONSE') return } @@ -486,9 +487,9 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { If we did not received an array, that probably means something is not good, and we got an HTML with the reason inside. */ - if (apiDataReceived.substring(0, 1) !== "[") { + if (apiResponseReceivedText.substring(0, 1) !== "[") { TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiDataReceived = " + apiDataReceived); + "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiResponseReceivedText = " + apiResponseReceivedText); resolve('UNEXPECTED_API_RESPONSE') return } @@ -498,9 +499,9 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { /* The actual data we need could be anywhere within the data structure received. The exact place is configured at the apiResponseSchemaNode property nodePath. - We will eval the nodePath property (this assumes that the apiDataObject is defined) + We will eval the nodePath property (this assumes that the apiResponseReceivedObject is defined) */ - let apiDataObject = JSON.parse(apiDataReceived) + apiResponseReceivedObject = JSON.parse(apiResponseReceivedText) apiData = eval(apiResponseSchemaNode.config.nodePath) /* We will expect the apiData to be an Array. Depending if it has data or not we will return @@ -549,7 +550,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { function appendToExistingDataset() { /* - We are going to append the curernt apiDataReceived to the existing file. + We are going to append the curernt apiResponseReceivedText to the existing file. */ let existingFileArray = JSON.parse(existingFileContent) /* @@ -644,11 +645,24 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { let apiResponseField = recordProperty.apiResponseFieldReference.referenceParent /* We will need to get the nodePath for this property, representing the path - to the value on the received data. + to the value on the received data. There are 2 ways to get this nodePath. + + 1) It is explicitly declared at the node. If this exists then we will use it. + 2) It was calculated automatically. */ - let nodePath = recordPropertiesNodePathMap.get(recordProperty.config.codeName) + + let nodePath = recordProperty.config.nodePath + + if (nodePath === undefined) { + nodePath = recordPropertiesNodePathMap.get(recordProperty.config.codeName) + } + let value = eval(nodePath) + if (recordProperty.config.isDate === true) { + value = (new Date(value)).valueOf() + } + /* Check that we do not accept values that will break the JSON format of the file. */