From 6130e9e1e543b4fc991e0595540cead83ab7f1b2 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 28 May 2021 19:10:51 +0200 Subject: [PATCH] https module replaced by node-fetch, plus bug fixes at API Data Fetcher Bot --- .../API-Data-Fetcher-Bot/FetchingProcess.js | 289 +++++++++++------- .../API-Data-Fetcher-Bot/package-lock.json | 11 + setup.js | 1 + 3 files changed, 186 insertions(+), 115 deletions(-) create mode 100644 Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/package-lock.json diff --git a/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js b/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js index 8f5cc25b3..c93bd96a6 100644 --- a/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js +++ b/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/FetchingProcess.js @@ -386,10 +386,11 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { } let separator = "" - pathString = '/' + parametersMap.forEach(addParameterToPath) function addParameterToPath(value, key, map) { + if (pathString === "") { pathString = '/' } let apiPathParameter = value let parameterValue = key // This is the default value if (apiPathParameter.config.replaceBy !== undefined) { @@ -517,7 +518,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { Now that we have the endpointNode, the parameters and all the information needed to place the call to the API. */ - let httpClient = require('https') + const fetch = require('node-fetch') let url = protocol + '://' + hostName + portNumber + @@ -532,140 +533,192 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { This is how we call the API. */ TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[INFO] start -> startProcess -> fetchAPIData -> httpClient.get -> url = " + url) + "[INFO] start -> startProcess -> fetchAPIData -> url = " + url) - httpClient.get(url, (response) => { + fetch(url) + .then((response) => { - const chunks = [] + let apiResponseSchemaNode + let errorCodeReceived - let apiResponseSchemaNode - let errorCodeReceived - - /* - At this point we have received the first http reponse from the API server, - and from it we can extract the status code, which is the html code of the - response of our request. - - With that code we can get the API Response Schema, which will tell us how - to extract data from the response. - */ - getApiResponseSchema(response.statusCode) - - if (apiResponseSchemaNode === undefined) { - errorCodeReceived = true - TS.projects.superalgos.utilities.errorHandlingFunctions.throwHandledException( - processIndex, - MODULE_NAME, - 'API Data Fetcher Bot', - { resposeCodeReceived: response.statusCode, detais: 'The response code ' + response.statusCode + ' received is not associated with any API Query Response node.' }, - 'Unexpected API Response Code', - endpointNode - ) - } - - function getApiResponseSchema(responseCode) { /* - The API Response Schema we will depend on the response code received from the - API server. Here we will locate the right Api Response Schema. + At this point we have received the first http reponse from the API server, + and from it we can extract the status code, which is the html code of the + response of our request. + + With that code we can get the API Response Schema, which will tell us how + to extract data from the response. */ - for (let j = 0; j < apiMap.apiVersions.length; j++) { - let version = apiMap.apiVersions[j] - for (let k = 0; k < version.apiEndpoints.length; k++) { - let endpoint = version.apiEndpoints[k] - if (endpointNode.id === endpoint.id) { - for (let i = 0; i < endpoint.apiQueryResponses.apiQueryResponses.length; i++) { - let apiQueryResponse = endpoint.apiQueryResponses.apiQueryResponses[i] + getApiResponseSchema(response.status) - if (Number(apiQueryResponse.config.codeName) === responseCode) { - apiResponseSchemaNode = apiQueryResponse.apiResponseSchema + if (apiResponseSchemaNode === undefined) { + errorCodeReceived = true + TS.projects.superalgos.utilities.errorHandlingFunctions.throwHandledException( + processIndex, + MODULE_NAME, + 'API Data Fetcher Bot', + { resposeCodeReceived: response.status, errorDetails: 'Calling the URL ' + url + '. The response code ' + response.status + ' received is not associated with any API Query Response node.' }, + 'Unexpected API Response Code', + endpointNode + ) + } else { + getResponseBody() + } - /* - We will check that what we received is not an error code. - */ - if (apiQueryResponse.config.isError === true) { - errorCodeReceived = true - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> getApiResponseSchema -> API Response is an Error -> responseCode = " + responseCode) + function getApiResponseSchema(responseCode) { + /* + The API Response Schema we will depend on the response code received from the + API server. Here we will locate the right Api Response Schema. + */ + for (let j = 0; j < apiMap.apiVersions.length; j++) { + let version = apiMap.apiVersions[j] + for (let k = 0; k < version.apiEndpoints.length; k++) { + let endpoint = version.apiEndpoints[k] + if (endpointNode.id === endpoint.id) { + for (let i = 0; i < endpoint.apiQueryResponses.apiQueryResponses.length; i++) { + let apiQueryResponse = endpoint.apiQueryResponses.apiQueryResponses[i] + + if (Number(apiQueryResponse.config.codeName) === responseCode) { + apiResponseSchemaNode = apiQueryResponse.apiResponseSchema + + /* + We will check that what we received is not an error code. + */ + if (apiQueryResponse.config.isError === true) { + errorCodeReceived = true + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> getApiResponseSchema -> API Response is an Error -> responseCode = " + responseCode) + } + return } - return } } } } } - } - response.on('data', (chunk) => { - chunks.push(chunk) - }); - - response.on('end', (d) => { - apiResponseReceivedText = Buffer.concat(chunks).toString('utf8') - /* - If we received an errror code, we abort the processing at this point. - */ - if (errorCodeReceived === true) { - resolve('ERROR_CODE_RECEIVED') - return - } - /* - Here we will validate that the overall format is what we are expecting. - */ - switch (apiResponseSchemaNode.apiResponseFields.config.fieldType) { - case 'object': { + function getResponseBody() { + response.text().then(body => { + apiResponseReceivedText = body /* - If we did not received an object, that probably means something is not - good, and we got an HTML with the reason inside. + If we received an errror code, we abort the processing at this point. */ - if (apiResponseReceivedText.substring(0, 1) !== "{") { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiResponseReceivedText = " + apiResponseReceivedText) - resolve('UNEXPECTED_API_RESPONSE') + if (errorCodeReceived === true) { + resolve('ERROR_CODE_RECEIVED') return } - break - } - case 'array': { /* - If we did not received an array, that probably means something is not - good, and we got an HTML with the reason inside. + Here we will validate that the overall format is what we are expecting. */ - if (apiResponseReceivedText.substring(0, 1) !== "[") { - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiResponseReceivedText = " + apiResponseReceivedText) - resolve('UNEXPECTED_API_RESPONSE') + switch (apiResponseSchemaNode.apiResponseFields.config.fieldType) { + case 'object': { + /* + If we did not received an object, that probably means something is not + good, and we got an HTML with the reason inside. + */ + if (apiResponseReceivedText.substring(0, 1) !== "{") { + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiResponseReceivedText = " + apiResponseReceivedText) + resolve('UNEXPECTED_API_RESPONSE') + return + } + break + } + case 'array': { + /* + If we did not received an array, that probably means something is not + good, and we got an HTML with the reason inside. + */ + if (apiResponseReceivedText.substring(0, 1) !== "[") { + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiResponseReceivedText = " + apiResponseReceivedText) + resolve('UNEXPECTED_API_RESPONSE') + return + } + break + } + } + /* + The actual data we need could be anywhere within the data structure received. + The exact place is configured at the apiResponseSchemaNode property nodePath. + We will eval the nodePath property (this assumes that the apiResponseReceivedObject is defined) + */ + apiResponseReceivedObject = JSON.parse(apiResponseReceivedText) + apiData = eval(apiResponseSchemaNode.config.nodePath) + /* + We will expect the apiData to be an Array. Depending if it has data or not we will return + NO_MORE_PAGES or PAGE_FETCHED so that pagination procedure knows when to stop. + */ + if (apiData.length === 0) { + resolve('NO_MORE_PAGES') + } else { + resolve('PAGE_FETCHED') + } + /* + If we received an errror code, we abort the processing at this point. + */ + if (errorCodeReceived === true) { + resolve('ERROR_CODE_RECEIVED') return } - break - } + /* + Here we will validate that the overall format is what we are expecting. + */ + switch (apiResponseSchemaNode.apiResponseFields.config.fieldType) { + case 'object': { + /* + If we did not received an object, that probably means something is not + good, and we got an HTML with the reason inside. + */ + if (apiResponseReceivedText.substring(0, 1) !== "{") { + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiResponseReceivedText = " + apiResponseReceivedText) + resolve('UNEXPECTED_API_RESPONSE') + return + } + break + } + case 'array': { + /* + If we did not received an array, that probably means something is not + good, and we got an HTML with the reason inside. + */ + if (apiResponseReceivedText.substring(0, 1) !== "[") { + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiResponseReceivedText = " + apiResponseReceivedText) + resolve('UNEXPECTED_API_RESPONSE') + return + } + break + } + } + /* + The actual data we need could be anywhere within the data structure received. + The exact place is configured at the apiResponseSchemaNode property nodePath. + We will eval the nodePath property (this assumes that the apiResponseReceivedObject is defined) + */ + apiResponseReceivedObject = JSON.parse(apiResponseReceivedText) + apiData = eval(apiResponseSchemaNode.config.nodePath) + /* + We will expect the apiData to be an Array. Depending if it has data or not we will return + NO_MORE_PAGES or PAGE_FETCHED so that pagination procedure knows when to stop. + */ + if (apiData.length === 0) { + resolve('NO_MORE_PAGES') + } else { + resolve('PAGE_FETCHED') + } + }) } - /* - The actual data we need could be anywhere within the data structure received. - The exact place is configured at the apiResponseSchemaNode property nodePath. - We will eval the nodePath property (this assumes that the apiResponseReceivedObject is defined) - */ - apiResponseReceivedObject = JSON.parse(apiResponseReceivedText) - apiData = eval(apiResponseSchemaNode.config.nodePath) - /* - We will expect the apiData to be an Array. Depending if it has data or not we will return - NO_MORE_PAGES or PAGE_FETCHED so that pagination procedure knows when to stop. - */ - if (apiData.length === 0) { - resolve('NO_MORE_PAGES') - } else { - resolve('PAGE_FETCHED') - } - }); - - }).on('error', (err) => { - console.error(err); - TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> httpClient.get -> err = " + err.stack) - TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, - "[ERROR] start -> httpClient.get -> Connection to the API Server Failed. Maybe there is no Internet connection. Retrying later. ") - resolve('NO_CONNECTION') - }); + }) + .catch(err => { + TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[ERROR] start -> fetch -> err = " + err.stack) + TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, + "[ERROR] start -> fetch -> Connection to the API Server Failed. Maybe there is no Internet connection. Retrying later. ") + resolve('NO_CONNECTION') + }) } ) @@ -947,7 +1000,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { /* We will need to get the nodePath for this property, representing the path to the value on the received data. There are 2 ways to get this nodePath. - + 1) It is explicitly declared at the node. If this exists then we will use it. 2) It was calculated automatically. */ @@ -1009,10 +1062,16 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) { thisReport.file = { lastRun: (new Date()).toISOString(), lastPage: lastPage, - lastFile: contextVariables.lastFile.toISOString(), beginingOfMarket: contextVariables.beginingOfMarket } + /* + For Single-File Datasets there is no lastFile defined. + */ + if (contextVariables.lastFile !== undefined) { + thisReport.file.lastFile = contextVariables.lastFile.toISOString() + } + if (thisReport.file.beginingOfMarket === undefined) { thisReport.file.beginingOfMarket = (new Date()).toISOString() } diff --git a/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/package-lock.json b/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/package-lock.json new file mode 100644 index 000000000..f32c07b11 --- /dev/null +++ b/Projects/Superalgos/TS/Bot-Modules/API-Data-Fetcher-Bot/package-lock.json @@ -0,0 +1,11 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + } + } +} diff --git a/setup.js b/setup.js index 14cdf84e3..07a3679ab 100644 --- a/setup.js +++ b/setup.js @@ -25,6 +25,7 @@ if (process.argv.includes("noShortcuts")) { let nodeModulesDirs = [ path.join( process.cwd(), "Client"), path.join( process.cwd(), "Projects", "Superalgos", "TS", "Bot-Modules", "Sensor-Bot", "Exchange-Raw-Data"), + path.join( process.cwd(), "Projects", "Superalgos", "TS", "Bot-Modules", "API-Data-Fetcher-Bot"), path.join( process.cwd(), "Projects", "Superalgos", "TS", "Bot-Modules", "Trading-Bot", "Announcements"), path.join( process.cwd(), "Projects", "Superalgos", "TS", "Bot-Modules", "Trading-Bot", "Low-Frequency-Trading", "APIs"), path.join( process.cwd(), "Projects", "Superalgos", "TS", "Task-Modules"),