Merge pull request #932 from Luis-Fernando-Molina/develop
Contribution: API Data Fetcher, now it can fetch fields from any part of the response
This commit is contained in:
19 files changed
+316
-15
No files matched your search
@@ -1 +1 @@
|
||||
{"type":"Workspace","name":"API Fetcher Demo","rootNodes":[{"type":"Network","name":"My Network","project":"Superalgos","networkNodes":[{"type":"Network Node","name":"My Computer","config":"{ \n\"host\": \"localhost\", \n\"webPort\": \"34248\", \n\"webSocketsPort\": \"18041\"\n}","project":"Superalgos","dataTasks":{"type":"Data Tasks","name":"New Data Tasks","project":"Superalgos","projectDataTasks":[{"type":"Project Data Tasks","name":"Superalgos","project":"Superalgos","exchangeDataTasks":[{"type":"Exchange Data Tasks","name":"Binance","project":"Superalgos","marketDataTasks":[{"type":"Market Data Tasks","name":"Binance BTC/USDT","project":"Superalgos","dataMineTasks":[{"type":"Data Mine Tasks","name":"Masters","project":"Superalgos","taskManagers":[{"type":"Task Manager","name":"Masters","project":"Superalgos","tasks":[{"type":"Task","name":"Exchange Raw Data","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Exchange Raw Data","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Exchange Raw Data Historic-OHLCVs","project":"Superalgos","id":"407b3902-312f-4074-a7b2-f4e9ce72db07","savedPayload":{"position":{"x":23468.38111670686,"y":30308.75940440786},"targetPosition":{"x":23685.50132813838,"y":30101.73570096323},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Historic-OHLCVs","id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc"}}}],"id":"4b5fb696-c41e-4cf0-92e2-e5a64a3871e4","savedPayload":{"position":{"x":23685.50132813838,"y":30101.73570096323},"targetPosition":{"x":23902.6215395699,"y":29894.711997518596},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"a5933e2f-5aa8-45dd-8f13-e50a42b795b2","savedPayload":{"position":{"x":23902.6215395699,"y":29894.711997518596},"targetPosition":{"x":24119.74175100142,"y":29687.688294073963},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Webhooks","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Webhooks","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Webhooks Check-Webhook","project":"Superalgos","id":"c5bc3847-e5b9-4345-92a7-319b3248a40e","savedPayload":{"position":{"x":23319.78984721199,"y":30100.092163628633},"targetPosition":{"x":23586.440481808466,"y":29962.62420711041},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Check-Webhook","id":"69f37820-ab16-4e19-bfd8-386dd6e249a3"}}}],"id":"64cef381-ea35-4f33-bb53-8e06ea009225","savedPayload":{"position":{"x":23586.440481808466,"y":29962.62420711041},"targetPosition":{"x":23853.091116404943,"y":29825.156250592187},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"d3837a8f-361a-4a28-9691-e0afac2fb8f0","savedPayload":{"position":{"x":23853.091116404943,"y":29825.156250592187},"targetPosition":{"x":24119.74175100142,"y":29687.688294073963},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Candles Volumes","project":"Superalgos","bot":{"type":"Indicator Bot Instance","name":"Candles Volumes","config":"{ \n\"parameter1\": \"Value for parameter1 defined at the Product Definition\"\n}","project":"Superalgos","processes":[{"type":"Indicator Process Instance","name":"Candles Volumes Multi-Period-Market","project":"Superalgos","id":"1e1ea090-43a5-457e-ac5a-d8c23e242b03","savedPayload":{"position":{"x":23260.73799407062,"y":29932.322394401017},"targetPosition":{"x":23530.5845326438,"y":29801.23904069021},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Multi-Period-Market","id":"9f683143-a33f-42c1-9fa8-c601f01ee4e1"}}},{"type":"Indicator Process Instance","name":"Candles Volumes Multi-Period-Daily","project":"Superalgos","id":"3bb72ff9-6ce3-42ac-86c5-33f6a3e980c3","savedPayload":{"position":{"x":23231.348898260723,"y":29779.83728573044},"targetPosition":{"x":23530.5845326438,"y":29801.23904069021},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObjLine truncated
|
||||
{"type":"Workspace","name":"API Fetcher Demo","rootNodes":[{"type":"Network","name":"My Network","project":"Superalgos","networkNodes":[{"type":"Network Node","name":"My Computer","config":"{ \n\"host\": \"localhost\", \n\"webPort\": \"34248\", \n\"webSocketsPort\": \"18041\"\n}","project":"Superalgos","dataTasks":{"type":"Data Tasks","name":"New Data Tasks","project":"Superalgos","projectDataTasks":[{"type":"Project Data Tasks","name":"Superalgos","project":"Superalgos","exchangeDataTasks":[{"type":"Exchange Data Tasks","name":"Binance","project":"Superalgos","marketDataTasks":[{"type":"Market Data Tasks","name":"Binance BTC/USDT","project":"Superalgos","dataMineTasks":[{"type":"Data Mine Tasks","name":"Masters","project":"Superalgos","taskManagers":[{"type":"Task Manager","name":"Masters","project":"Superalgos","tasks":[{"type":"Task","name":"Exchange Raw Data","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Exchange Raw Data","config":"{\n \"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Exchange Raw Data Historic-OHLCVs","project":"Superalgos","id":"407b3902-312f-4074-a7b2-f4e9ce72db07","savedPayload":{"position":{"x":23468.38111670686,"y":30308.75940440786},"targetPosition":{"x":23685.50132813838,"y":30101.73570096323},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Historic-OHLCVs","id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc"}}}],"id":"4b5fb696-c41e-4cf0-92e2-e5a64a3871e4","savedPayload":{"position":{"x":23685.50132813838,"y":30101.73570096323},"targetPosition":{"x":23902.6215395699,"y":29894.711997518596},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"a5933e2f-5aa8-45dd-8f13-e50a42b795b2","savedPayload":{"position":{"x":23902.6215395699,"y":29894.711997518596},"targetPosition":{"x":24119.74175100142,"y":29687.688294073963},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Webhooks","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Webhooks","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Webhooks Check-Webhook","project":"Superalgos","id":"c5bc3847-e5b9-4345-92a7-319b3248a40e","savedPayload":{"position":{"x":23319.78984721199,"y":30100.092163628633},"targetPosition":{"x":23586.440481808466,"y":29962.62420711041},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Check-Webhook","id":"69f37820-ab16-4e19-bfd8-386dd6e249a3"}}}],"id":"64cef381-ea35-4f33-bb53-8e06ea009225","savedPayload":{"position":{"x":23586.440481808466,"y":29962.62420711041},"targetPosition":{"x":23853.091116404943,"y":29825.156250592187},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"d3837a8f-361a-4a28-9691-e0afac2fb8f0","savedPayload":{"position":{"x":23853.091116404943,"y":29825.156250592187},"targetPosition":{"x":24119.74175100142,"y":29687.688294073963},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Candles Volumes","project":"Superalgos","bot":{"type":"Indicator Bot Instance","name":"Candles Volumes","config":"{ \n\"parameter1\": \"Value for parameter1 defined at the Product Definition\"\n}","project":"Superalgos","processes":[{"type":"Indicator Process Instance","name":"Candles Volumes Multi-Period-Market","project":"Superalgos","id":"1e1ea090-43a5-457e-ac5a-d8c23e242b03","savedPayload":{"position":{"x":23260.73799407062,"y":29932.322394401017},"targetPosition":{"x":23530.5845326438,"y":29801.23904069021},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Multi-Period-Market","id":"9f683143-a33f-42c1-9fa8-c601f01ee4e1"}}},{"type":"Indicator Process Instance","name":"Candles Volumes Multi-Period-Daily","project":"Superalgos","id":"3bb72ff9-6ce3-42ac-86c5-33f6a3e980c3","savedPayload":{"position":{"x":23231.348898260723,"y":29779.83728573044},"targetPosition":{"x":23530.5845326438,"y":29801.23904069021},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiLine truncated
|
||||
@@ -28,7 +28,7 @@
|
||||
"config": true
|
||||
},
|
||||
"initialValues": {
|
||||
"config": "{\"protocol\": \"https\",\"portNumber\":\"\"\"portNumber\":\"\",\"hostName\": \"\",\"path\": \"\"}"
|
||||
"config": "{\"protocol\": \"https\",\"portNumber\":\"\",\"hostName\": \"\",\"path\": \"\"}"
|
||||
},
|
||||
"isTitleAllwaysVisible": true,
|
||||
"isHierarchyHead": true,
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"type": "API Data Fetcher Bot Instance",
|
||||
"definition": {
|
||||
"text": "This node represents an instance of an API Data Fetcher Bot. ",
|
||||
"updated": 1621023247380
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Right click and select the pencil button to enter edit mode."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"type": "API Data Fetcher Bot",
|
||||
"definition": {
|
||||
"text": "It is a type of bot that can fetch data from any API on the internet as long as it is provided with a API Map at runtime.",
|
||||
"updated": 1621022644745
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "An API Data Fetcher Bots is similar to a Sensor Bot, except that the second extracts data from exchanges, while the first one can do it from any reachable API, including exchanges.",
|
||||
"updated": 1621022763112
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"type": "API Data Fetcher Process Instance",
|
||||
"definition": {
|
||||
"text": "This node represents an instance of a API Data Fetcher process. When running it will display the process's status.",
|
||||
"updated": 1621023315985
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Right click and select the pencil button to enter edit mode."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"type": "API Endpoint",
|
||||
"definition": {
|
||||
"text": "Endpoints are important aspects of interacting with server-side web APIs, as they specify where resources lie that can be accessed by third party software. Usually the access is via a URI to which HTTP requests are posted, and from which the response is thus expected.",
|
||||
"updated": 1621017376741
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Right click and select the pencil button to enter edit mode."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"type": "API Map Reference",
|
||||
"definition": {
|
||||
"text": "This node allows users to define a reference to an API Map, so that when the bot is ran, it uses that API Map to exctract data from.",
|
||||
"updated": 1621023375333
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Right click and select the pencil button to enter edit mode."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"type": "API Map",
|
||||
"definition": {
|
||||
"text": "An API Map represents the mapping of an API so that is can be consumed by bots within Superalgos. API Maps are created by users and shared as Plugins so that any other community member can use them with their bots. ",
|
||||
"updated": 1621016177698
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The way to create an API Map is to read the API's documentation and for each command and their responses, generate the node structure that represents it.",
|
||||
"updated": 1621016234649
|
||||
},
|
||||
{
|
||||
"style": "Subtitle",
|
||||
"text": "Configuration"
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "protocol: the protocol to be used to access the API. For now either http or https."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "portNumber: the port at which the API server is listening."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "hostName: the domain name or IP address of the API server."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "path: the URL path to where the API is located within the remote server.",
|
||||
"updated": 1621016495461
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "millisecondsBetweenCalls: [Optional] the number of milliseconds to wait before calling the remote host. This is particuary usefull if there is pagination, so that there is a delay before requesting each page."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"type": "API Path Parameter",
|
||||
"definition": {
|
||||
"text": "Path parameter is the type of parameters that is embedded as part of an URL path. It could be fixed string of text or it could be a parameter that is replaced by the API Fetcher Bot for some value at runtime.",
|
||||
"updated": 1621018173339
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Subtitle",
|
||||
"text": "Configuration",
|
||||
"updated": 1621018220723
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Important Properties"
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "replaceBy: this property allows you to instruct the API Fetcher Bot to set a value to this parameter. The possible values are: @BaseAsset and @QuotedAsset",
|
||||
"updated": 1621018303336
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"type": "API Path Parameters",
|
||||
"definition": {
|
||||
"text": "Path parameters are the type of parameters that are embedded as part of an URL path. They could be fixed string of texts or they could be a parameter that is replaced by the API Fetcher Bot for some value at runtime.",
|
||||
"updated": 1621018106262
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "When found at an API Map they define the default values for each parameter, whicle when found at a Data Mine, they define the actual values to be used, which overrides the defaults.",
|
||||
"updated": 1621022867854
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"type": "API Query Parameter",
|
||||
"definition": {
|
||||
"text": "This node represent a single query string parameter.",
|
||||
"updated": 1621022326307
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Subtitle",
|
||||
"text": "Configuration",
|
||||
"updated": 1621022390398
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "codeName: This must match with the name of the query parameter defined at the API docs."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "value: It is the value to be used at this parameter. The API Map can define the value, that would be considered a default value. The same parameter defined at the Data Mine will have the actual value to be used replacing the default.",
|
||||
"updated": 1621022463369
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"type": "API Query Parameters",
|
||||
"definition": {
|
||||
"text": "Is a query string, that is a part of a uniform resource locator (URL) that assigns values to specified parameters. The query parameters commonly includes fields added to the base URL.",
|
||||
"updated": 1621022301794
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "When found at the API Map will define the default values for each parameter. When found at the Data Mine will define the actual values to be used, these would replace the defaults.",
|
||||
"updated": 1621022566230
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"type": "API Query Response",
|
||||
"definition": {
|
||||
"text": "This node enables the definition of a particular response of the API call. ",
|
||||
"updated": 1621021001505
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Subtitle",
|
||||
"text": "Configuration",
|
||||
"updated": 1621021155356
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "codeName: At the codeName property, the API Fetcher Bot will expect to find the http response code for the API call. You will need to have at least one of these nodes with a codeName = \"200\" that is the standard code for success."
|
||||
},
|
||||
{
|
||||
"style": "Success",
|
||||
"text": "If you wish to be able to react to other error codes, you should also have more of this nodes with the codeName of each of the status you want to cover."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"type": "API Query Responses",
|
||||
"definition": {
|
||||
"text": "API calls producee responses. Under this section they are defined.",
|
||||
"updated": 1621020920908
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Right click and select the pencil button to enter edit mode."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"type": "API Response Field Reference",
|
||||
"definition": {
|
||||
"text": "This node allow users to reference an API Data Field from a Record Property. In other words it allows to link each piece of data the user wants to be saved locally to the field at the API Map that will hold the data at runtime.",
|
||||
"updated": 1621022976968
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Once created, you need to reference an API Data Field.",
|
||||
"updated": 1621023087914
|
||||
},
|
||||
{
|
||||
"style": "Important",
|
||||
"text": "All references originating at the same Record Definition must be to API Data Field s living at the same API Endpoint."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"type": "API Response Field",
|
||||
"definition": {
|
||||
"text": "The API responds with a JSON object. Each property of the object is mapped as node of this type. This node can have children of the same type allowing users to map a complex data structure.",
|
||||
"updated": 1621021740145
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Subtitle",
|
||||
"text": "Configuration",
|
||||
"updated": 1621021763335
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "codeName: The codeName needs to be the exact name of the property at the response JSON object holding the value. For the first object, since it is a special case, we usually use the codeName of rootObject.",
|
||||
"updated": 1621021843683
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "fieldType: This property declares if the data is number, a string an object or an array, those are the 4 possible values.",
|
||||
"updated": 1621022104672
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"type": "API Response Schema",
|
||||
"definition": {
|
||||
"text": "This node represents the mapping of the JSON response of the API into a data structure built by API Response Field nodes.",
|
||||
"updated": 1621021486718
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Subtitle",
|
||||
"text": "Configuration",
|
||||
"updated": 1621021492973
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "nodePath: the response from the server might include some header type of data and the actual data requested. Since the response is a JSON object, then the actual data might be located somewhere within the structure of that object. This property allows you to declare the path to the actual data requested, so as to jump the header-like information."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"type": "API Version",
|
||||
"definition": {
|
||||
"text": "An API might have different versions. This node is used for specifying the version that it's children are defining. At the codeName property we expect the version string as needed on the URL when calling the API Server.",
|
||||
"updated": 1621016746266
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Right click and select the pencil button to enter edit mode."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -135,10 +135,11 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
let existingFileContent // This holds the data of the existing dataset file.
|
||||
let queryString = "" // This holds the query string to be sent at the API call.
|
||||
let pathString = "" // This holds the path string to be sent at the API call.
|
||||
let apiDataReceived // This hold the data received from the latest API call.
|
||||
let apiData // This holds the specific object containing the data to be stored inside the structure received by the API call.
|
||||
let apiResponseReceivedText // This hold the whole text received from the latest API call.
|
||||
let apiResponseReceivedObject // This is the whole response received turned into a JSON object.
|
||||
let apiData // This holds the specific object containing the data -to be stored- inside the structure received by the API call.
|
||||
let dataReceivedArray = [] // This hold the cumulative data received from all calls to the API (multiple pages of data).
|
||||
let dataReceivedObject // This hold a single data object when there is no pagination.
|
||||
let dataReceivedObject // This hold a single data object when there is no pagination. The Data object is a node within the whole response received from the API.
|
||||
let pageNumberParameter // This holds the node that represents a Page Number parameter.
|
||||
let pageQueryString = "" // This holds the node part of query sting that deals with page numbers
|
||||
let recordPropertiesNodePathMap = new Map() // This holds the calculated nodePath for each record property, in order to help find the property value on the data received.
|
||||
@@ -456,7 +457,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
}
|
||||
|
||||
function onEnd() {
|
||||
apiDataReceived = Buffer.concat(chunks).toString('utf8')
|
||||
apiResponseReceivedText = Buffer.concat(chunks).toString('utf8')
|
||||
/*
|
||||
If we received an errror code, we abort the processing at this point.
|
||||
*/
|
||||
@@ -473,9 +474,9 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
If we did not received an object, that probably means something is not
|
||||
good, and we got an HTML with the reason inside.
|
||||
*/
|
||||
if (apiDataReceived.substring(0, 1) !== "{") {
|
||||
if (apiResponseReceivedText.substring(0, 1) !== "{") {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiDataReceived = " + apiDataReceived);
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiResponseReceivedText = " + apiResponseReceivedText);
|
||||
resolve('UNEXPECTED_API_RESPONSE')
|
||||
return
|
||||
}
|
||||
@@ -486,9 +487,9 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
If we did not received an array, that probably means something is not
|
||||
good, and we got an HTML with the reason inside.
|
||||
*/
|
||||
if (apiDataReceived.substring(0, 1) !== "[") {
|
||||
if (apiResponseReceivedText.substring(0, 1) !== "[") {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiDataReceived = " + apiDataReceived);
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiResponseReceivedText = " + apiResponseReceivedText);
|
||||
resolve('UNEXPECTED_API_RESPONSE')
|
||||
return
|
||||
}
|
||||
@@ -498,9 +499,9 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
/*
|
||||
The actual data we need could be anywhere within the data structure received.
|
||||
The exact place is configured at the apiResponseSchemaNode property nodePath.
|
||||
We will eval the nodePath property (this assumes that the apiDataObject is defined)
|
||||
We will eval the nodePath property (this assumes that the apiResponseReceivedObject is defined)
|
||||
*/
|
||||
let apiDataObject = JSON.parse(apiDataReceived)
|
||||
apiResponseReceivedObject = JSON.parse(apiResponseReceivedText)
|
||||
apiData = eval(apiResponseSchemaNode.config.nodePath)
|
||||
/*
|
||||
We will expect the apiData to be an Array. Depending if it has data or not we will return
|
||||
@@ -549,7 +550,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
|
||||
function appendToExistingDataset() {
|
||||
/*
|
||||
We are going to append the curernt apiDataReceived to the existing file.
|
||||
We are going to append the curernt apiResponseReceivedText to the existing file.
|
||||
*/
|
||||
let existingFileArray = JSON.parse(existingFileContent)
|
||||
/*
|
||||
@@ -644,11 +645,24 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
let apiResponseField = recordProperty.apiResponseFieldReference.referenceParent
|
||||
/*
|
||||
We will need to get the nodePath for this property, representing the path
|
||||
to the value on the received data.
|
||||
to the value on the received data. There are 2 ways to get this nodePath.
|
||||
|
||||
1) It is explicitly declared at the node. If this exists then we will use it.
|
||||
2) It was calculated automatically.
|
||||
*/
|
||||
let nodePath = recordPropertiesNodePathMap.get(recordProperty.config.codeName)
|
||||
|
||||
let nodePath = recordProperty.config.nodePath
|
||||
|
||||
if (nodePath === undefined) {
|
||||
nodePath = recordPropertiesNodePathMap.get(recordProperty.config.codeName)
|
||||
}
|
||||
|
||||
let value = eval(nodePath)
|
||||
|
||||
if (recordProperty.config.isDate === true) {
|
||||
value = (new Date(value)).valueOf()
|
||||
}
|
||||
|
||||
/*
|
||||
Check that we do not accept values that will break the JSON format of the file.
|
||||
*/
|
||||
|
||||
Reference in new issue
Block a user