API Fetcher Demo saving Messari Asset Metrics

This commit is contained in:
Luis Fernando Molina committed 2021-05-13 22:11:17 +02:00
1 parent b52ed20caf
commit 3c12df4245
7 files changed
+171 -19

No files matched your search

@@ -1 +1 @@
{"type":"Workspace","name":"API Fetcher Demo","rootNodes":[{"type":"Network","name":"My Network","project":"Superalgos","networkNodes":[{"type":"Network Node","name":"My Computer","config":"{ \n\"host\": \"localhost\", \n\"webPort\": \"34248\", \n\"webSocketsPort\": \"18041\"\n}","project":"Superalgos","dataTasks":{"type":"Data Tasks","name":"New Data Tasks","project":"Superalgos","projectDataTasks":[{"type":"Project Data Tasks","name":"Superalgos","project":"Superalgos","exchangeDataTasks":[{"type":"Exchange Data Tasks","name":"Binance","project":"Superalgos","marketDataTasks":[{"type":"Market Data Tasks","name":"Binance BTC/USDT","project":"Superalgos","dataMineTasks":[{"type":"Data Mine Tasks","name":"Masters","project":"Superalgos","taskManagers":[{"type":"Task Manager","name":"Masters","project":"Superalgos","tasks":[{"type":"Task","name":"Exchange Raw Data","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Exchange Raw Data","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Exchange Raw Data Historic-OHLCVs","project":"Superalgos","id":"407b3902-312f-4074-a7b2-f4e9ce72db07","savedPayload":{"position":{"x":25257.356993272086,"y":30595.11115747996},"targetPosition":{"x":25233.356993272086,"y":30679.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Historic-OHLCVs","id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc"}}}],"id":"4b5fb696-c41e-4cf0-92e2-e5a64a3871e4","savedPayload":{"position":{"x":25233.356993272086,"y":30679.11115747996},"targetPosition":{"x":25311.356993272086,"y":30585.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"a5933e2f-5aa8-45dd-8f13-e50a42b795b2","savedPayload":{"position":{"x":25311.356993272086,"y":30585.11115747996},"targetPosition":{"x":25217.356993272086,"y":30681.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Webhooks","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Webhooks","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Webhooks Check-Webhook","project":"Superalgos","id":"c5bc3847-e5b9-4345-92a7-319b3248a40e","savedPayload":{"position":{"x":25139.356993272086,"y":30657.11115747996},"targetPosition":{"x":25226.356993272086,"y":30745.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Check-Webhook","id":"69f37820-ab16-4e19-bfd8-386dd6e249a3"}}}],"id":"64cef381-ea35-4f33-bb53-8e06ea009225","savedPayload":{"position":{"x":25226.356993272086,"y":30745.11115747996},"targetPosition":{"x":25188.356993272086,"y":30686.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"d3837a8f-361a-4a28-9691-e0afac2fb8f0","savedPayload":{"position":{"x":25188.356993272086,"y":30686.11115747996},"targetPosition":{"x":25217.356993272086,"y":30681.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Candles Volumes","project":"Superalgos","bot":{"type":"Indicator Bot Instance","name":"Candles Volumes","config":"{ \n\"parameter1\": \"Value for parameter1 defined at the Product Definition\"\n}","project":"Superalgos","processes":[{"type":"Indicator Process Instance","name":"Candles Volumes Multi-Period-Market","project":"Superalgos","id":"1e1ea090-43a5-457e-ac5a-d8c23e242b03","savedPayload":{"position":{"x":25045.356993272086,"y":30709.11115747996},"targetPosition":{"x":25134.356993272086,"y":30790.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Multi-Period-Market","id":"9f683143-a33f-42c1-9fa8-c601f01ee4e1"}}},{"type":"Indicator Process Instance","name":"Candles Volumes Multi-Period-Daily","project":"Superalgos","id":"3bb72ff9-6ce3-42ac-86c5-33f6a3e980c3","savedPayload":{"position":{"x":25046.356993272086,"y":30835.11115747996},"targetPosition":{"x":25134.356993272086,"y":30790.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0}Line truncated
{"type":"Workspace","name":"API Fetcher Demo","rootNodes":[{"type":"Network","name":"My Network","project":"Superalgos","networkNodes":[{"type":"Network Node","name":"My Computer","config":"{ \n\"host\": \"localhost\", \n\"webPort\": \"34248\", \n\"webSocketsPort\": \"18041\"\n}","project":"Superalgos","dataTasks":{"type":"Data Tasks","name":"New Data Tasks","project":"Superalgos","projectDataTasks":[{"type":"Project Data Tasks","name":"Superalgos","project":"Superalgos","exchangeDataTasks":[{"type":"Exchange Data Tasks","name":"Binance","project":"Superalgos","marketDataTasks":[{"type":"Market Data Tasks","name":"Binance BTC/USDT","project":"Superalgos","dataMineTasks":[{"type":"Data Mine Tasks","name":"Masters","project":"Superalgos","taskManagers":[{"type":"Task Manager","name":"Masters","project":"Superalgos","tasks":[{"type":"Task","name":"Exchange Raw Data","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Exchange Raw Data","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Exchange Raw Data Historic-OHLCVs","project":"Superalgos","id":"407b3902-312f-4074-a7b2-f4e9ce72db07","savedPayload":{"position":{"x":23468.38111670686,"y":30308.75940440786},"targetPosition":{"x":23685.50132813838,"y":30101.73570096323},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Historic-OHLCVs","id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc"}}}],"id":"4b5fb696-c41e-4cf0-92e2-e5a64a3871e4","savedPayload":{"position":{"x":23685.50132813838,"y":30101.73570096323},"targetPosition":{"x":23902.6215395699,"y":29894.711997518596},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"a5933e2f-5aa8-45dd-8f13-e50a42b795b2","savedPayload":{"position":{"x":23902.6215395699,"y":29894.711997518596},"targetPosition":{"x":24119.74175100142,"y":29687.688294073963},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Webhooks","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Webhooks","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Webhooks Check-Webhook","project":"Superalgos","id":"c5bc3847-e5b9-4345-92a7-319b3248a40e","savedPayload":{"position":{"x":23319.78984721199,"y":30100.092163628633},"targetPosition":{"x":23586.440481808466,"y":29962.62420711041},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Check-Webhook","id":"69f37820-ab16-4e19-bfd8-386dd6e249a3"}}}],"id":"64cef381-ea35-4f33-bb53-8e06ea009225","savedPayload":{"position":{"x":23586.440481808466,"y":29962.62420711041},"targetPosition":{"x":23853.091116404943,"y":29825.156250592187},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"d3837a8f-361a-4a28-9691-e0afac2fb8f0","savedPayload":{"position":{"x":23853.091116404943,"y":29825.156250592187},"targetPosition":{"x":24119.74175100142,"y":29687.688294073963},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Candles Volumes","project":"Superalgos","bot":{"type":"Indicator Bot Instance","name":"Candles Volumes","config":"{ \n\"parameter1\": \"Value for parameter1 defined at the Product Definition\"\n}","project":"Superalgos","processes":[{"type":"Indicator Process Instance","name":"Candles Volumes Multi-Period-Market","project":"Superalgos","id":"1e1ea090-43a5-457e-ac5a-d8c23e242b03","savedPayload":{"position":{"x":23260.73799407062,"y":29932.322394401017},"targetPosition":{"x":23530.5845326438,"y":29801.23904069021},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Multi-Period-Market","id":"9f683143-a33f-42c1-9fa8-c601f01ee4e1"}}},{"type":"Indicator Process Instance","name":"Candles Volumes Multi-Period-Daily","project":"Superalgos","id":"3bb72ff9-6ce3-42ac-86c5-33f6a3e980c3","savedPayload":{"position":{"x":23231.348898260723,"y":29779.83728573044},"targetPosition":{"x":23530.5845326438,"y":29801.23904069021},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObjLine truncated
@@ -16,6 +16,14 @@
"relatedUiObject": "API Query Parameters",
"actionFunction": "payload.executeAction"
},
{
"action": "Add UI Object",
"disableIfPropertyIsDefined": true,
"propertyToCheckFor": "apiPathParameters",
"label": "Add API Path Parameters",
"relatedUiObject": "API Path Parameters",
"actionFunction": "payload.executeAction"
},
{
"action": "Add UI Object",
"disableIfPropertyIsDefined": true,
@@ -54,6 +62,12 @@
"childType": "API Query Parameters",
"autoAdd": true
},
{
"name": "apiPathParameters",
"type": "node",
"childType": "API Path Parameters",
"autoAdd": true
},
{
"name": "apiQueryResponses",
"type": "node",
@@ -0,0 +1,35 @@
{
"type": "API Path Parameter",
"menuItems": [
{
"action": "Configure",
"actionFunction": "uiObject.configEditor.activate",
"label": "Configure",
"iconPathOn": "configuration",
"iconPathOff": "configuration"
},
{
"action": "Delete UI Object",
"askConfirmation": true,
"confirmationLabel": "Confirm to Delete",
"label": "Delete",
"iconPathOn": "delete-entity",
"iconPathOff": "delete-entity",
"actionFunction": "payload.executeAction"
}
],
"editors": {
"config": true
},
"initialValues": {
"config": "{\"codeName\": \"\", \"isString\": false, \"isOptional\": false, \"description\": \"\"}"
},
"isTitleAllwaysVisible": true,
"addLeftIcons": true,
"level": 4,
"attachingRules": {
"compatibleTypes": "->API Path Parameters->"
},
"propertyNameAtParent": "apiPathParameters",
"icon": "bitcoin"
}
@@ -0,0 +1,35 @@
{
"type": "API Path Parameters",
"menuItems": [
{
"action": "Add UI Object",
"label": "Add API Path Parameter",
"relatedUiObject": "API Path Parameter",
"actionFunction": "payload.executeAction"
},
{
"action": "Delete UI Object",
"askConfirmation": true,
"confirmationLabel": "Confirm to Delete",
"label": "Delete",
"iconPathOn": "delete-entity",
"iconPathOff": "delete-entity",
"actionFunction": "payload.executeAction"
}
],
"addLeftIcons": true,
"level": 3,
"attachingRules": {
"compatibleTypes": "->API Endpoint->Product Definition->"
},
"propertyNameAtParent": "apiPathParameters",
"childrenNodesProperties": [
{
"name": "apiPathParameters",
"type": "array",
"childType": "API Path Parameter",
"autoAdd": true
}
],
"icon": "bitcoin"
}
@@ -82,6 +82,12 @@
"type": "node",
"childType": "API Query Parameters",
"autoAdd": false
},
{
"name": "apiPathParameters",
"type": "node",
"childType": "API Path Parameters",
"autoAdd": false
}
],
"propertyNameAtParent": "products"
@@ -134,16 +134,19 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
let endpointNode // This holds the Endpoint Node at the API Map
let existingFileContent // This holds the data of the existing dataset file.
let queryString = "" // This holds the query string to be sent at the API call.
let pathString = "" // This holds the path string to be sent at the API call.
let apiDataReceived // This hold the data received from the latest API call.
let apiData // This holds the specific object containing the data to be stored inside the structure received by the API call.
let dataReceivedArray = [] // This hold the cumulative data received from all calls to the API (multiple pages of data).
let dataReceivedObject // This hold a single data object when there is no pagination.
let pageNumberParameter // This holds the node that represents a Page Number parameter.
let pageQueryString // This holds the node part of query sting that deals with page numbers
let pageQueryString = "" // This holds the node part of query sting that deals with page numbers
let recordPropertiesNodePathMap = new Map() // This holds the calculated nodePath for each record property, in order to help find the property value on the data received.
getApiEndpointAndSchema()
calculateRecordPropertiesNodePathMap()
getEndpointQueryParameters()
getEndpointPathParameters()
await fetchAllPages()
await readDatasetFile()
appendToExistingDataset()
@@ -233,7 +236,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
function getEndpointQueryParameters() {
/*
Parameters to tue query of the API are all optional. Here we will see if
Parameters to the query of the API are all optional. Here we will see if
we need to place the API call with Parameters...
*/
let separator = ""
@@ -257,6 +260,36 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
}
}
function getEndpointPathParameters() {
/*
Parameters to the path of the API are all optional. Here we will see if
we need to place the API call with Parameters...
*/
let separator = ""
if (productDefinition.apiPathParameters !== undefined) {
pathString = '/'
for (j = 0; j < productDefinition.apiPathParameters.apiPathParameters.length; j++) {
let apiPathParameter = productDefinition.apiPathParameters.apiPathParameters[j]
let parameterValue = apiPathParameter.config.codeName // This is the default value
if (apiPathParameter.config.replaceBy !== undefined) {
switch (apiPathParameter.config.replaceBy) {
case '@BaseAsset': {
parameterValue = TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName
break
}
case '@QuotedAsset': {
parameterValue = TS.projects.superalgos.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName
break
}
}
}
pathString = pathString + separator + parameterValue
separator = "/"
}
}
}
async function fetchAllPages() {
if (pageNumberParameter === undefined) {
@@ -266,7 +299,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
*/
await sleep(apiMap.config.millisecondsBetweenCalls)
await fetchAPIData()
dataReceivedArray = JSON.parse(apiDataReceived)
dataReceivedObject = apiData
} else {
/*
There is a paging mechanism at this Endpoint, so we will iterate through all
@@ -349,6 +382,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
path +
'/' + endpointNode.parentNode.config.codeName +
'/' + endpointNode.config.codeName +
pathString +
queryString +
pageQueryString
@@ -368,10 +402,20 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
let apiResponseSchemaNode
let errorCodeReceived
/*
At this point we have received the first http reponse from the API server,
and from it we can extract the status code, which is the html code of the
response of our request.
With that code we can get the API Response Schema, which will tell us how
to extract data from the response.
*/
getApiResponseSchema(response.statusCode)
if (apiResponseSchemaNode === undefined) {
// TODO Error Handling
console.log('The response code ' + response.statusCode + ' received is not associated with any API Response Schema.')
errorCodeReceived = true
return
}
@@ -551,23 +595,35 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
existingKeys.set(key, record)
}
/*
Scan the dataReceivedArray array and insert records into
the existingFileArray array only when they are not going to
duplicate a primary key.
If we received an array of data then we will try to append it to the current file.
*/
for (let i = 0; i < dataReceivedArray.length; i++) {
let record = getRecord(dataReceivedArray[i])
if (dataReceivedArray.length > 0) {
/*
Scan the dataReceivedArray array and insert records into
the existingFileArray array only when they are not going to
duplicate a primary key.
*/
for (let i = 0; i < dataReceivedArray.length; i++) {
let record = getRecord(dataReceivedArray[i])
let key = ""
for (j = 0; j < primaryKeys.length; j++) {
let keyValue = record.map.get(primaryKeys[j])
key = key + '->' + keyValue
}
if (existingKeys.get(key) === undefined) {
existingFileArray.push(record.values)
existingKeys.set(key, record.values)
let key = ""
for (j = 0; j < primaryKeys.length; j++) {
let keyValue = record.map.get(primaryKeys[j])
key = key + '->' + keyValue
}
if (existingKeys.get(key) === undefined) {
existingFileArray.push(record.values)
existingKeys.set(key, record.values)
}
}
}
/*
If we received a data object then we will try to add it to the current existing file.
*/
if (dataReceivedObject !== undefined) {
let record = getRecord(dataReceivedObject)
existingFileArray.push(record.values)
}
existingFileContent = JSON.stringify(existingFileArray)
@@ -592,6 +648,12 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
*/
let nodePath = recordPropertiesNodePathMap.get(recordProperty.config.codeName)
let value = eval(nodePath)
/*
Check that we do not accept values that will break the JSON format of the file.
*/
if (isNaN(value)) { value = 0 }
if (value === null || value === undefined) { value = 0 }
record.values.push(value)
record.headers.push(recordProperty.config.codeName)
record.map.set(recordProperty.config.codeName, value)
@@ -128,7 +128,7 @@ function newSuperalgosFunctionLibraryTaskFunctions() {
'Process Definition->' +
'Process Output->' +
'Output Dataset Folder->Output Dataset Folder->Output Dataset Folder->Output Dataset Folder->Output Dataset Folder->' +
'Output Dataset->Dataset Definition->Product Definition->API Query Parameters->API Query Parameter->' +
'Output Dataset->Dataset Definition->Product Definition->API Query Parameters->API Query Parameter->API Path Parameters->API Path Parameter->' +
'Product Definition Folder->Product Definition Folder->Product Definition Folder->Product Definition Folder->Product Definition Folder->' +
'Process Dependencies->' +
'Status Dependency->Status Report->Process Definition->' +
@@ -156,7 +156,7 @@ function newSuperalgosFunctionLibraryTaskFunctions() {
'Product Definition Folder->Product Definition Folder->Product Definition Folder->Product Definition Folder->Product Definition Folder->' +
'Data Mine->Trading Mine->Learning Mine->' +
'API Map Reference->' +
'API Map->API Version->API Endpoint->API Query Parameters->API Query Parameter->API Query Responses->API Query Response->API Response Schema->' +
'API Map->API Version->API Endpoint->API Query Parameters->API Query Parameter->API Path Parameters->API Path Parameter->API Query Responses->API Query Response->API Response Schema->' +
'API Response Field->API Response Field->API Response Field->API Response Field->API Response Field->API Response Field->API Response Field->API Response Field->API Response Field->'
let taskDefinition = UI.projects.superalgos.functionLibraries.protocolNode.getProtocolNode(node, false, true, true, false, false, taskLightingPath)