API Fetcher bot, primary keys management
This commit is contained in:
Luis Fernando Molina committed
1 parent
2049216ab7
commit
3e70cda005
2 files changed
+55
-8
No files matched your search
@@ -1 +1 @@
|
||||
{"type":"Data Mine","name":"Masters","config":"{\n\"codeName\": \"Masters\"\n}","project":"Superalgos","sensorBots":[{"type":"Sensor Bot","name":"Exchange Raw Data","config":"{\n \"codeName\": \"Exchange-Raw-Data\"\n}","project":"Superalgos","processes":[{"type":"Process Definition","name":"Historic-OHLCVs","config":"{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical maket data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}","project":"Superalgos","processOutput":{"type":"Process Output","name":"New Process Output","project":"Superalgos","outputDatasets":[],"outputDatasetFolders":[],"id":"dfe2556e-a765-4e38-9d30-7f91090c0069","savedPayload":{"position":{"x":19705.67390477555,"y":18305.314530096774},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","project":"Superalgos","statusDependencies":[{"type":"Status Dependency","name":"Exchange Raw Data Historic-OHLCVs","config":"{\n \"mainUtility\": \"Self Reference\"\n}","project":"Superalgos","id":"ea39c98b-18e2-44ce-9f7f-86a8a76a688a","savedPayload":{"position":{"x":20031.77290451477,"y":18126.735984334173},"targetPosition":{"x":19849.144475812154,"y":18364.741986421544},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":1,"distanceToParent":3,"arrangementStyle":0},"uiObject":{},"referenceParent":{"type":"Status Report","name":"New Status Report","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e"}}}],"dataDependencies":[],"dataMineDataDependencies":[],"id":"5c7fae5a-6da6-4864-a167-f2f21d578c72","savedPayload":{"position":{"x":19849.144475812154,"y":18364.741986421544},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"statusReport":{"type":"Status Report","name":"New Status Report","project":"Superalgos","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e","savedPayload":{"position":{"x":19943.679906862923,"y":18487.94295879939},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionStartedEvent":{"type":"Execution Started Event","name":"","project":"Superalgos","id":"957e7e3a-b167-40be-93b5-9ba4ceff56e5","savedPayload":{"position":{"x":19963.94950552168,"y":18641.90584617493},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionFinishedEvent":{"type":"Execution Finished Event","name":"New Execution Finished Event","project":"Superalgos","id":"fe40a546-43ee-49df-908a-7e72f598f0cd","savedPayload":{"position":{"x":19904.52204919691,"y":18785.376417211533},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc","savedPayload":{"position":{"x":19666.516047109537,"y":18602.747988508916},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}}],"products":[{"type":"Product Definition","name":"Candles","config":"{\n\"codeName\": \"Candles\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-Min\"\n}","project":"Superalgos","id":"1c3696e4-c28e-4744-9100-706d6dce0c3c","savedPayload":{"position":{"x":19839.35218735615,"y":18717.55301821844},"targetPosition":{"x":19689.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":1,"distanceToParent":2,"arrangementStyle":0},"uiObject":{}}}],"id":"2443883b-6959-42a3-ae3b-ef3102ffcdcc","savedPayload":{"position":{"x":19689.35218735615,"y":18717.55301821844},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Product Definition","name":"Volumes","config":"{\n\"codeName\": \"Volumes\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-MiLine truncated
|
||||
{"type":"Data Mine","name":"Masters","config":"{\n\"codeName\": \"Masters\"\n}","project":"Superalgos","sensorBots":[{"type":"Sensor Bot","name":"Exchange Raw Data","config":"{\n \"codeName\": \"Exchange-Raw-Data\"\n}","project":"Superalgos","processes":[{"type":"Process Definition","name":"Historic-OHLCVs","config":"{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical maket data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}","project":"Superalgos","processOutput":{"type":"Process Output","name":"New Process Output","project":"Superalgos","outputDatasets":[],"outputDatasetFolders":[],"id":"dfe2556e-a765-4e38-9d30-7f91090c0069","savedPayload":{"position":{"x":19705.67390477555,"y":18305.314530096774},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","project":"Superalgos","statusDependencies":[{"type":"Status Dependency","name":"Exchange Raw Data Historic-OHLCVs","config":"{\n \"mainUtility\": \"Self Reference\"\n}","project":"Superalgos","id":"ea39c98b-18e2-44ce-9f7f-86a8a76a688a","savedPayload":{"position":{"x":20031.77290451477,"y":18126.735984334173},"targetPosition":{"x":19849.144475812154,"y":18364.741986421544},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":1,"distanceToParent":3,"arrangementStyle":0},"uiObject":{},"referenceParent":{"type":"Status Report","name":"New Status Report","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e"}}}],"dataDependencies":[],"dataMineDataDependencies":[],"id":"5c7fae5a-6da6-4864-a167-f2f21d578c72","savedPayload":{"position":{"x":19849.144475812154,"y":18364.741986421544},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"statusReport":{"type":"Status Report","name":"New Status Report","project":"Superalgos","id":"181c7c34-7243-497a-9ddb-1b0067c6d66e","savedPayload":{"position":{"x":19943.679906862923,"y":18487.94295879939},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionStartedEvent":{"type":"Execution Started Event","name":"","project":"Superalgos","id":"957e7e3a-b167-40be-93b5-9ba4ceff56e5","savedPayload":{"position":{"x":19963.94950552168,"y":18641.90584617493},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionFinishedEvent":{"type":"Execution Finished Event","name":"New Execution Finished Event","project":"Superalgos","id":"fe40a546-43ee-49df-908a-7e72f598f0cd","savedPayload":{"position":{"x":19904.52204919691,"y":18785.376417211533},"targetPosition":{"x":19666.516047109537,"y":18602.747988508916},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc","savedPayload":{"position":{"x":19666.516047109537,"y":18602.747988508916},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}}],"products":[{"type":"Product Definition","name":"Candles","config":"{\n\"codeName\": \"Candles\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-Min\"\n}","project":"Superalgos","id":"1c3696e4-c28e-4744-9100-706d6dce0c3c","savedPayload":{"position":{"x":19839.35218735615,"y":18717.55301821844},"targetPosition":{"x":19689.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":1,"distanceToParent":2,"arrangementStyle":0},"uiObject":{}}}],"id":"2443883b-6959-42a3-ae3b-ef3102ffcdcc","savedPayload":{"position":{"x":19689.35218735615,"y":18717.55301821844},"targetPosition":{"x":19389.35218735615,"y":18717.55301821844},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Product Definition","name":"Volumes","config":"{\n\"codeName\": \"Volumes\"\n}","project":"Superalgos","datasets":[{"type":"Dataset Definition","name":"One-Min","config":"{\n\"codeName\": \"One-MiLine truncated
|
||||
@@ -140,9 +140,9 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
/* Async call to the API Server */
|
||||
await fetchAPIData()
|
||||
|
||||
readDatasetFile()
|
||||
readDatasetFile()
|
||||
appendToExistingDataset()
|
||||
saveDatasetFile()
|
||||
saveDatasetFile()
|
||||
|
||||
function getApiEndpoint() {
|
||||
/*
|
||||
@@ -273,15 +273,62 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
function appendToExistingDataset() {
|
||||
if (existingFileContent !== undefined) {
|
||||
// we are going to append the curernt apiDataReceived to the existing file.
|
||||
let fileContentArray = JSON.parse(existingFileContent)
|
||||
let existingFileArray = JSON.parse(existingFileContent)
|
||||
let dataReceivedArray = JSON.parse(apiDataReceived)
|
||||
|
||||
for (let i = 0; i < dataReceivedArray.length; i++) {
|
||||
let message = dataReceivedArray[i]
|
||||
fileContentArray.push(message)
|
||||
/*
|
||||
We will create a map with all the existing record primary keys, so as to use it
|
||||
later to avoid appending records that already exists. The first step is to
|
||||
build an array with the primary keys of the file. After that we will populate
|
||||
the existingKeys map.
|
||||
*/
|
||||
let primaryKeys = []
|
||||
let existingKeys = new Map()
|
||||
|
||||
/*
|
||||
Setup the primaryKeys array.
|
||||
*/
|
||||
for (let j = 0; j < productDefinition.record.properties.length; j++) {
|
||||
let recordProperty = productDefinition.record.properties[j]
|
||||
if (recordProperty.config.primaryKey === true) {
|
||||
primaryKeys.push(recordProperty.config.codeName)
|
||||
}
|
||||
}
|
||||
|
||||
existingFileContent = JSON.stringify(fileContentArray)
|
||||
/*
|
||||
Setup the existingKeys map.
|
||||
*/
|
||||
for (let i = 0; i < existingFileArray.length; i++) {
|
||||
let record = existingFileArray[i]
|
||||
let key = ""
|
||||
for (j = 0; j < primaryKeys.length; j++) {
|
||||
let keyValue = record[primaryKeys[j]]
|
||||
key = key + '->' + keyValue
|
||||
}
|
||||
existingKeys.set(key, record)
|
||||
}
|
||||
|
||||
/*
|
||||
Scan the dataReceivedArray array and insert records into
|
||||
the existingFileArray array only when they are not going to
|
||||
duplicate a primary key.
|
||||
*/
|
||||
for (let i = 0; i < dataReceivedArray.length; i++) {
|
||||
let record = dataReceivedArray[i]
|
||||
|
||||
let key = ""
|
||||
for (j = 0; j < primaryKeys.length; j++) {
|
||||
let keyValue = record[primaryKeys[j]]
|
||||
key = key + '->' + keyValue
|
||||
}
|
||||
if (existingKeys.get(key) === undefined) {
|
||||
existingFileArray.push(record)
|
||||
existingKeys.set(key, record)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
existingFileContent = JSON.stringify(existingFileArray)
|
||||
} else {
|
||||
// we are going to save the current apiDataReceived.
|
||||
existingFileContent = apiDataReceived
|
||||
|
||||
Reference in new issue
Block a user