From 48f22e3eef6ed6fed266316f41d9afbefb20095e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 4 Apr 2020 19:55:22 +0200 Subject: [PATCH 1/3] @product --- src/DailyFiles.js | 2 +- src/FileCloud.js | 8 +++++--- src/FileCursor.js | 4 ++-- src/FileSequence.js | 8 ++++---- src/MarketFiles.js | 4 ++-- src/SingleFile.js | 2 +- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 3ea192514..faf199ce8 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -64,7 +64,7 @@ function newDailyFiles () { /* First we will get the Data Range */ - fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) function onDataRangeReceived (err, pFile) { try { diff --git a/src/FileCloud.js b/src/FileCloud.js index 1b159317d..4ee129964 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -25,15 +25,15 @@ function newFileCloud () { blobService = newFileStorage() } - function getFile (pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } const MAX_RETRIES = 3 - getFileRecursively(0, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pDataset.code.filePath + '-' + pDataset.code.fileName) } @@ -103,6 +103,8 @@ function newFileCloud () { } filePath = filePath.replace('@Period', pPeriodName) + filePath = filePath.replace('@Dataset', pDataset.code.codeName) + filePath = filePath.replace('@Product', pProduct.code.codeName) if (pDatetime !== undefined) { filePath = filePath.replace('@Year', pDatetime.getUTCFullYear()) diff --git a/src/FileCursor.js b/src/FileCursor.js index 1608fc7c3..d35cad879 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -147,7 +147,7 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -447,7 +447,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 759eab6d3..d06eb5bc1 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -83,7 +83,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -128,7 +128,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -193,7 +193,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -229,7 +229,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 7dcda95d0..1954bc5e9 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -95,7 +95,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (dataset.code.validTimeFrames.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -142,7 +142,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (dataset.code.validTimeFrames.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index 1b12b44ef..ba9971e6f 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -28,7 +28,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try { From 51393ad327b119471aed25b95dba536f194d2be8 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 6 Apr 2020 08:21:51 +0200 Subject: [PATCH 2/3] refactoring --- src/FileCursor.js | 8 ++++---- src/FileSequence.js | 4 ++-- src/MarketFiles.js | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index d35cad879..d1bede545 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -44,8 +44,8 @@ function newFileCursor () { function finalize () { try { - systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) - systemEventHandler.stopListening('Data Range Updated', eventSubscriptionIdDataRangeUpdated) + eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) + eventsServerClient.stopListening('Data Range Updated', eventSubscriptionIdDataRangeUpdated) thisObject.eventHandler = undefined @@ -90,10 +90,10 @@ function newFileCursor () { endDateRange = pEndDateRange let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) + eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) + eventsServerClient.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) diff --git a/src/FileSequence.js b/src/FileSequence.js index d06eb5bc1..feedf0b32 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -37,7 +37,7 @@ function newFileSequence () { function finalize () { try { - systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) + eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) thisObject.eventHandler.finalize() thisObject.eventHandler = undefined @@ -75,7 +75,7 @@ function newFileSequence () { callerId = newUniqueId() let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) + eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { eventSubscriptionIdDatasetUpdated = message.eventSubscriptionId diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 1954bc5e9..734314cfd 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -43,7 +43,7 @@ function newMarketFiles () { function finalize () { try { for (let i = 0; i < eventSubscriptionIdDatasetUpdated.length; i++) { - systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated[i]) + eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated[i]) } thisObject.eventHandler.finalize() @@ -109,7 +109,7 @@ function newMarketFiles () { if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) + eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) From 50264e967596b4613faa41d5bcb4a824cf413072 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 6 Apr 2020 17:48:32 +0200 Subject: [PATCH 3/3] Bug fixed with caller Id --- src/FileCursor.js | 6 ++++-- src/MarketFiles.js | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index d1bede545..88c1463e1 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -90,10 +90,12 @@ function newFileCursor () { endDateRange = pEndDateRange let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) + let callerId = key + '-' + periodName + newUniqueId() + eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponseDataSet, updateFiles) key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - eventsServerClient.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) + callerId = key + '-' + periodName + newUniqueId() + eventsServerClient.listenToEvent(key, 'Data Range Updated', undefined, callerId, onResponseDataRange, updateDataRange) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 734314cfd..ec1a45ee2 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -109,7 +109,8 @@ function newMarketFiles () { if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) + let callerId = key + '-' + periodName + newUniqueId() + eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject)