From 66cdd3f3fd70c7464e96f7db90650669cd0c88d4 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 26 Dec 2019 10:45:42 +0100 Subject: [PATCH 1/8] Data Mine Refactoring --- src/DailyFiles.js | 8 ++++---- src/FileCloud.js | 20 ++++++++++---------- src/FileCursor.js | 16 ++++++++-------- src/FileSequence.js | 20 ++++++++++---------- src/MarketFiles.js | 16 ++++++++-------- src/SingleFile.js | 8 ++++---- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index e91ef8f04..744ded18f 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -45,14 +45,14 @@ function newDailyFiles () { } } - function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { try { callBackWhenFileReceived = callBackFunction let exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this product of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this product of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } fileCloud = newFileCloud() @@ -60,7 +60,7 @@ function newDailyFiles () { /* First we will get the Data Range */ - fileCloud.getFile(pDevTeam, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) function onDataRangeReceived (err, pFile) { try { @@ -110,7 +110,7 @@ function newDailyFiles () { if (pSet.validPeriods.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDevTeam, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) function onInitialized (err) { try { switch (err.result) { diff --git a/src/FileCloud.js b/src/FileCloud.js index 85843234b..c113521c4 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -46,18 +46,18 @@ function newFileCloud () { } } - function getFile (pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } const MAX_RETRIES = 3 - getFileRecursively(0, pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(0, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } let fileName let filePath @@ -85,7 +85,7 @@ function newFileCloud () { if (fileName === undefined) { logger.write('[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: ') - logger.write('[ERROR] getFile -> getFileRecursively -> pDevTeam = ' + JSON.stringify(pDevTeam)) + logger.write('[ERROR] getFile -> getFileRecursively -> pDataMine = ' + JSON.stringify(pDataMine)) logger.write('[ERROR] getFile -> getFileRecursively -> pBot = ' + JSON.stringify(pBot)) logger.write('[ERROR] getFile -> getFileRecursively -> pSet = ' + JSON.stringify(pSet)) logger.write('[ERROR] getFile -> getFileRecursively -> pExchange = ' + JSON.stringify(pExchange)) @@ -100,8 +100,8 @@ function newFileCloud () { fileName = fileName.replace('@AssetB', pMarket.assetB) } - if (pDevTeam !== undefined) { - filePath = filePath.replace('@DevTeam', pDevTeam.codeName) + if (pDataMine !== undefined) { + filePath = filePath.replace('@DataMine', pDataMine.codeName) } if (pBot !== undefined) { @@ -151,9 +151,9 @@ function newFileCloud () { let containerName - containerName = pDevTeam.codeName.toLowerCase() + containerName = pDataMine.codeName.toLowerCase() - blobService.getBlobToText(containerName, filePath + '/' + fileName, pDevTeam.host, onFileReceived) + blobService.getBlobToText(containerName, filePath + '/' + fileName, pDataMine.host, onFileReceived) function onFileReceived (err, text, response) { try { @@ -190,7 +190,7 @@ function newFileCloud () { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = ' + MAX_RETRIES) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = ' + pRetryCounter) } - getFileRecursively(pRetryCounter + 1, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(pRetryCounter + 1, pDataMine, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) return } else { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. ') } diff --git a/src/FileCursor.js b/src/FileCursor.js index 16b97ecfc..22bc7507e 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -25,7 +25,7 @@ function newFileCursor () { let market let exchange let fileCloud - let devTeam + let dataMine let bot let session let product @@ -52,7 +52,7 @@ function newFileCursor () { market = undefined exchange = undefined fileCloud = undefined - devTeam = undefined + dataMine = undefined bot = undefined session = undefined product = undefined @@ -73,12 +73,12 @@ function newFileCursor () { } } - function initialize (pFileCloud, pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { try { market = pMarket exchange = pExchange fileCloud = pFileCloud - devTeam = pDevTeam + dataMine = pDataMine bot = pBot session = pSession product = pProduct @@ -89,10 +89,10 @@ function newFileCursor () { beginDateRange = pBeginDateRange endDateRange = pEndDateRange - let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) - key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + pPeriodName + key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + pPeriodName systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) @@ -147,7 +147,7 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -447,7 +447,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 40fa3eb46..3a17299cd 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -20,7 +20,7 @@ function newFileSequence () { let files = new Map() let maxSequence = -1 // This is replaced by the content of the sequence file, which contains an index that starts on zero. In the case that the sequence file is not found the default value is -1 so when you add 1 it gives you the amount of files in the sequence, zero. let market - let devTeam + let dataMine let bot let session let thisSet @@ -47,7 +47,7 @@ function newFileSequence () { files = undefined maxSequence = undefined market = undefined - devTeam = undefined + dataMine = undefined bot = undefined session = undefined thisSet = undefined @@ -59,16 +59,16 @@ function newFileSequence () { } } - function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } market = pMarket - devTeam = pDevTeam + dataMine = pDataMine bot = pBot session = pSession thisSet = pSet @@ -79,7 +79,7 @@ function newFileSequence () { callerId = newUniqueId() - let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { @@ -88,7 +88,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -133,7 +133,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -198,7 +198,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -234,7 +234,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 717f0766d..e36443c60 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -24,7 +24,7 @@ function newMarketFiles () { let market let exchange - let devTeam + let dataMine let bot let session let thisSet @@ -52,7 +52,7 @@ function newMarketFiles () { market = undefined exchange = undefined - devTeam = undefined + dataMine = undefined bot = undefined session = undefined thisSet = undefined @@ -65,16 +65,16 @@ function newMarketFiles () { } } - function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } market = pMarket - devTeam = pDevTeam + dataMine = pDataMine bot = pBot session = pSession thisSet = pSet @@ -90,7 +90,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -103,7 +103,7 @@ function newMarketFiles () { } if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { - let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) @@ -137,7 +137,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index b8d109aca..4673a3be2 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -16,17 +16,17 @@ function newSingleFile () { return thisObject - function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } let exchange = ecosystem.getExchange(pProduct, pExchange) if (pExchange !== undefined && exchange === undefined) { // We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product. - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } fileCloud = newFileCloud() @@ -34,7 +34,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDevTeam, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try { From 754435d8cc96ba3a256072c192420af45c65f976 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 27 Dec 2019 13:27:21 +0100 Subject: [PATCH 2/8] Time Frame Refectoring --- src/DailyFiles.js | 16 ++++++++-------- src/FileCursor.js | 30 +++++++++++++++--------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 744ded18f..8ee0a0097 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -9,7 +9,7 @@ function newDailyFiles () { eventHandler: undefined, getFileCursor: getFileCursor, setDatetime: setDatetime, - setTimePeriod: setTimePeriod, + setTimeFrame: setTimeFrame, getExpectedFiles: getExpectedFiles, getFilesLoaded: getFilesLoaded, initialize: initialize, @@ -45,7 +45,7 @@ function newDailyFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimeFrame, callBackFunction) { try { callBackWhenFileReceived = callBackFunction @@ -110,7 +110,7 @@ function newDailyFiles () { if (pSet.validPeriods.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, onInitialized) function onInitialized (err) { try { switch (err.result) { @@ -215,19 +215,19 @@ function newDailyFiles () { } } - function setTimePeriod (pTimePeriod, pDatetime) { + function setTimeFrame (pTimeFrame, pDatetime) { try { filesLoaded = 0 expectedFiles = 0 - fileCursors.forEach(setTimePeriodToEach) + fileCursors.forEach(setTimeFrameToEach) - function setTimePeriodToEach (fileCursor, key, map) { - fileCursor.setTimePeriod(pTimePeriod, pDatetime) + function setTimeFrameToEach (fileCursor, key, map) { + fileCursor.setTimeFrame(pTimeFrame, pDatetime) expectedFiles = expectedFiles + fileCursor.getExpectedFiles() fileCursor.reload(onFileReceived) } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> err = ' + err.stack) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimeFrame -> err = ' + err.stack) } } } diff --git a/src/FileCursor.js b/src/FileCursor.js index 22bc7507e..860ab221d 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -12,7 +12,7 @@ function newFileCursor () { eventHandler: undefined, // received from parent reload: reload, setDatetime: setDatetime, - setTimePeriod: setTimePeriod, + setTimeFrame: setTimeFrame, files: new Map(), getExpectedFiles: getExpectedFiles, initialize: initialize, @@ -31,7 +31,7 @@ function newFileCursor () { let product let thisSet let periodName - let timePeriod + let timeFrame let beginDateRange let endDateRange @@ -58,7 +58,7 @@ function newFileCursor () { product = undefined thisSet = undefined periodName = undefined - timePeriod = undefined + timeFrame = undefined beginDateRange = undefined endDateRange = undefined @@ -66,14 +66,14 @@ function newFileCursor () { cursorDate = undefined fileCloud = undefined periodName = undefined - timePeriod = undefined + timeFrame = undefined finalized = true } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err.stack) } } } - function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimeFrame, pCursorDate, pCurrentTimeFrame, pBeginDateRange, pEndDateRange, callBackFunction) { try { market = pMarket exchange = pExchange @@ -85,7 +85,7 @@ function newFileCursor () { thisSet = pSet periodName = pPeriodName cursorDate = removeTime(pCursorDate) - timePeriod = pTimePeriod + timeFrame = pTimeFrame beginDateRange = pBeginDateRange endDateRange = pEndDateRange @@ -167,7 +167,7 @@ function newFileCursor () { } } - function setTimePeriod (pTimePeriod, pDatetime) { + function setTimeFrame (pTimeFrame, pDatetime) { try { if (finalized === true) { return } @@ -177,7 +177,7 @@ function newFileCursor () { We say there is a saving mode where the cursor is running at a minimum size. When the end user aproaches the time period the cursor is set, then it should exit the saving mode and go to its actual size. - To do this we are going to measure the distance from the Time Period received to the one the cursors was initialized with. + To do this we are going to measure the distance from the Time Frame received to the one the cursors was initialized with. If these periods are consecutive, it means that the cursor should exit saving mode and load its full size. */ @@ -189,11 +189,11 @@ function newFileCursor () { for (let i = 0; i < dailyFilePeriods.length; i++) { let period = dailyFilePeriods[i] - if (period[0] === pTimePeriod) { + if (period[0] === pTimeFrame) { positionA = i } - if (period[0] === timePeriod) { + if (period[0] === timeFrame) { positionB = i } } @@ -206,7 +206,7 @@ function newFileCursor () { function enterSavingMode () { try { - switch (timePeriod) { + switch (timeFrame) { case _45_MINUTES_IN_MILISECONDS: { @@ -277,13 +277,13 @@ function newFileCursor () { default: } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> enterSavingMode -> err = ' + err.stack) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimeFrame -> enterSavingMode -> err = ' + err.stack) } } } function exitSavingMode () { try { - switch (timePeriod) { + switch (timeFrame) { case _45_MINUTES_IN_MILISECONDS: { @@ -354,11 +354,11 @@ function newFileCursor () { default: } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> exitSavingMode -> err = ' + err.stack) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimeFrame -> exitSavingMode -> err = ' + err.stack) } } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> err = ' + err.stack) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimeFrame -> err = ' + err.stack) } } } From 6f5e5b3f09cd5d277daf1981746404f459c1826c Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 5 Jan 2020 15:15:16 +0100 Subject: [PATCH 3/8] baseAsset and quotedAsset refactoring --- src/FileCloud.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index c113521c4..b7652eac2 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -96,8 +96,8 @@ function newFileCloud () { } if (pMarket !== undefined) { - fileName = fileName.replace('@AssetA', pMarket.assetA) - fileName = fileName.replace('@AssetB', pMarket.assetB) + fileName = fileName.replace('@BaseAsset', pMarket.baseAsset) + fileName = fileName.replace('@QuotedAsset', pMarket.quotedAsset) } if (pDataMine !== undefined) { From a2d23584c84cd769fd369510076c11cc278c17e3 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 15 Jan 2020 08:29:50 +0100 Subject: [PATCH 4/8] Integrating UI data --- src/DailyFiles.js | 12 ++++----- src/FileCloud.js | 55 ++++++++++++-------------------------- src/FileCursor.js | 16 ++++++------ src/FileSequence.js | 20 +++++++------- src/FileStorage.js | 64 +++++++-------------------------------------- src/MarketFiles.js | 20 +++++++------- src/SingleFile.js | 8 +++--- 7 files changed, 65 insertions(+), 130 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 8ee0a0097..9a5820253 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -45,14 +45,14 @@ function newDailyFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimeFrame, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, callBackFunction) { try { callBackWhenFileReceived = callBackFunction let exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this product of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this product of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } fileCloud = newFileCloud() @@ -60,7 +60,7 @@ function newDailyFiles () { /* First we will get the Data Range */ - fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) function onDataRangeReceived (err, pFile) { try { @@ -107,10 +107,10 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pSet.validPeriods.includes(periodName) === true) { + if (pDataset.validPeriods.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, onInitialized) + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, onInitialized) function onInitialized (err) { try { switch (err.result) { @@ -143,7 +143,7 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pSet.validPeriods.includes(periodName) === true) { + if (pDataset.validPeriods.includes(periodName) === true) { let fileCursor = fileCursors.get(periodTime) fileCursor.reload(onFileReceived) } diff --git a/src/FileCloud.js b/src/FileCloud.js index b7652eac2..97b3885f5 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -22,53 +22,32 @@ function newFileCloud () { return thisObject function initialize (pBot) { - try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> pBot = ' + pBot.codeName) } - - switch (STORAGE_PROVIDER) { - case 'Azure': { - blobService = AzureStorage.Blob.createBlobServiceWithSas(pBot.storage.fileUri, pBot.storage.sas) - break - } - case 'AAWeb': { - blobService = newFileStorage() - break - } - default: { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> Storage Provider not supported -> process.env.STORAGE_PROVIDER = ' + STORAGE_PROVIDER) } - return - } - } - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } - // callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); TODO> Handle this callback. - } + blobService = newFileStorage() } - function getFile (pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } const MAX_RETRIES = 3 - getFileRecursively(0, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(0, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pDataset.filePath + '-' + pDataset.fileName) } let fileName let filePath if (pDataRange === undefined) { - fileName = pSet.fileName - filePath = pSet.filePath + fileName = pDataset.fileName + filePath = pDataset.filePath } else { - if (pSet.dataRange !== undefined) { - fileName = pSet.dataRange.fileName - filePath = pSet.dataRange.filePath + if (pDataset.dataRange !== undefined) { + fileName = pDataset.dataRange.fileName + filePath = pDataset.dataRange.filePath } else { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, @@ -87,7 +66,7 @@ function newFileCloud () { logger.write('[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: ') logger.write('[ERROR] getFile -> getFileRecursively -> pDataMine = ' + JSON.stringify(pDataMine)) logger.write('[ERROR] getFile -> getFileRecursively -> pBot = ' + JSON.stringify(pBot)) - logger.write('[ERROR] getFile -> getFileRecursively -> pSet = ' + JSON.stringify(pSet)) + logger.write('[ERROR] getFile -> getFileRecursively -> pDataset = ' + JSON.stringify(pDataset)) logger.write('[ERROR] getFile -> getFileRecursively -> pExchange = ' + JSON.stringify(pExchange)) logger.write('[ERROR] getFile -> getFileRecursively -> pMarket = ' + JSON.stringify(pMarket)) logger.write('[ERROR] getFile -> getFileRecursively -> pPeriodName = ' + JSON.stringify(pPeriodName)) @@ -101,14 +80,14 @@ function newFileCloud () { } if (pDataMine !== undefined) { - filePath = filePath.replace('@DataMine', pDataMine.codeName) + filePath = filePath.replace('@DataMine', pDataMine.code.codeName) } if (pBot !== undefined) { if (pBot.cloneId !== undefined) { - filePath = filePath.replace('@Bot', pBot.codeName + '-' + pBot.cloneId) + filePath = filePath.replace('@Bot', pBot.code.codeName + '-' + pBot.cloneId) } else { - filePath = filePath.replace('@Bot', pBot.codeName) + filePath = filePath.replace('@Bot', pBot.code.codeName) } } @@ -129,7 +108,7 @@ function newFileCloud () { } if (pExchange !== undefined) { - filePath = filePath.replace('@Exchange', pExchange.name) + filePath = filePath.replace('@Exchange', pExchange.code.code.codeName) } filePath = filePath.replace('@Period', pPeriodName) @@ -151,7 +130,7 @@ function newFileCloud () { let containerName - containerName = pDataMine.codeName.toLowerCase() + containerName = pDataMine.code.codeName.toLowerCase() blobService.getBlobToText(containerName, filePath + '/' + fileName, pDataMine.host, onFileReceived) @@ -190,7 +169,7 @@ function newFileCloud () { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = ' + MAX_RETRIES) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = ' + pRetryCounter) } - getFileRecursively(pRetryCounter + 1, pDataMine, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(pRetryCounter + 1, pDataMine, pBot, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) return } else { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. ') } diff --git a/src/FileCursor.js b/src/FileCursor.js index 860ab221d..439ffe270 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -29,7 +29,7 @@ function newFileCursor () { let bot let session let product - let thisSet + let dataset let periodName let timeFrame let beginDateRange @@ -56,7 +56,7 @@ function newFileCursor () { bot = undefined session = undefined product = undefined - thisSet = undefined + dataset = undefined periodName = undefined timeFrame = undefined beginDateRange = undefined @@ -73,7 +73,7 @@ function newFileCursor () { } } - function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimeFrame, pCursorDate, pCurrentTimeFrame, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pTimeFrame, pCursorDate, pCurrentTimeFrame, pBeginDateRange, pEndDateRange, callBackFunction) { try { market = pMarket exchange = pExchange @@ -82,17 +82,17 @@ function newFileCursor () { bot = pBot session = pSession product = pProduct - thisSet = pSet + dataset = pDataset periodName = pPeriodName cursorDate = removeTime(pCursorDate) timeFrame = pTimeFrame beginDateRange = pBeginDateRange endDateRange = pEndDateRange - let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) - key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + pPeriodName + key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + pPeriodName systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) @@ -147,7 +147,7 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -447,7 +447,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 3a17299cd..cc9682be7 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -23,7 +23,7 @@ function newFileSequence () { let dataMine let bot let session - let thisSet + let dataset let product let finalized = false let initialized = false @@ -50,7 +50,7 @@ function newFileSequence () { dataMine = undefined bot = undefined session = undefined - thisSet = undefined + dataset = undefined product = undefined finalized = true @@ -59,19 +59,19 @@ function newFileSequence () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } market = pMarket dataMine = pDataMine bot = pBot session = pSession - thisSet = pSet + dataset = pDataset product = pProduct fileCloud = newFileCloud() @@ -79,7 +79,7 @@ function newFileSequence () { callerId = newUniqueId() - let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { @@ -88,7 +88,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -133,7 +133,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -198,7 +198,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -234,7 +234,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/FileStorage.js b/src/FileStorage.js index e19f33a90..8eb476629 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -1,5 +1,5 @@ -function newFileStorage() { - const MODULE_NAME = 'File Cloud' +function newFileStorage () { + const MODULE_NAME = 'File Storage' const INFO_LOG = false const logger = newWebDebugLog() logger.fileName = MODULE_NAME @@ -29,61 +29,17 @@ function newFileStorage() { return thisObject - async function getBlobToText(container, filePath, host, callBackFunction) { + async function getBlobToText (container, filePath, host, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } - if (host.url.indexOf('localhost') !== -1) { - callServer(undefined, 'Storage/' + container + '/' + filePath, (response, fileContent) => { - if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) - } else { - callBackFunction(response) - } - }) - } else { - let headers - let accessToken = window.localStorage.getItem('access_token') - if (accessToken !== null) { - headers = { - authorization: 'Bearer ' + accessToken - } - } - - let response = await axios({ - url: host.url + 'graphql', - method: 'post', - data: { - query: ` - query web_FileContent($file: web_FileInput){ - web_FileContent(file: $file) - } - `, - variables: { - file: { - container: container.toLowerCase(), - filePath, - storage: host.storage, - accessKey: host.accessKey - } - } - }, - headers: headers - }) - - if (response.data.errors) { - let error = { code: response.data.errors[0] } - callBackFunction(error) - return - } - - if (response.data.data.web_FileContent) { - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response.data.data.web_FileContent) + callServer(undefined, 'Storage/' + container + '/' + filePath, (response, fileContent) => { + if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) } else { - callBackFunction({ code: 'The specified key does not exist.' }) + callBackFunction(response) } - } - + }) } catch (err) { if (verifyRetry(err.code) && currentRetry < MAX_RETRY) { currentRetry++ @@ -99,9 +55,9 @@ function newFileStorage() { } } - function verifyRetry(errorCode) { + function verifyRetry (errorCode) { for (let i = 0; i < recoverableErrors.length; i++) { - const error = recoverableErrors[i]; + const error = recoverableErrors[i] if (error === errorCode) { return true } diff --git a/src/MarketFiles.js b/src/MarketFiles.js index e36443c60..e9c6f961c 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -27,7 +27,7 @@ function newMarketFiles () { let dataMine let bot let session - let thisSet + let dataset let product let periodName @@ -55,7 +55,7 @@ function newMarketFiles () { dataMine = undefined bot = undefined session = undefined - thisSet = undefined + dataset = undefined product = undefined periodName = undefined @@ -65,19 +65,19 @@ function newMarketFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } market = pMarket dataMine = pDataMine bot = pBot session = pSession - thisSet = pSet + dataset = pDataset product = pProduct fileCloud = newFileCloud() @@ -89,8 +89,8 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + if (dataset.validPeriods.includes(periodName) === true) { + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -103,7 +103,7 @@ function newMarketFiles () { } if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { - let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) @@ -136,8 +136,8 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + if (dataset.validPeriods.includes(periodName) === true) { + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index 4673a3be2..031a553c9 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -16,17 +16,17 @@ function newSingleFile () { return thisObject - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pProduct.code.codeName) } let exchange = ecosystem.getExchange(pProduct, pExchange) if (pExchange !== undefined && exchange === undefined) { // We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product. - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } fileCloud = newFileCloud() @@ -34,7 +34,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try { From 406499438a9211a5586bda801565abaec816b00e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 15 Jan 2020 09:04:54 +0100 Subject: [PATCH 5/8] Bugs Fixed --- src/DailyFiles.js | 10 +++------- src/FileCloud.js | 22 +++++++++------------- src/FileSequence.js | 7 +------ src/MarketFiles.js | 11 +++-------- src/SingleFile.js | 8 +------- 5 files changed, 17 insertions(+), 41 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 9a5820253..37b7c7464 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -49,11 +49,7 @@ function newDailyFiles () { try { callBackWhenFileReceived = callBackFunction - let exchange = ecosystem.getExchange(pProduct, pExchange) - - if (exchange === undefined) { - throw 'Exchange not supoorted by this product of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange - } + let exchange = pExchange fileCloud = newFileCloud() fileCloud.initialize(pBot) @@ -107,7 +103,7 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pDataset.validPeriods.includes(periodName) === true) { + if (pDataset.code.validTimeFrames.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, onInitialized) @@ -143,7 +139,7 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pDataset.validPeriods.includes(periodName) === true) { + if (pDataset.code.validTimeFrames.includes(periodName) === true) { let fileCursor = fileCursors.get(periodTime) fileCursor.reload(onFileReceived) } diff --git a/src/FileCloud.js b/src/FileCloud.js index 97b3885f5..3cb31e8d5 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -36,18 +36,18 @@ function newFileCloud () { function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pDataset.filePath + '-' + pDataset.fileName) } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pDataset.code.filePath + '-' + pDataset.code.fileName) } let fileName let filePath if (pDataRange === undefined) { - fileName = pDataset.fileName - filePath = pDataset.filePath + fileName = pDataset.code.fileName + filePath = pDataset.code.filePath } else { - if (pDataset.dataRange !== undefined) { - fileName = pDataset.dataRange.fileName - filePath = pDataset.dataRange.filePath + if (pDataset.code.dataRange !== undefined) { + fileName = pDataset.code.dataRange.fileName + filePath = pDataset.code.dataRange.filePath } else { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, @@ -84,11 +84,7 @@ function newFileCloud () { } if (pBot !== undefined) { - if (pBot.cloneId !== undefined) { - filePath = filePath.replace('@Bot', pBot.code.codeName + '-' + pBot.cloneId) - } else { - filePath = filePath.replace('@Bot', pBot.code.codeName) - } + filePath = filePath.replace('@Bot', pBot.code.codeName) } if (pSession !== undefined) { @@ -108,7 +104,7 @@ function newFileCloud () { } if (pExchange !== undefined) { - filePath = filePath.replace('@Exchange', pExchange.code.code.codeName) + filePath = filePath.replace('@Exchange', pExchange.code.codeName) } filePath = filePath.replace('@Period', pPeriodName) @@ -130,7 +126,7 @@ function newFileCloud () { let containerName - containerName = pDataMine.code.codeName.toLowerCase() + containerName = pDataMine.codeName.toLowerCase() blobService.getBlobToText(containerName, filePath + '/' + fileName, pDataMine.host, onFileReceived) diff --git a/src/FileSequence.js b/src/FileSequence.js index cc9682be7..3646cfcc1 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -61,12 +61,7 @@ function newFileSequence () { function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { - exchange = ecosystem.getExchange(pProduct, pExchange) - - if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange - } - + exchange = pExchange market = pMarket dataMine = pDataMine bot = pBot diff --git a/src/MarketFiles.js b/src/MarketFiles.js index e9c6f961c..272f35476 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -67,12 +67,7 @@ function newMarketFiles () { function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { - exchange = ecosystem.getExchange(pProduct, pExchange) - - if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange - } - + exchange = pExchange market = pMarket dataMine = pDataMine bot = pBot @@ -89,7 +84,7 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (dataset.validPeriods.includes(periodName) === true) { + if (dataset.code.validTimeFrames.includes(periodName) === true) { fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { @@ -136,7 +131,7 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (dataset.validPeriods.includes(periodName) === true) { + if (dataset.code.validTimeFrames.includes(periodName) === true) { fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { diff --git a/src/SingleFile.js b/src/SingleFile.js index 031a553c9..1b12b44ef 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -21,13 +21,7 @@ function newSingleFile () { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pProduct.code.codeName) } - let exchange = ecosystem.getExchange(pProduct, pExchange) - - if (pExchange !== undefined && exchange === undefined) { - // We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product. - - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange - } + let exchange = pExchange fileCloud = newFileCloud() fileCloud.initialize(pBot) From b0fdfc8a4be626fac15e5f30726bad56a203b149 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 15 Jan 2020 09:11:43 +0100 Subject: [PATCH 6/8] Bug Fixed --- src/FileCloud.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index 3cb31e8d5..f7bd02161 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -126,9 +126,9 @@ function newFileCloud () { let containerName - containerName = pDataMine.codeName.toLowerCase() + containerName = pDataMine.code.codeName.toLowerCase() - blobService.getBlobToText(containerName, filePath + '/' + fileName, pDataMine.host, onFileReceived) + blobService.getBlobToText(containerName, filePath + '/' + fileName, undefined, onFileReceived) function onFileReceived (err, text, response) { try { From 797fb06544041ff010b794f15d0467316e63d88d Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 21 Jan 2020 14:44:58 +0100 Subject: [PATCH 7/8] adapting --- src/FileCloud.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index f7bd02161..b7569918b 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -91,13 +91,8 @@ function newFileCloud () { let code let sessionFolderName = pSession.id if (pSession.code !== undefined) { - try { - code = JSON.parse(pSession.code) - if (code.folderName !== undefined) { - sessionFolderName = code.folderName + '-' + pSession.id - } - } catch (err) { - sessionFolderName = pSession.id + if (pSession.code.folderName !== undefined) { + sessionFolderName = pSession.code.folderName + '-' + pSession.id } } filePath = filePath.replace('@Session', sessionFolderName) From 1909e8c54ffbacb3231815bee9de3f769a182ccc Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 28 Jan 2020 13:28:54 +0100 Subject: [PATCH 8/8] Removing Container Name from Paths --- FileStorage | 54 ---------------------------------------------- src/FileCloud.js | 8 +------ src/FileStorage.js | 6 +++--- 3 files changed, 4 insertions(+), 64 deletions(-) delete mode 100644 FileStorage diff --git a/FileStorage b/FileStorage deleted file mode 100644 index c6c12d006..000000000 --- a/FileStorage +++ /dev/null @@ -1,54 +0,0 @@ -function newFileStorage() { - - const MODULE_NAME = 'File Cloud' - const INFO_LOG = false - const ERROR_LOG = true - const logger = newWebDebugLog() - logger.fileName = MODULE_NAME - - thisObject = { - getBlobToText: getBlobToText - }; - - return thisObject; - - function getBlobToText(pContainerName, pPath, callBackFunction) { - - let xhttp = new XMLHttpRequest(); - xhttp.onreadystatechange = function () { - if (this.readyState === 4 && this.status === 200) { - - try { - let response = JSON.parse(xhttp.responseText); - callBackFunction(response.err, response.text, ""); - } catch (err) { - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. "); } - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] AppPreLoader -> getBlobToText -> response.text = " & response.text); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, "", ""); - } - } - }; - - try { - - let request = { - conatinerName: pContainerName, - path: pPath - }; - - request = JSON.stringify(request); - - let path = "FileService"; - - let blob = new Blob([request], { type: 'text/plain' }); - - xhttp.open("GET", path, true); - xhttp.send(blob); - - } catch (err) { - - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] getBlobToText -> err.message = " & err.message); } - - } - } -} \ No newline at end of file diff --git a/src/FileCloud.js b/src/FileCloud.js index b7569918b..79caf7511 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -119,11 +119,7 @@ function newFileCloud () { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> filePath = ' + filePath) } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> fileName = ' + fileName) } - let containerName - - containerName = pDataMine.code.codeName.toLowerCase() - - blobService.getBlobToText(containerName, filePath + '/' + fileName, undefined, onFileReceived) + blobService.getBlobToText(filePath + '/' + fileName, undefined, onFileReceived) function onFileReceived (err, text, response) { try { @@ -146,7 +142,6 @@ function newFileCloud () { return } - if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> containerName = ' + containerName) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Unexpected Error Ocurred.') } @@ -181,7 +176,6 @@ function newFileCloud () { return } catch (err) { if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> err = ' + err.stack) } - if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> containerName = ' + containerName) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } diff --git a/src/FileStorage.js b/src/FileStorage.js index 8eb476629..9a9abd787 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -29,11 +29,11 @@ function newFileStorage () { return thisObject - async function getBlobToText (container, filePath, host, callBackFunction) { + async function getBlobToText (filePath, host, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } - callServer(undefined, 'Storage/' + container + '/' + filePath, (response, fileContent) => { + callServer(undefined, 'Storage/' + filePath, (response, fileContent) => { if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) } else { @@ -44,7 +44,7 @@ function newFileStorage () { if (verifyRetry(err.code) && currentRetry < MAX_RETRY) { currentRetry++ if (INFO_LOG === true) { console.log('[INFO] getTextFile -> Retrying connection to the server because received error: ' + err.code + '. Retry #: ' + currentRetry) } - getBlobToText(container, filePath, host, callBackFunction) + getBlobToText(filePath, host, callBackFunction) } else if (err.message === 'Request aborted') { let err = { code: 'The specified key does not exist.' } callBackFunction(err)