diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 8ee0a0097..9a5820253 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -45,14 +45,14 @@ function newDailyFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimeFrame, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, callBackFunction) { try { callBackWhenFileReceived = callBackFunction let exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this product of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this product of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } fileCloud = newFileCloud() @@ -60,7 +60,7 @@ function newDailyFiles () { /* First we will get the Data Range */ - fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) function onDataRangeReceived (err, pFile) { try { @@ -107,10 +107,10 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pSet.validPeriods.includes(periodName) === true) { + if (pDataset.validPeriods.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, onInitialized) + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, onInitialized) function onInitialized (err) { try { switch (err.result) { @@ -143,7 +143,7 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pSet.validPeriods.includes(periodName) === true) { + if (pDataset.validPeriods.includes(periodName) === true) { let fileCursor = fileCursors.get(periodTime) fileCursor.reload(onFileReceived) } diff --git a/src/FileCloud.js b/src/FileCloud.js index b7652eac2..97b3885f5 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -22,53 +22,32 @@ function newFileCloud () { return thisObject function initialize (pBot) { - try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> pBot = ' + pBot.codeName) } - - switch (STORAGE_PROVIDER) { - case 'Azure': { - blobService = AzureStorage.Blob.createBlobServiceWithSas(pBot.storage.fileUri, pBot.storage.sas) - break - } - case 'AAWeb': { - blobService = newFileStorage() - break - } - default: { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> Storage Provider not supported -> process.env.STORAGE_PROVIDER = ' + STORAGE_PROVIDER) } - return - } - } - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } - // callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); TODO> Handle this callback. - } + blobService = newFileStorage() } - function getFile (pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } const MAX_RETRIES = 3 - getFileRecursively(0, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(0, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pDataset.filePath + '-' + pDataset.fileName) } let fileName let filePath if (pDataRange === undefined) { - fileName = pSet.fileName - filePath = pSet.filePath + fileName = pDataset.fileName + filePath = pDataset.filePath } else { - if (pSet.dataRange !== undefined) { - fileName = pSet.dataRange.fileName - filePath = pSet.dataRange.filePath + if (pDataset.dataRange !== undefined) { + fileName = pDataset.dataRange.fileName + filePath = pDataset.dataRange.filePath } else { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, @@ -87,7 +66,7 @@ function newFileCloud () { logger.write('[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: ') logger.write('[ERROR] getFile -> getFileRecursively -> pDataMine = ' + JSON.stringify(pDataMine)) logger.write('[ERROR] getFile -> getFileRecursively -> pBot = ' + JSON.stringify(pBot)) - logger.write('[ERROR] getFile -> getFileRecursively -> pSet = ' + JSON.stringify(pSet)) + logger.write('[ERROR] getFile -> getFileRecursively -> pDataset = ' + JSON.stringify(pDataset)) logger.write('[ERROR] getFile -> getFileRecursively -> pExchange = ' + JSON.stringify(pExchange)) logger.write('[ERROR] getFile -> getFileRecursively -> pMarket = ' + JSON.stringify(pMarket)) logger.write('[ERROR] getFile -> getFileRecursively -> pPeriodName = ' + JSON.stringify(pPeriodName)) @@ -101,14 +80,14 @@ function newFileCloud () { } if (pDataMine !== undefined) { - filePath = filePath.replace('@DataMine', pDataMine.codeName) + filePath = filePath.replace('@DataMine', pDataMine.code.codeName) } if (pBot !== undefined) { if (pBot.cloneId !== undefined) { - filePath = filePath.replace('@Bot', pBot.codeName + '-' + pBot.cloneId) + filePath = filePath.replace('@Bot', pBot.code.codeName + '-' + pBot.cloneId) } else { - filePath = filePath.replace('@Bot', pBot.codeName) + filePath = filePath.replace('@Bot', pBot.code.codeName) } } @@ -129,7 +108,7 @@ function newFileCloud () { } if (pExchange !== undefined) { - filePath = filePath.replace('@Exchange', pExchange.name) + filePath = filePath.replace('@Exchange', pExchange.code.code.codeName) } filePath = filePath.replace('@Period', pPeriodName) @@ -151,7 +130,7 @@ function newFileCloud () { let containerName - containerName = pDataMine.codeName.toLowerCase() + containerName = pDataMine.code.codeName.toLowerCase() blobService.getBlobToText(containerName, filePath + '/' + fileName, pDataMine.host, onFileReceived) @@ -190,7 +169,7 @@ function newFileCloud () { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = ' + MAX_RETRIES) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = ' + pRetryCounter) } - getFileRecursively(pRetryCounter + 1, pDataMine, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(pRetryCounter + 1, pDataMine, pBot, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) return } else { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. ') } diff --git a/src/FileCursor.js b/src/FileCursor.js index 860ab221d..439ffe270 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -29,7 +29,7 @@ function newFileCursor () { let bot let session let product - let thisSet + let dataset let periodName let timeFrame let beginDateRange @@ -56,7 +56,7 @@ function newFileCursor () { bot = undefined session = undefined product = undefined - thisSet = undefined + dataset = undefined periodName = undefined timeFrame = undefined beginDateRange = undefined @@ -73,7 +73,7 @@ function newFileCursor () { } } - function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimeFrame, pCursorDate, pCurrentTimeFrame, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pTimeFrame, pCursorDate, pCurrentTimeFrame, pBeginDateRange, pEndDateRange, callBackFunction) { try { market = pMarket exchange = pExchange @@ -82,17 +82,17 @@ function newFileCursor () { bot = pBot session = pSession product = pProduct - thisSet = pSet + dataset = pDataset periodName = pPeriodName cursorDate = removeTime(pCursorDate) timeFrame = pTimeFrame beginDateRange = pBeginDateRange endDateRange = pEndDateRange - let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) - key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + pPeriodName + key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + pPeriodName systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) @@ -147,7 +147,7 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -447,7 +447,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 3a17299cd..cc9682be7 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -23,7 +23,7 @@ function newFileSequence () { let dataMine let bot let session - let thisSet + let dataset let product let finalized = false let initialized = false @@ -50,7 +50,7 @@ function newFileSequence () { dataMine = undefined bot = undefined session = undefined - thisSet = undefined + dataset = undefined product = undefined finalized = true @@ -59,19 +59,19 @@ function newFileSequence () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } market = pMarket dataMine = pDataMine bot = pBot session = pSession - thisSet = pSet + dataset = pDataset product = pProduct fileCloud = newFileCloud() @@ -79,7 +79,7 @@ function newFileSequence () { callerId = newUniqueId() - let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { @@ -88,7 +88,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -133,7 +133,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -198,7 +198,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -234,7 +234,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/FileStorage.js b/src/FileStorage.js index e19f33a90..8eb476629 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -1,5 +1,5 @@ -function newFileStorage() { - const MODULE_NAME = 'File Cloud' +function newFileStorage () { + const MODULE_NAME = 'File Storage' const INFO_LOG = false const logger = newWebDebugLog() logger.fileName = MODULE_NAME @@ -29,61 +29,17 @@ function newFileStorage() { return thisObject - async function getBlobToText(container, filePath, host, callBackFunction) { + async function getBlobToText (container, filePath, host, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } - if (host.url.indexOf('localhost') !== -1) { - callServer(undefined, 'Storage/' + container + '/' + filePath, (response, fileContent) => { - if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) - } else { - callBackFunction(response) - } - }) - } else { - let headers - let accessToken = window.localStorage.getItem('access_token') - if (accessToken !== null) { - headers = { - authorization: 'Bearer ' + accessToken - } - } - - let response = await axios({ - url: host.url + 'graphql', - method: 'post', - data: { - query: ` - query web_FileContent($file: web_FileInput){ - web_FileContent(file: $file) - } - `, - variables: { - file: { - container: container.toLowerCase(), - filePath, - storage: host.storage, - accessKey: host.accessKey - } - } - }, - headers: headers - }) - - if (response.data.errors) { - let error = { code: response.data.errors[0] } - callBackFunction(error) - return - } - - if (response.data.data.web_FileContent) { - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response.data.data.web_FileContent) + callServer(undefined, 'Storage/' + container + '/' + filePath, (response, fileContent) => { + if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) } else { - callBackFunction({ code: 'The specified key does not exist.' }) + callBackFunction(response) } - } - + }) } catch (err) { if (verifyRetry(err.code) && currentRetry < MAX_RETRY) { currentRetry++ @@ -99,9 +55,9 @@ function newFileStorage() { } } - function verifyRetry(errorCode) { + function verifyRetry (errorCode) { for (let i = 0; i < recoverableErrors.length; i++) { - const error = recoverableErrors[i]; + const error = recoverableErrors[i] if (error === errorCode) { return true } diff --git a/src/MarketFiles.js b/src/MarketFiles.js index e36443c60..e9c6f961c 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -27,7 +27,7 @@ function newMarketFiles () { let dataMine let bot let session - let thisSet + let dataset let product let periodName @@ -55,7 +55,7 @@ function newMarketFiles () { dataMine = undefined bot = undefined session = undefined - thisSet = undefined + dataset = undefined product = undefined periodName = undefined @@ -65,19 +65,19 @@ function newMarketFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } market = pMarket dataMine = pDataMine bot = pBot session = pSession - thisSet = pSet + dataset = pDataset product = pProduct fileCloud = newFileCloud() @@ -89,8 +89,8 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + if (dataset.validPeriods.includes(periodName) === true) { + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -103,7 +103,7 @@ function newMarketFiles () { } if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { - let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) @@ -136,8 +136,8 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + if (dataset.validPeriods.includes(periodName) === true) { + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index 4673a3be2..031a553c9 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -16,17 +16,17 @@ function newSingleFile () { return thisObject - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pProduct.code.codeName) } let exchange = ecosystem.getExchange(pProduct, pExchange) if (pExchange !== undefined && exchange === undefined) { // We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product. - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } fileCloud = newFileCloud() @@ -34,7 +34,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try {