diff --git a/src/DailyFiles.js b/src/DailyFiles.js index d84a67d97..52305e64a 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -50,40 +50,37 @@ function newDailyFiles () { callBackWhenFileReceived = callBackFunction let exchange = pExchange + let beginDateRange + let endDateRange fileCloud = newFileCloud() fileCloud.initialize(pBot, pHost, pPort) - /* Some Validations */ + /* Config Validations */ if (pDataset.config.validTimeFrames === undefined) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Daily Files for bot ' + pBot.name) } if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) return } - /* First we will get the Data Range */ + getDataRangeFile() - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) - - function onDataRangeReceived (err, pFile) { - try { - let beginDateRange - let endDateRange + function getDataRangeFile () { + /* First we will get the Data Range */ + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived) + function onDataRangeReceived (err, pFile) { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { beginDateRange = pFile.begin endDateRange = pFile.end - break } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) return } - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { if (err.message === 'File does not exist.') { beginDateRange = new Date() @@ -97,7 +94,6 @@ function newDailyFiles () { endDateRange = new Date() break } - /* If none of the previous conditions are met, the we return the err to the caller. */ callBackFunction(err) return } @@ -106,60 +102,56 @@ function newDailyFiles () { return } } - /* Now we will get the daily files */ - for (i = 0; i < dailyFilePeriods.length; i++) { - let periodTime = dailyFilePeriods[i][0] - let periodName = dailyFilePeriods[i][1] + createFileCursors() + } + } - if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = newFileCursor() - fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) - function onInitialized (err) { - try { - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - break - } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) - return - } - default: { - callBackWhenFileReceived(err) - return - } + function createFileCursors () { + /* Now we will get the daily files */ + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] + + if (pDataset.config.validTimeFrames.includes(periodName) === true) { + let fileCursor = newFileCursor() + fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) + function onInitialized (err) { + try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + default: { + callBackWhenFileReceived(err) + return } - fileCursors.set(periodTime, fileCursor) - expectedFiles = expectedFiles + fileCursor.getExpectedFiles() - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + fileCursors.set(periodTime, fileCursor) + expectedFiles = expectedFiles + fileCursor.getExpectedFiles() + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } - loadThemAll() + } + loadThemAll() + } - function loadThemAll () { - try { - for (i = 0; i < dailyFilePeriods.length; i++) { - let periodTime = dailyFilePeriods[i][0] - let periodName = dailyFilePeriods[i][1] + function loadThemAll () { + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] - if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = fileCursors.get(periodTime) - fileCursor.reload(onFileReceived) - } - } - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> loadThemAll -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) - } + if (pDataset.config.validTimeFrames.includes(periodName) === true) { + let fileCursor = fileCursors.get(periodTime) + fileCursor.reload(onFileReceived) } - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { diff --git a/src/FileCloud.js b/src/FileCloud.js index 69dfac0d8..06bd8ac9d 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -1,15 +1,12 @@ function newFileCloud () { const MODULE_NAME = 'File Cloud' - const INFO_LOG = false const ERROR_LOG = true const logger = newWebDebugLog() logger.fileName = MODULE_NAME /* - This is the module in the system that actually connects to the cloud storage and grabs from there the needed files. - */ let thisObject = { @@ -18,47 +15,55 @@ function newFileCloud () { } let fileStorage - return thisObject function initialize (pBot, pHost, pPort) { fileStorage = newFileStorage(pHost, pPort) } - function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } - const MAX_RETRIES = 3 + getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) - getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - - function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.config.codeName + '-' + pBot.config.codeName + '-' + pDataset.config.filePath + '-' + pDataset.config.fileName) } - let fileName let filePath - if (pDataRange === undefined) { + if (pDataRange === undefined && pTimeFrames === undefined) { fileName = pDataset.config.fileName filePath = pDataset.config.filePath } else { - if (pDataset.config.dataRange !== undefined) { - fileName = pDataset.config.dataRange.fileName - filePath = pDataset.config.dataRange.filePath - } else { - let customErr = { - result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, - message: 'Missing Configuration.' + if (pDataRange !== undefined) { + if (pDataset.config.dataRange !== undefined) { + fileName = pDataset.config.dataRange.fileName + filePath = pDataset.config.dataRange.filePath + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Missing Configuration.' + } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found at Dataset ' + pDataset.name) } + callBackFunction(customErr) + return + } + } + if (pTimeFrames !== undefined) { + if (pTimeFrames.config.timeFrames !== undefined) { + fileName = pTimeFrames.config.timeFrames.fileName + filePath = pTimeFrames.config.timeFrames.filePath + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Missing Configuration.' + } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Time Frames configuration could not be found at Dataset ' + pDataset.name) } + callBackFunction(customErr) + return } - - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found. ') } - - callBackFunction(customErr) - return } } @@ -118,17 +123,10 @@ function newFileCloud () { fileName = fileName.replace('@Sequence', pSequence) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> filePath = ' + filePath) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> fileName = ' + fileName) } - fileStorage.getFileFromHost(filePath + '/' + fileName, onFileReceived) function onFileReceived (err, text, response) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } - let data if (err && err.result !== GLOBAL.DEFAULT_OK_RESPONSE.result) { @@ -137,9 +135,6 @@ function newFileCloud () { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, message: 'File does not exist.' } - - if (INFO_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } - callBackFunction(customErr) return } @@ -171,9 +166,6 @@ function newFileCloud () { } else { try { data = JSON.parse(text) - - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Data received is valid JSON.') } - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data) return } catch (err) { diff --git a/src/FileCursor.js b/src/FileCursor.js index 85f3bcc8c..134442d34 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -168,7 +168,7 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -468,7 +468,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 187576503..67ac3b493 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -97,7 +97,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -142,7 +142,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -207,7 +207,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -243,7 +243,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index b605e579b..31c8945cb 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -109,7 +109,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (dataset.config.validTimeFrames.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -161,7 +161,7 @@ function newMarketFiles () { if (message.event.timeFrames !== undefined) { if (message.event.timeFrames.includes(periodName) !== true) { continue } } - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index 9a6fdbae8..3103d1207 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -39,7 +39,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try {