diff --git a/src/DailyFiles.js b/src/DailyFiles.js index d84a67d97..f28017eb2 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -1,5 +1,5 @@ -  -function newDailyFiles () { + +function newDailyFiles() { const MODULE_NAME = 'Daily Files' const ERROR_LOG = true const logger = newWebDebugLog() @@ -26,7 +26,7 @@ function newDailyFiles () { return thisObject - function finalize () { + function finalize() { try { thisObject.eventHandler.finalize() thisObject.eventHandler = undefined @@ -37,7 +37,7 @@ function newDailyFiles () { fileCursors.forEach(finalizeEach) - function finalizeEach (item, key, mapObj) { + function finalizeEach(item, key, mapObj) { item.finalize() } } catch (err) { @@ -45,45 +45,43 @@ function newDailyFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, pEventsServerClient, callBackFunction) { + function initialize(pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, pEventsServerClient, callBackFunction) { try { callBackWhenFileReceived = callBackFunction let exchange = pExchange + let beginDateRange + let endDateRange + let timeFrames fileCloud = newFileCloud() fileCloud.initialize(pBot, pHost, pPort) - /* Some Validations */ + /* Config Validations */ if (pDataset.config.validTimeFrames === undefined) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Daily Files for bot ' + pBot.name) } if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) return } - /* First we will get the Data Range */ + getDataRangeFile() - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) - - function onDataRangeReceived (err, pFile) { - try { - let beginDateRange - let endDateRange + function getDataRangeFile() { + /* First we will get the Data Range */ + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived) + function onDataRangeReceived(err, pFile) { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { beginDateRange = pFile.begin endDateRange = pFile.end - break } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) return } - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { if (err.message === 'File does not exist.') { beginDateRange = new Date() @@ -97,7 +95,6 @@ function newDailyFiles () { endDateRange = new Date() break } - /* If none of the previous conditions are met, the we return the err to the caller. */ callBackFunction(err) return } @@ -106,59 +103,115 @@ function newDailyFiles () { return } } - /* Now we will get the daily files */ + getTimeFramesFile() + } + } + + function getTimeFramesFile() { + /* First we will get the Data Range */ + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) + + function onTimeFramesReceived(err, pFile) { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + timeFrames = pFile + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (err.message === 'File does not exist.') { + /* It is ok, we will deal with this later */ + break + } + if (err.message === 'Configured to not Support This.') { + /* It is ok, we will deal with this later */ + break + } + if (err.message === 'Missing Configuration.') { + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration to locate the timeFrames is missing at the dataSet.') } + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> Time Frames Filters will be ignored -> Product = ' + pProduct.config.codeName) } + break + } + callBackFunction(err) + return + } + default: { + callBackFunction(err) + return + } + } + createFileCursors() + } + } + + function createFileCursors() { + try { + /* Now we will get the daily files */ + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] + + if (pDataset.config.validTimeFrames.includes(periodName) === false) { + continue + } + if (timeFrames !== undefined) { + if (timeFrames.includes(periodName) === false) { + continue + } + } + + let fileCursor = newFileCursor() + fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) + function onInitialized(err) { + try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + default: { + callBackWhenFileReceived(err) + return + } + } + fileCursors.set(periodTime, fileCursor) + expectedFiles = expectedFiles + fileCursor.getExpectedFiles() + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + } + } + } + loadThemAll() + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> createFileCursors -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + } + } + + function loadThemAll() { + try { for (i = 0; i < dailyFilePeriods.length; i++) { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = newFileCursor() - fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) - function onInitialized (err) { - try { - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - break - } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) - return - } - default: { - callBackWhenFileReceived(err) - return - } - } - fileCursors.set(periodTime, fileCursor) - expectedFiles = expectedFiles + fileCursor.getExpectedFiles() - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) - } + let fileCursor = fileCursors.get(periodTime) + if (fileCursor !== undefined) { + fileCursor.reload(onFileReceived) } } } - loadThemAll() - - function loadThemAll () { - try { - for (i = 0; i < dailyFilePeriods.length; i++) { - let periodTime = dailyFilePeriods[i][0] - let periodName = dailyFilePeriods[i][1] - - if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = fileCursors.get(periodTime) - fileCursor.reload(onFileReceived) - } - } - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> loadThemAll -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) - } - } + callBackWhenFileReceived(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> loadThemAll -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } @@ -168,7 +221,7 @@ function newDailyFiles () { } } - function onFileReceived (err) { + function onFileReceived(err) { try { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { @@ -195,7 +248,7 @@ function newDailyFiles () { } } - function getFileCursor (pPeriod) { + function getFileCursor(pPeriod) { try { return fileCursors.get(pPeriod) } catch (err) { @@ -203,13 +256,13 @@ function newDailyFiles () { } } - function setDatetime (pDatetime) { + function setDatetime(pDatetime) { try { filesLoaded = 0 expectedFiles = 0 fileCursors.forEach(setDatetimeToEach) - function setDatetimeToEach (fileCursor, key, map) { + function setDatetimeToEach(fileCursor, key, map) { fileCursor.setDatetime(pDatetime) expectedFiles = expectedFiles + fileCursor.getExpectedFiles() fileCursor.reload(onFileReceived) @@ -219,13 +272,13 @@ function newDailyFiles () { } } - function setTimeFrame (pTimeFrame, pDatetime) { + function setTimeFrame(pTimeFrame, pDatetime) { try { filesLoaded = 0 expectedFiles = 0 fileCursors.forEach(setTimeFrameToEach) - function setTimeFrameToEach (fileCursor, key, map) { + function setTimeFrameToEach(fileCursor, key, map) { fileCursor.setTimeFrame(pTimeFrame, pDatetime) expectedFiles = expectedFiles + fileCursor.getExpectedFiles() fileCursor.reload(onFileReceived) @@ -235,11 +288,11 @@ function newDailyFiles () { } } - function getExpectedFiles () { + function getExpectedFiles() { return expectedFiles } - function getFilesLoaded () { + function getFilesLoaded() { return filesLoaded } } diff --git a/src/FileCloud.js b/src/FileCloud.js index 69dfac0d8..cfc819f98 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -1,15 +1,12 @@ function newFileCloud () { const MODULE_NAME = 'File Cloud' - const INFO_LOG = false const ERROR_LOG = true const logger = newWebDebugLog() logger.fileName = MODULE_NAME /* - This is the module in the system that actually connects to the cloud storage and grabs from there the needed files. - */ let thisObject = { @@ -18,47 +15,64 @@ function newFileCloud () { } let fileStorage - return thisObject function initialize (pBot, pHost, pPort) { fileStorage = newFileStorage(pHost, pPort) } - function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } - const MAX_RETRIES = 3 + getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) - getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - - function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.config.codeName + '-' + pBot.config.codeName + '-' + pDataset.config.filePath + '-' + pDataset.config.fileName) } - let fileName let filePath - if (pDataRange === undefined) { + if (pDataRange === undefined && pTimeFrames === undefined) { fileName = pDataset.config.fileName filePath = pDataset.config.filePath } else { - if (pDataset.config.dataRange !== undefined) { - fileName = pDataset.config.dataRange.fileName - filePath = pDataset.config.dataRange.filePath - } else { - let customErr = { - result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, - message: 'Missing Configuration.' + if (pDataRange !== undefined) { + if (pDataset.config.dataRange !== undefined) { + fileName = pDataset.config.dataRange.fileName + filePath = pDataset.config.dataRange.filePath + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Missing Configuration.' + } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found at Dataset ' + pDataset.name) } + callBackFunction(customErr) + return + } + } + if (pTimeFrames !== undefined) { + if (pDataset.config.timeFrames !== undefined) { + if (pDataset.config.timeFrames.fileName !== '') { + fileName = pDataset.config.timeFrames.fileName + filePath = pDataset.config.timeFrames.filePath + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Configured to not Support This.' + } + callBackFunction(customErr) + return + } + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Missing Configuration.' + } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Time Frames configuration could not be found at Dataset ' + pDataset.name) } + callBackFunction(customErr) + return } - - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found. ') } - - callBackFunction(customErr) - return } } @@ -89,10 +103,10 @@ function newFileCloud () { if (pSession !== undefined) { let config - let sessionFolderName = pSession.id + let sessionFolderName = pSession.type.replace(' ', '-').replace(' ', '-') + '-' + pSession.id if (pSession.config !== undefined) { if (pSession.config.folderName !== undefined) { - sessionFolderName = pSession.config.folderName + '-' + pSession.id + sessionFolderName = pSession.type.replace(' ', '-').replace(' ', '-') + '-' + pSession.config.folderName } } filePath = filePath.replace('@Session', sessionFolderName) @@ -118,17 +132,10 @@ function newFileCloud () { fileName = fileName.replace('@Sequence', pSequence) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> filePath = ' + filePath) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> fileName = ' + fileName) } - fileStorage.getFileFromHost(filePath + '/' + fileName, onFileReceived) function onFileReceived (err, text, response) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } - let data if (err && err.result !== GLOBAL.DEFAULT_OK_RESPONSE.result) { @@ -137,9 +144,6 @@ function newFileCloud () { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, message: 'File does not exist.' } - - if (INFO_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } - callBackFunction(customErr) return } @@ -171,9 +175,6 @@ function newFileCloud () { } else { try { data = JSON.parse(text) - - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Data received is valid JSON.') } - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data) return } catch (err) { diff --git a/src/FileCursor.js b/src/FileCursor.js index 85f3bcc8c..4d46879ad 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -1,5 +1,5 @@ -function newFileCursor () { +function newFileCursor() { const MODULE_NAME = 'File Cursor' const INFO_LOG = false const ERROR_LOG = true @@ -43,7 +43,7 @@ function newFileCursor () { return thisObject - function finalize () { + function finalize() { try { eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) eventsServerClient.stopListening('Data Range Updated', eventSubscriptionIdDataRangeUpdated) @@ -75,7 +75,7 @@ function newFileCursor () { } } - function initialize ( + function initialize( pFileCloud, pDataMine, pBot, @@ -118,11 +118,11 @@ function newFileCursor () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) - function onResponseDataSet (message) { + function onResponseDataSet(message) { eventSubscriptionIdDatasetUpdated = message.eventSubscriptionId } - function onResponseDataRange (message) { + function onResponseDataRange(message) { eventSubscriptionIdDataRangeUpdated = message.eventSubscriptionId } } catch (err) { @@ -131,7 +131,7 @@ function newFileCursor () { } } - function updateDataRange (message) { + function updateDataRange(message) { try { if (finalized === true) { return } @@ -156,7 +156,7 @@ function newFileCursor () { } } - function updateFiles (message) { + function updateFiles(message) { try { if (finalized === true) { return } @@ -168,9 +168,9 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (finalized === true) { return } @@ -188,7 +188,7 @@ function newFileCursor () { } } - function setTimeFrame (pTimeFrame, pDatetime) { + function setTimeFrame(pTimeFrame, pDatetime) { try { if (finalized === true) { return } @@ -225,7 +225,7 @@ function newFileCursor () { enterSavingMode() } - function enterSavingMode () { + function enterSavingMode() { try { switch (timeFrame) { @@ -302,7 +302,7 @@ function newFileCursor () { } } - function exitSavingMode () { + function exitSavingMode() { try { switch (timeFrame) { @@ -383,7 +383,7 @@ function newFileCursor () { } } - function setDatetime (pDatetime) { + function setDatetime(pDatetime) { try { if (finalized === true) { return } @@ -398,7 +398,7 @@ function newFileCursor () { } } - function reload (callBackFunction) { + function reload(callBackFunction) { try { if (finalized === true) { return } @@ -411,7 +411,7 @@ function newFileCursor () { } } - function getFiles (callBackFunction) { + function getFiles(callBackFunction) { try { if (finalized === true) { return } @@ -422,7 +422,7 @@ function newFileCursor () { getNextFile() - function getNextFile () { + function getNextFile() { try { let targetDate = new Date(cursorDate) targetDate.setUTCDate(targetDate.getUTCDate() + j) @@ -442,7 +442,7 @@ function newFileCursor () { if (beginDateRange !== undefined) { let compareBeginDate = removeTime(beginDateRange) - if (compareTargetDate.valueOf() < compareBeginDate) { + if (compareTargetDate.valueOf() < compareBeginDate.valueOf()) { controlLoop() return } @@ -451,7 +451,7 @@ function newFileCursor () { if (endDateRange !== undefined) { let compareEndDate = removeTime(endDateRange) - if (compareTargetDate.valueOf() > compareEndDate) { + if (compareTargetDate.valueOf() >= compareEndDate.valueOf()) { controlLoop() return } @@ -468,7 +468,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) } else { controlLoop() } @@ -479,7 +479,7 @@ function newFileCursor () { } } - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> onFileReceived -> Entering function.') } @@ -524,7 +524,7 @@ function newFileCursor () { } } - function controlLoop () { + function controlLoop() { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> controlLoop -> Entering function.') } @@ -548,7 +548,7 @@ function newFileCursor () { } } - function collectGarbage (callBackFunction) { + function collectGarbage(callBackFunction) { try { if (finalized === true) { return } @@ -570,7 +570,7 @@ function newFileCursor () { } } - function getExpectedFiles () { + function getExpectedFiles() { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } diff --git a/src/FileSequence.js b/src/FileSequence.js index 187576503..67ac3b493 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -97,7 +97,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -142,7 +142,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -207,7 +207,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -243,7 +243,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/FileStorage.js b/src/FileStorage.js index 937b293c7..113e9f2aa 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -27,9 +27,6 @@ function newFileStorage(host, port) { getFileFromHost: getFileFromHost } - if (host === '' || host === undefined) { host = 'localhost' } - if (port === '' || port === undefined) { port = '34248' } - return thisObject async function getFileFromHost(filePath, callBackFunction, pathComplete) { @@ -41,8 +38,10 @@ function newFileStorage(host, port) { folder = 'Storage/' } let url = folder + filePath - if (host !== undefined) { + if (host !== undefined && port !== undefined) { url = 'http://' + host + ':' + port + '/' + folder + filePath + } else { + url = folder + filePath } callServer(undefined, url, (response, fileContent) => { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index b605e579b..cbb8028c8 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -1,5 +1,5 @@ -  -function newMarketFiles () { + +function newMarketFiles() { const MODULE_NAME = 'Market Files' const ERROR_LOG = true const logger = newWebDebugLog() @@ -17,6 +17,7 @@ function newMarketFiles () { let filesLoaded = 0 let filesNotLoaded = 0 + let filesExpected = 0 let fileCloud @@ -41,7 +42,7 @@ function newMarketFiles () { return thisObject - function finalize () { + function finalize() { try { for (let i = 0; i < eventSubscriptionIdDatasetUpdated.length; i++) { eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated[i]) @@ -69,7 +70,7 @@ function newMarketFiles () { } } - function initialize ( + function initialize( pDataMine, pBot, pSession, @@ -82,6 +83,8 @@ function newMarketFiles () { pEventsServerClient, callBackFunction) { try { + let timeFrames + exchange = pExchange market = pMarket dataMine = pDataMine @@ -102,16 +105,69 @@ function newMarketFiles () { return } - /* Now we will get the market files */ + getTimeFramesFile() - for (let i = 0; i < marketFilesPeriods.length; i++) { - let periodTime = marketFilesPeriods[i][0] - let periodName = marketFilesPeriods[i][1] + function getTimeFramesFile() { + /* First we will get the Data Range */ + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) - if (dataset.config.validTimeFrames.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + function onTimeFramesReceived(err, pFile) { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + timeFrames = pFile + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (err.message === 'File does not exist.') { + /* It is ok, we will deal with this later */ + break + } + if (err.message === 'Configured to not Support This.') { + /* It is ok, we will deal with this later */ + break + } + if (err.message === 'Missing Configuration.') { + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration to locate the timeFrames is missing at the dataSet.') } + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> Time Frames Filters will be ignored -> Product = ' + pProduct.config.codeName) } + break + } + callBackFunction(err) + return + } + default: { + callBackFunction(err) + return + } + } + getMarketFiles() + } + } - function onFileReceived (err, file) { + function getMarketFiles() { + /* Now we will get the market files */ + for (let i = 0; i < marketFilesPeriods.length; i++) { + let periodTime = marketFilesPeriods[i][0] + let periodName = marketFilesPeriods[i][1] + + if (dataset.config.validTimeFrames.includes(periodName) === false) { + filesNotLoaded++ + continue + } + if (timeFrames !== undefined) { + if (timeFrames.includes(periodName) === false) { + filesNotLoaded++ + continue + } + } + + filesExpected++ + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) + + function onFileReceived(err, file) { try { if (finalized === true) { return } if (err.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { @@ -128,7 +184,7 @@ function newMarketFiles () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) - function onResponse (message) { + function onResponse(message) { eventSubscriptionIdDatasetUpdated.push(message.eventSubscriptionId) } } @@ -138,6 +194,8 @@ function newMarketFiles () { } } } + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) } } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } @@ -145,31 +203,32 @@ function newMarketFiles () { } } - function updateFiles (message) { + function updateFiles(message) { try { if (finalized === true) { return } let updatedFiles = 0 - if (message.event === undefined) { return } - /* Now we will get the market files */ + /* Now we will get the market files */ for (let i = 0; i < marketFilesPeriods.length; i++) { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] if (dataset.config.validTimeFrames.includes(periodName) !== true) { continue } - if (message.event.timeFrames !== undefined) { - if (message.event.timeFrames.includes(periodName) !== true) { continue } + if (message.event !== undefined) { + if (message.event.timeFrames !== undefined) { + if (message.event.timeFrames.includes(periodName) !== true) { continue } + } } - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (finalized === true) { return } files.set(periodTime, file) updatedFiles++ - if (updatedFiles === marketFilesPeriods.length) { + if (updatedFiles === filesExpected) { thisObject.eventHandler.raiseEvent('Files Updated') } } catch (err) { @@ -182,19 +241,19 @@ function newMarketFiles () { } } - function getFile (pPeriod) { + function getFile(pPeriod) { return files.get(pPeriod) } - function getExpectedFiles () { + function getExpectedFiles() { return marketFilesPeriods.length } - function getFilesLoaded () { + function getFilesLoaded() { return filesLoaded } - function getFilesNotLoaded () { + function getFilesNotLoaded() { return filesNotLoaded } } diff --git a/src/SingleFile.js b/src/SingleFile.js index 9a6fdbae8..3103d1207 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -39,7 +39,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try {