From 6681c23a18e8573f41567b1efba5cbfa54e3c0e7 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 10:41:13 +0200 Subject: [PATCH 01/13] Preparing to read the Time Frames file --- src/DailyFiles.js | 108 ++++++++++++++++++++------------------------ src/FileCloud.js | 72 +++++++++++++---------------- src/FileCursor.js | 4 +- src/FileSequence.js | 8 ++-- src/MarketFiles.js | 4 +- src/SingleFile.js | 2 +- 6 files changed, 91 insertions(+), 107 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index d84a67d97..52305e64a 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -50,40 +50,37 @@ function newDailyFiles () { callBackWhenFileReceived = callBackFunction let exchange = pExchange + let beginDateRange + let endDateRange fileCloud = newFileCloud() fileCloud.initialize(pBot, pHost, pPort) - /* Some Validations */ + /* Config Validations */ if (pDataset.config.validTimeFrames === undefined) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Daily Files for bot ' + pBot.name) } if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) return } - /* First we will get the Data Range */ + getDataRangeFile() - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) - - function onDataRangeReceived (err, pFile) { - try { - let beginDateRange - let endDateRange + function getDataRangeFile () { + /* First we will get the Data Range */ + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived) + function onDataRangeReceived (err, pFile) { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { beginDateRange = pFile.begin endDateRange = pFile.end - break } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) return } - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { if (err.message === 'File does not exist.') { beginDateRange = new Date() @@ -97,7 +94,6 @@ function newDailyFiles () { endDateRange = new Date() break } - /* If none of the previous conditions are met, the we return the err to the caller. */ callBackFunction(err) return } @@ -106,60 +102,56 @@ function newDailyFiles () { return } } - /* Now we will get the daily files */ - for (i = 0; i < dailyFilePeriods.length; i++) { - let periodTime = dailyFilePeriods[i][0] - let periodName = dailyFilePeriods[i][1] + createFileCursors() + } + } - if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = newFileCursor() - fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) - function onInitialized (err) { - try { - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - break - } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) - return - } - default: { - callBackWhenFileReceived(err) - return - } + function createFileCursors () { + /* Now we will get the daily files */ + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] + + if (pDataset.config.validTimeFrames.includes(periodName) === true) { + let fileCursor = newFileCursor() + fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) + function onInitialized (err) { + try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + default: { + callBackWhenFileReceived(err) + return } - fileCursors.set(periodTime, fileCursor) - expectedFiles = expectedFiles + fileCursor.getExpectedFiles() - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + fileCursors.set(periodTime, fileCursor) + expectedFiles = expectedFiles + fileCursor.getExpectedFiles() + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } - loadThemAll() + } + loadThemAll() + } - function loadThemAll () { - try { - for (i = 0; i < dailyFilePeriods.length; i++) { - let periodTime = dailyFilePeriods[i][0] - let periodName = dailyFilePeriods[i][1] + function loadThemAll () { + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] - if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = fileCursors.get(periodTime) - fileCursor.reload(onFileReceived) - } - } - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> loadThemAll -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) - } + if (pDataset.config.validTimeFrames.includes(periodName) === true) { + let fileCursor = fileCursors.get(periodTime) + fileCursor.reload(onFileReceived) } - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { diff --git a/src/FileCloud.js b/src/FileCloud.js index 69dfac0d8..06bd8ac9d 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -1,15 +1,12 @@ function newFileCloud () { const MODULE_NAME = 'File Cloud' - const INFO_LOG = false const ERROR_LOG = true const logger = newWebDebugLog() logger.fileName = MODULE_NAME /* - This is the module in the system that actually connects to the cloud storage and grabs from there the needed files. - */ let thisObject = { @@ -18,47 +15,55 @@ function newFileCloud () { } let fileStorage - return thisObject function initialize (pBot, pHost, pPort) { fileStorage = newFileStorage(pHost, pPort) } - function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } - const MAX_RETRIES = 3 + getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) - getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - - function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.config.codeName + '-' + pBot.config.codeName + '-' + pDataset.config.filePath + '-' + pDataset.config.fileName) } - let fileName let filePath - if (pDataRange === undefined) { + if (pDataRange === undefined && pTimeFrames === undefined) { fileName = pDataset.config.fileName filePath = pDataset.config.filePath } else { - if (pDataset.config.dataRange !== undefined) { - fileName = pDataset.config.dataRange.fileName - filePath = pDataset.config.dataRange.filePath - } else { - let customErr = { - result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, - message: 'Missing Configuration.' + if (pDataRange !== undefined) { + if (pDataset.config.dataRange !== undefined) { + fileName = pDataset.config.dataRange.fileName + filePath = pDataset.config.dataRange.filePath + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Missing Configuration.' + } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found at Dataset ' + pDataset.name) } + callBackFunction(customErr) + return + } + } + if (pTimeFrames !== undefined) { + if (pTimeFrames.config.timeFrames !== undefined) { + fileName = pTimeFrames.config.timeFrames.fileName + filePath = pTimeFrames.config.timeFrames.filePath + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Missing Configuration.' + } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Time Frames configuration could not be found at Dataset ' + pDataset.name) } + callBackFunction(customErr) + return } - - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found. ') } - - callBackFunction(customErr) - return } } @@ -118,17 +123,10 @@ function newFileCloud () { fileName = fileName.replace('@Sequence', pSequence) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> filePath = ' + filePath) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> fileName = ' + fileName) } - fileStorage.getFileFromHost(filePath + '/' + fileName, onFileReceived) function onFileReceived (err, text, response) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } - let data if (err && err.result !== GLOBAL.DEFAULT_OK_RESPONSE.result) { @@ -137,9 +135,6 @@ function newFileCloud () { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, message: 'File does not exist.' } - - if (INFO_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } - callBackFunction(customErr) return } @@ -171,9 +166,6 @@ function newFileCloud () { } else { try { data = JSON.parse(text) - - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Data received is valid JSON.') } - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data) return } catch (err) { diff --git a/src/FileCursor.js b/src/FileCursor.js index 85f3bcc8c..134442d34 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -168,7 +168,7 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -468,7 +468,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 187576503..67ac3b493 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -97,7 +97,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -142,7 +142,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -207,7 +207,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -243,7 +243,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index b605e579b..31c8945cb 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -109,7 +109,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (dataset.config.validTimeFrames.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -161,7 +161,7 @@ function newMarketFiles () { if (message.event.timeFrames !== undefined) { if (message.event.timeFrames.includes(periodName) !== true) { continue } } - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index 9a6fdbae8..3103d1207 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -39,7 +39,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try { From 3b6ed944fa697342cd12bc3cbeb30c51a12f1628 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 11:31:51 +0200 Subject: [PATCH 02/13] Time Frames Files in Daily --- src/DailyFiles.js | 105 +++++++++++++++++++++++++++++++++------------- src/FileCloud.js | 6 +-- 2 files changed, 79 insertions(+), 32 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 52305e64a..6540e2fa1 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -52,6 +52,7 @@ function newDailyFiles () { let exchange = pExchange let beginDateRange let endDateRange + let timeFrames fileCloud = newFileCloud() fileCloud.initialize(pBot, pHost, pPort) @@ -102,6 +103,42 @@ function newDailyFiles () { return } } + getTimeFramesFile() + } + } + + function getTimeFramesFile () { + /* First we will get the Data Range */ + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) + + function onTimeFramesReceived (err, pFile) { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + timeFrames = pFile + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (err.message === 'File does not exist.') { + /* It is ok, we will deal with this later */ + break + } + if (err.message === 'Missing Configuration.') { + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration to locate the timeFrames is missing at the dataSet.') } + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> Time Frames Filters will be ignored -> Product = ' + pProduct.config.codeName) } + break + } + callBackFunction(err) + return + } + default: { + callBackFunction(err) + return + } + } createFileCursors() } } @@ -112,31 +149,34 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = newFileCursor() - fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) - function onInitialized (err) { - try { - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - break - } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) - return - } - default: { - callBackWhenFileReceived(err) - return - } + if (pDataset.config.validTimeFrames.includes(periodName) === false) { continue } + if (timeFrames !== undefined) { + if (timeFrames.includes(periodName) === false) { continue } + } + + let fileCursor = newFileCursor() + fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) + function onInitialized (err) { + try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + default: { + callBackWhenFileReceived(err) + return } - fileCursors.set(periodTime, fileCursor) - expectedFiles = expectedFiles + fileCursor.getExpectedFiles() - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + fileCursors.set(periodTime, fileCursor) + expectedFiles = expectedFiles + fileCursor.getExpectedFiles() + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } @@ -144,14 +184,21 @@ function newDailyFiles () { } function loadThemAll () { - for (i = 0; i < dailyFilePeriods.length; i++) { - let periodTime = dailyFilePeriods[i][0] - let periodName = dailyFilePeriods[i][1] + try { + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] - if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = fileCursors.get(periodTime) - fileCursor.reload(onFileReceived) + if (pDataset.config.validTimeFrames.includes(periodName) === true) { + let fileCursor = fileCursors.get(periodTime) + if (fileCursor !== undefined) { + fileCursor.reload(onFileReceived) + } + } } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> loadThemAll -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { diff --git a/src/FileCloud.js b/src/FileCloud.js index 06bd8ac9d..6cfb1772b 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -51,9 +51,9 @@ function newFileCloud () { } } if (pTimeFrames !== undefined) { - if (pTimeFrames.config.timeFrames !== undefined) { - fileName = pTimeFrames.config.timeFrames.fileName - filePath = pTimeFrames.config.timeFrames.filePath + if (pDataset.config.timeFrames !== undefined) { + fileName = pDataset.config.timeFrames.fileName + filePath = pDataset.config.timeFrames.filePath } else { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, From d4443b03505eb6593d86c417553627ea75876eae Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 12:06:52 +0200 Subject: [PATCH 03/13] Market Files reading Time Frames Files --- src/DailyFiles.js | 69 +++++++++++++++++++++++++++------------------- src/MarketFiles.js | 60 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 95 insertions(+), 34 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 6540e2fa1..63aee23be 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -144,43 +144,54 @@ function newDailyFiles () { } function createFileCursors () { + try { /* Now we will get the daily files */ - for (i = 0; i < dailyFilePeriods.length; i++) { - let periodTime = dailyFilePeriods[i][0] - let periodName = dailyFilePeriods[i][1] + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] - if (pDataset.config.validTimeFrames.includes(periodName) === false) { continue } - if (timeFrames !== undefined) { - if (timeFrames.includes(periodName) === false) { continue } - } + if (pDataset.config.validTimeFrames.includes(periodName) === false) { + filesLoaded++ + continue + } + if (timeFrames !== undefined) { + if (timeFrames.includes(periodName) === false) { + filesLoaded++ + continue + } + } - let fileCursor = newFileCursor() - fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) - function onInitialized (err) { - try { - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - break + let fileCursor = newFileCursor() + fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) + function onInitialized (err) { + try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + default: { + callBackWhenFileReceived(err) + return + } } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) - return - } - default: { - callBackWhenFileReceived(err) - return - } + fileCursors.set(periodTime, fileCursor) + expectedFiles = expectedFiles + fileCursor.getExpectedFiles() + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } - fileCursors.set(periodTime, fileCursor) - expectedFiles = expectedFiles + fileCursor.getExpectedFiles() - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } + loadThemAll() + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> createFileCursors -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } - loadThemAll() } function loadThemAll () { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 31c8945cb..64bd607af 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -82,6 +82,8 @@ function newMarketFiles () { pEventsServerClient, callBackFunction) { try { + let timeFrames + exchange = pExchange market = pMarket dataMine = pDataMine @@ -102,13 +104,61 @@ function newMarketFiles () { return } - /* Now we will get the market files */ + getTimeFramesFile() - for (let i = 0; i < marketFilesPeriods.length; i++) { - let periodTime = marketFilesPeriods[i][0] - let periodName = marketFilesPeriods[i][1] + function getTimeFramesFile () { + /* First we will get the Data Range */ + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) + + function onTimeFramesReceived (err, pFile) { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + timeFrames = pFile + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (err.message === 'File does not exist.') { + /* It is ok, we will deal with this later */ + break + } + if (err.message === 'Missing Configuration.') { + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration to locate the timeFrames is missing at the dataSet.') } + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> Time Frames Filters will be ignored -> Product = ' + pProduct.config.codeName) } + break + } + callBackFunction(err) + return + } + default: { + callBackFunction(err) + return + } + } + getMarketFiles() + } + } + + function getMarketFiles () { + /* Now we will get the market files */ + for (let i = 0; i < marketFilesPeriods.length; i++) { + let periodTime = marketFilesPeriods[i][0] + let periodName = marketFilesPeriods[i][1] + + if (dataset.config.validTimeFrames.includes(periodName) === false) { + filesNotLoaded++ + continue + } + if (timeFrames !== undefined) { + if (timeFrames.includes(periodName) === false) { + filesNotLoaded++ + continue + } + } - if (dataset.config.validTimeFrames.includes(periodName) === true) { fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { From b38d32b3ca4350b119e3436f790727e960d6696b Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 12:47:54 +0200 Subject: [PATCH 04/13] Configured to not Support Time Frames --- src/DailyFiles.js | 4 ++++ src/FileCloud.js | 13 +++++++++++-- src/MarketFiles.js | 4 ++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 63aee23be..47c6b1ce6 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -126,6 +126,10 @@ function newDailyFiles () { /* It is ok, we will deal with this later */ break } + if (err.message === 'Configured to not Support This.') { + /* It is ok, we will deal with this later */ + break + } if (err.message === 'Missing Configuration.') { if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration to locate the timeFrames is missing at the dataSet.') } if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> Time Frames Filters will be ignored -> Product = ' + pProduct.config.codeName) } diff --git a/src/FileCloud.js b/src/FileCloud.js index 6cfb1772b..157ed7ff7 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -52,8 +52,17 @@ function newFileCloud () { } if (pTimeFrames !== undefined) { if (pDataset.config.timeFrames !== undefined) { - fileName = pDataset.config.timeFrames.fileName - filePath = pDataset.config.timeFrames.filePath + if (pDataset.config.timeFrames.fileName !== '') { + fileName = pDataset.config.timeFrames.fileName + filePath = pDataset.config.timeFrames.filePath + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Configured to not Support This.' + } + callBackFunction(customErr) + return + } } else { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 64bd607af..15aff831b 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -125,6 +125,10 @@ function newMarketFiles () { /* It is ok, we will deal with this later */ break } + if (err.message === 'Configured to not Support This.') { + /* It is ok, we will deal with this later */ + break + } if (err.message === 'Missing Configuration.') { if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration to locate the timeFrames is missing at the dataSet.') } if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> Time Frames Filters will be ignored -> Product = ' + pProduct.config.codeName) } From a908ae5e4ab25e7f0c456a39a8c1d328c07a1f88 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 17:06:19 +0200 Subject: [PATCH 05/13] Update DailyFiles.js --- src/DailyFiles.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 47c6b1ce6..cf2a9638f 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -155,12 +155,10 @@ function newDailyFiles () { let periodName = dailyFilePeriods[i][1] if (pDataset.config.validTimeFrames.includes(periodName) === false) { - filesLoaded++ continue } if (timeFrames !== undefined) { if (timeFrames.includes(periodName) === false) { - filesLoaded++ continue } } From b60c4576cb6a4d39f12b67e6df77a2f626780988 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 17:31:42 +0200 Subject: [PATCH 06/13] Update MarketFiles.js --- src/MarketFiles.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 15aff831b..1a59120cd 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -17,6 +17,7 @@ function newMarketFiles () { let filesLoaded = 0 let filesNotLoaded = 0 + let filesExpected = 0 let fileCloud @@ -163,6 +164,7 @@ function newMarketFiles () { } } + filesExpected++ fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { @@ -223,7 +225,7 @@ function newMarketFiles () { files.set(periodTime, file) updatedFiles++ - if (updatedFiles === marketFilesPeriods.length) { + if (updatedFiles === filesExpected) { thisObject.eventHandler.raiseEvent('Files Updated') } } catch (err) { From daa66ddb0ec7e7d8231d6233e2ad470f54c48844 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 17:59:49 +0200 Subject: [PATCH 07/13] Update DailyFiles.js --- src/DailyFiles.js | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index cf2a9638f..decef072c 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -1,5 +1,5 @@ -  -function newDailyFiles () { + +function newDailyFiles() { const MODULE_NAME = 'Daily Files' const ERROR_LOG = true const logger = newWebDebugLog() @@ -26,7 +26,7 @@ function newDailyFiles () { return thisObject - function finalize () { + function finalize() { try { thisObject.eventHandler.finalize() thisObject.eventHandler = undefined @@ -37,7 +37,7 @@ function newDailyFiles () { fileCursors.forEach(finalizeEach) - function finalizeEach (item, key, mapObj) { + function finalizeEach(item, key, mapObj) { item.finalize() } } catch (err) { @@ -45,7 +45,7 @@ function newDailyFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, pEventsServerClient, callBackFunction) { + function initialize(pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, pEventsServerClient, callBackFunction) { try { callBackWhenFileReceived = callBackFunction @@ -67,11 +67,11 @@ function newDailyFiles () { getDataRangeFile() - function getDataRangeFile () { + function getDataRangeFile() { /* First we will get the Data Range */ fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived) - function onDataRangeReceived (err, pFile) { + function onDataRangeReceived(err, pFile) { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { beginDateRange = pFile.begin @@ -107,11 +107,11 @@ function newDailyFiles () { } } - function getTimeFramesFile () { + function getTimeFramesFile() { /* First we will get the Data Range */ fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) - function onTimeFramesReceived (err, pFile) { + function onTimeFramesReceived(err, pFile) { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { timeFrames = pFile @@ -147,9 +147,9 @@ function newDailyFiles () { } } - function createFileCursors () { + function createFileCursors() { try { - /* Now we will get the daily files */ + /* Now we will get the daily files */ for (i = 0; i < dailyFilePeriods.length; i++) { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] @@ -166,7 +166,7 @@ function newDailyFiles () { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) - function onInitialized (err) { + function onInitialized(err) { try { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { @@ -196,7 +196,7 @@ function newDailyFiles () { } } - function loadThemAll () { + function loadThemAll() { try { for (i = 0; i < dailyFilePeriods.length; i++) { let periodTime = dailyFilePeriods[i][0] @@ -220,7 +220,7 @@ function newDailyFiles () { } } - function onFileReceived (err) { + function onFileReceived(err) { try { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { @@ -247,7 +247,7 @@ function newDailyFiles () { } } - function getFileCursor (pPeriod) { + function getFileCursor(pPeriod) { try { return fileCursors.get(pPeriod) } catch (err) { @@ -255,13 +255,13 @@ function newDailyFiles () { } } - function setDatetime (pDatetime) { + function setDatetime(pDatetime) { try { filesLoaded = 0 expectedFiles = 0 fileCursors.forEach(setDatetimeToEach) - function setDatetimeToEach (fileCursor, key, map) { + function setDatetimeToEach(fileCursor, key, map) { fileCursor.setDatetime(pDatetime) expectedFiles = expectedFiles + fileCursor.getExpectedFiles() fileCursor.reload(onFileReceived) @@ -271,13 +271,13 @@ function newDailyFiles () { } } - function setTimeFrame (pTimeFrame, pDatetime) { + function setTimeFrame(pTimeFrame, pDatetime) { try { filesLoaded = 0 expectedFiles = 0 fileCursors.forEach(setTimeFrameToEach) - function setTimeFrameToEach (fileCursor, key, map) { + function setTimeFrameToEach(fileCursor, key, map) { fileCursor.setTimeFrame(pTimeFrame, pDatetime) expectedFiles = expectedFiles + fileCursor.getExpectedFiles() fileCursor.reload(onFileReceived) @@ -287,11 +287,11 @@ function newDailyFiles () { } } - function getExpectedFiles () { + function getExpectedFiles() { return expectedFiles } - function getFilesLoaded () { + function getFilesLoaded() { return filesLoaded } } From 280344605d8f102bae063100c1a07ff8a7ab97bf Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 28 Jul 2020 12:46:56 +0200 Subject: [PATCH 08/13] improvements --- src/DailyFiles.js | 1 + src/MarketFiles.js | 34 ++++++++++++++++++---------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index decef072c..f28017eb2 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -209,6 +209,7 @@ function newDailyFiles() { } } } + callBackWhenFileReceived(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> loadThemAll -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 1a59120cd..e399df108 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -1,5 +1,5 @@ -  -function newMarketFiles () { + +function newMarketFiles() { const MODULE_NAME = 'Market Files' const ERROR_LOG = true const logger = newWebDebugLog() @@ -42,7 +42,7 @@ function newMarketFiles () { return thisObject - function finalize () { + function finalize() { try { for (let i = 0; i < eventSubscriptionIdDatasetUpdated.length; i++) { eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated[i]) @@ -70,7 +70,7 @@ function newMarketFiles () { } } - function initialize ( + function initialize( pDataMine, pBot, pSession, @@ -107,11 +107,11 @@ function newMarketFiles () { getTimeFramesFile() - function getTimeFramesFile () { + function getTimeFramesFile() { /* First we will get the Data Range */ fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) - function onTimeFramesReceived (err, pFile) { + function onTimeFramesReceived(err, pFile) { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { timeFrames = pFile @@ -147,7 +147,7 @@ function newMarketFiles () { } } - function getMarketFiles () { + function getMarketFiles() { /* Now we will get the market files */ for (let i = 0; i < marketFilesPeriods.length; i++) { let periodTime = marketFilesPeriods[i][0] @@ -167,7 +167,7 @@ function newMarketFiles () { filesExpected++ fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (finalized === true) { return } if (err.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { @@ -184,7 +184,7 @@ function newMarketFiles () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) - function onResponse (message) { + function onResponse(message) { eventSubscriptionIdDatasetUpdated.push(message.eventSubscriptionId) } } @@ -194,6 +194,8 @@ function newMarketFiles () { } } } + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) } } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } @@ -201,13 +203,13 @@ function newMarketFiles () { } } - function updateFiles (message) { + function updateFiles(message) { try { if (finalized === true) { return } let updatedFiles = 0 if (message.event === undefined) { return } - /* Now we will get the market files */ + /* Now we will get the market files */ for (let i = 0; i < marketFilesPeriods.length; i++) { let periodTime = marketFilesPeriods[i][0] @@ -219,7 +221,7 @@ function newMarketFiles () { } fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (finalized === true) { return } files.set(periodTime, file) @@ -238,19 +240,19 @@ function newMarketFiles () { } } - function getFile (pPeriod) { + function getFile(pPeriod) { return files.get(pPeriod) } - function getExpectedFiles () { + function getExpectedFiles() { return marketFilesPeriods.length } - function getFilesLoaded () { + function getFilesLoaded() { return filesLoaded } - function getFilesNotLoaded () { + function getFilesNotLoaded() { return filesNotLoaded } } From df7bea0836b760d1ae806554e1cae4909ab389d1 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 28 Jul 2020 17:27:48 +0200 Subject: [PATCH 09/13] Bug Dataset Updated Fixed --- src/FileCursor.js | 38 +++++++++++++++++++------------------- src/MarketFiles.js | 7 ++++--- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index 134442d34..9b11c6b6c 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -1,5 +1,5 @@ -function newFileCursor () { +function newFileCursor() { const MODULE_NAME = 'File Cursor' const INFO_LOG = false const ERROR_LOG = true @@ -43,7 +43,7 @@ function newFileCursor () { return thisObject - function finalize () { + function finalize() { try { eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) eventsServerClient.stopListening('Data Range Updated', eventSubscriptionIdDataRangeUpdated) @@ -75,7 +75,7 @@ function newFileCursor () { } } - function initialize ( + function initialize( pFileCloud, pDataMine, pBot, @@ -118,11 +118,11 @@ function newFileCursor () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) - function onResponseDataSet (message) { + function onResponseDataSet(message) { eventSubscriptionIdDatasetUpdated = message.eventSubscriptionId } - function onResponseDataRange (message) { + function onResponseDataRange(message) { eventSubscriptionIdDataRangeUpdated = message.eventSubscriptionId } } catch (err) { @@ -131,7 +131,7 @@ function newFileCursor () { } } - function updateDataRange (message) { + function updateDataRange(message) { try { if (finalized === true) { return } @@ -156,7 +156,7 @@ function newFileCursor () { } } - function updateFiles (message) { + function updateFiles(message) { try { if (finalized === true) { return } @@ -170,7 +170,7 @@ function newFileCursor () { fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (finalized === true) { return } @@ -188,7 +188,7 @@ function newFileCursor () { } } - function setTimeFrame (pTimeFrame, pDatetime) { + function setTimeFrame(pTimeFrame, pDatetime) { try { if (finalized === true) { return } @@ -225,7 +225,7 @@ function newFileCursor () { enterSavingMode() } - function enterSavingMode () { + function enterSavingMode() { try { switch (timeFrame) { @@ -302,7 +302,7 @@ function newFileCursor () { } } - function exitSavingMode () { + function exitSavingMode() { try { switch (timeFrame) { @@ -383,7 +383,7 @@ function newFileCursor () { } } - function setDatetime (pDatetime) { + function setDatetime(pDatetime) { try { if (finalized === true) { return } @@ -398,7 +398,7 @@ function newFileCursor () { } } - function reload (callBackFunction) { + function reload(callBackFunction) { try { if (finalized === true) { return } @@ -411,7 +411,7 @@ function newFileCursor () { } } - function getFiles (callBackFunction) { + function getFiles(callBackFunction) { try { if (finalized === true) { return } @@ -422,7 +422,7 @@ function newFileCursor () { getNextFile() - function getNextFile () { + function getNextFile() { try { let targetDate = new Date(cursorDate) targetDate.setUTCDate(targetDate.getUTCDate() + j) @@ -479,7 +479,7 @@ function newFileCursor () { } } - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> onFileReceived -> Entering function.') } @@ -524,7 +524,7 @@ function newFileCursor () { } } - function controlLoop () { + function controlLoop() { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> controlLoop -> Entering function.') } @@ -548,7 +548,7 @@ function newFileCursor () { } } - function collectGarbage (callBackFunction) { + function collectGarbage(callBackFunction) { try { if (finalized === true) { return } @@ -570,7 +570,7 @@ function newFileCursor () { } } - function getExpectedFiles () { + function getExpectedFiles() { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } diff --git a/src/MarketFiles.js b/src/MarketFiles.js index e399df108..cbb8028c8 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -207,7 +207,6 @@ function newMarketFiles() { try { if (finalized === true) { return } let updatedFiles = 0 - if (message.event === undefined) { return } /* Now we will get the market files */ @@ -216,8 +215,10 @@ function newMarketFiles() { let periodName = marketFilesPeriods[i][1] if (dataset.config.validTimeFrames.includes(periodName) !== true) { continue } - if (message.event.timeFrames !== undefined) { - if (message.event.timeFrames.includes(periodName) !== true) { continue } + if (message.event !== undefined) { + if (message.event.timeFrames !== undefined) { + if (message.event.timeFrames.includes(periodName) !== true) { continue } + } } fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) From 5deecf8055200535dde45d1b570f45af88becc81 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 14 Aug 2020 13:16:27 +0200 Subject: [PATCH 10/13] Possible bug fixed --- src/FileCursor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index 9b11c6b6c..12d5ab981 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -442,7 +442,7 @@ function newFileCursor() { if (beginDateRange !== undefined) { let compareBeginDate = removeTime(beginDateRange) - if (compareTargetDate.valueOf() < compareBeginDate) { + if (compareTargetDate.valueOf() < compareBeginDate.valueOf()) { controlLoop() return } @@ -451,7 +451,7 @@ function newFileCursor() { if (endDateRange !== undefined) { let compareEndDate = removeTime(endDateRange) - if (compareTargetDate.valueOf() > compareEndDate) { + if (compareTargetDate.valueOf() > compareEndDate.valueOf()) { controlLoop() return } From 13403ad0eed598760cdbc3e8e3f5e5af2c3b6850 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 15 Aug 2020 11:14:10 +0200 Subject: [PATCH 11/13] Update FileCursor.js --- src/FileCursor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index 12d5ab981..4d46879ad 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -451,7 +451,7 @@ function newFileCursor() { if (endDateRange !== undefined) { let compareEndDate = removeTime(endDateRange) - if (compareTargetDate.valueOf() > compareEndDate.valueOf()) { + if (compareTargetDate.valueOf() >= compareEndDate.valueOf()) { controlLoop() return } From de918745d6376f94350f2a884b8e767530c241e0 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 16 Aug 2020 16:21:01 +0200 Subject: [PATCH 12/13] Update FileCloud.js --- src/FileCloud.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index 157ed7ff7..cfc819f98 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -103,10 +103,10 @@ function newFileCloud () { if (pSession !== undefined) { let config - let sessionFolderName = pSession.id + let sessionFolderName = pSession.type.replace(' ', '-').replace(' ', '-') + '-' + pSession.id if (pSession.config !== undefined) { if (pSession.config.folderName !== undefined) { - sessionFolderName = pSession.config.folderName + '-' + pSession.id + sessionFolderName = pSession.type.replace(' ', '-').replace(' ', '-') + '-' + pSession.config.folderName } } filePath = filePath.replace('@Session', sessionFolderName) From 750a3f91548807eb00535a2a4591d543e72519b2 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 23 Aug 2020 09:39:55 +0200 Subject: [PATCH 13/13] Update FileStorage.js --- src/FileStorage.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/FileStorage.js b/src/FileStorage.js index 937b293c7..113e9f2aa 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -27,9 +27,6 @@ function newFileStorage(host, port) { getFileFromHost: getFileFromHost } - if (host === '' || host === undefined) { host = 'localhost' } - if (port === '' || port === undefined) { port = '34248' } - return thisObject async function getFileFromHost(filePath, callBackFunction, pathComplete) { @@ -41,8 +38,10 @@ function newFileStorage(host, port) { folder = 'Storage/' } let url = folder + filePath - if (host !== undefined) { + if (host !== undefined && port !== undefined) { url = 'http://' + host + ':' + port + '/' + folder + filePath + } else { + url = folder + filePath } callServer(undefined, url, (response, fileContent) => {