Preparing to read the Time Frames file

This commit is contained in:
Luis Fernando Molina committed 2020-07-27 10:41:13 +02:00
1 parent 41d978212c
commit 6681c23a18
6 files changed
+91 -107

No files matched your search

+50 -58
View File
@@ -50,40 +50,37 @@ function newDailyFiles () {
callBackWhenFileReceived = callBackFunction
let exchange = pExchange
let beginDateRange
let endDateRange
fileCloud = newFileCloud()
fileCloud.initialize(pBot, pHost, pPort)
/* Some Validations */
/* Config Validations */
if (pDataset.config.validTimeFrames === undefined) {
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) }
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Daily Files for bot ' + pBot.name) }
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') }
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
return
}
/* First we will get the Data Range */
getDataRangeFile()
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived)
function onDataRangeReceived (err, pFile) {
try {
let beginDateRange
let endDateRange
function getDataRangeFile () {
/* First we will get the Data Range */
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived)
function onDataRangeReceived (err, pFile) {
switch (err.result) {
case GLOBAL.DEFAULT_OK_RESPONSE.result: {
beginDateRange = pFile.begin
endDateRange = pFile.end
break
}
case GLOBAL.DEFAULT_FAIL_RESPONSE.result: {
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
return
}
case GLOBAL.CUSTOM_FAIL_RESPONSE.result: {
if (err.message === 'File does not exist.') {
beginDateRange = new Date()
@@ -97,7 +94,6 @@ function newDailyFiles () {
endDateRange = new Date()
break
}
/* If none of the previous conditions are met, the we return the err to the caller. */
callBackFunction(err)
return
}
@@ -106,60 +102,56 @@ function newDailyFiles () {
return
}
}
/* Now we will get the daily files */
for (i = 0; i < dailyFilePeriods.length; i++) {
let periodTime = dailyFilePeriods[i][0]
let periodName = dailyFilePeriods[i][1]
createFileCursors()
}
}
if (pDataset.config.validTimeFrames.includes(periodName) === true) {
let fileCursor = newFileCursor()
fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s
fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized)
function onInitialized (err) {
try {
switch (err.result) {
case GLOBAL.DEFAULT_OK_RESPONSE.result: {
break
}
case GLOBAL.DEFAULT_FAIL_RESPONSE.result: {
callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE)
return
}
default: {
callBackWhenFileReceived(err)
return
}
function createFileCursors () {
/* Now we will get the daily files */
for (i = 0; i < dailyFilePeriods.length; i++) {
let periodTime = dailyFilePeriods[i][0]
let periodName = dailyFilePeriods[i][1]
if (pDataset.config.validTimeFrames.includes(periodName) === true) {
let fileCursor = newFileCursor()
fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s
fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized)
function onInitialized (err) {
try {
switch (err.result) {
case GLOBAL.DEFAULT_OK_RESPONSE.result: {
break
}
case GLOBAL.DEFAULT_FAIL_RESPONSE.result: {
callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE)
return
}
default: {
callBackWhenFileReceived(err)
return
}
fileCursors.set(periodTime, fileCursor)
expectedFiles = expectedFiles + fileCursor.getExpectedFiles()
} catch (err) {
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) }
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
}
fileCursors.set(periodTime, fileCursor)
expectedFiles = expectedFiles + fileCursor.getExpectedFiles()
} catch (err) {
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) }
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
}
}
}
loadThemAll()
}
loadThemAll()
}
function loadThemAll () {
try {
for (i = 0; i < dailyFilePeriods.length; i++) {
let periodTime = dailyFilePeriods[i][0]
let periodName = dailyFilePeriods[i][1]
function loadThemAll () {
for (i = 0; i < dailyFilePeriods.length; i++) {
let periodTime = dailyFilePeriods[i][0]
let periodName = dailyFilePeriods[i][1]
if (pDataset.config.validTimeFrames.includes(periodName) === true) {
let fileCursor = fileCursors.get(periodTime)
fileCursor.reload(onFileReceived)
}
}
} catch (err) {
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> loadThemAll -> err = ' + err.stack) }
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
}
if (pDataset.config.validTimeFrames.includes(periodName) === true) {
let fileCursor = fileCursors.get(periodTime)
fileCursor.reload(onFileReceived)
}
} catch (err) {
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) }
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
}
}
} catch (err) {
+32 -40
View File
@@ -1,15 +1,12 @@
function newFileCloud () {
const MODULE_NAME = 'File Cloud'
const INFO_LOG = false
const ERROR_LOG = true
const logger = newWebDebugLog()
logger.fileName = MODULE_NAME
/*
This is the module in the system that actually connects to the cloud storage and grabs from there the needed files.
*/
let thisObject = {
@@ -18,47 +15,55 @@ function newFileCloud () {
}
let fileStorage
return thisObject
function initialize (pBot, pHost, pPort) {
fileStorage = newFileStorage(pHost, pPort)
}
function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) {
function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) {
try {
if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') }
const MAX_RETRIES = 3
getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction)
getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction)
function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) {
function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) {
try {
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') }
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.config.codeName + '-' + pBot.config.codeName + '-' + pDataset.config.filePath + '-' + pDataset.config.fileName) }
let fileName
let filePath
if (pDataRange === undefined) {
if (pDataRange === undefined && pTimeFrames === undefined) {
fileName = pDataset.config.fileName
filePath = pDataset.config.filePath
} else {
if (pDataset.config.dataRange !== undefined) {
fileName = pDataset.config.dataRange.fileName
filePath = pDataset.config.dataRange.filePath
} else {
let customErr = {
result: GLOBAL.CUSTOM_FAIL_RESPONSE.result,
message: 'Missing Configuration.'
if (pDataRange !== undefined) {
if (pDataset.config.dataRange !== undefined) {
fileName = pDataset.config.dataRange.fileName
filePath = pDataset.config.dataRange.filePath
} else {
let customErr = {
result: GLOBAL.CUSTOM_FAIL_RESPONSE.result,
message: 'Missing Configuration.'
}
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) }
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found at Dataset ' + pDataset.name) }
callBackFunction(customErr)
return
}
}
if (pTimeFrames !== undefined) {
if (pTimeFrames.config.timeFrames !== undefined) {
fileName = pTimeFrames.config.timeFrames.fileName
filePath = pTimeFrames.config.timeFrames.filePath
} else {
let customErr = {
result: GLOBAL.CUSTOM_FAIL_RESPONSE.result,
message: 'Missing Configuration.'
}
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) }
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Time Frames configuration could not be found at Dataset ' + pDataset.name) }
callBackFunction(customErr)
return
}
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) }
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found. ') }
callBackFunction(customErr)
return
}
}
@@ -118,17 +123,10 @@ function newFileCloud () {
fileName = fileName.replace('@Sequence', pSequence)
}
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> filePath = ' + filePath) }
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> fileName = ' + fileName) }
fileStorage.getFileFromHost(filePath + '/' + fileName, onFileReceived)
function onFileReceived (err, text, response) {
try {
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function.') }
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) }
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) }
let data
if (err && err.result !== GLOBAL.DEFAULT_OK_RESPONSE.result) {
@@ -137,9 +135,6 @@ function newFileCloud () {
result: GLOBAL.CUSTOM_FAIL_RESPONSE.result,
message: 'File does not exist.'
}
if (INFO_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) }
callBackFunction(customErr)
return
}
@@ -171,9 +166,6 @@ function newFileCloud () {
} else {
try {
data = JSON.parse(text)
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Data received is valid JSON.') }
callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data)
return
} catch (err) {
+2 -2
View File
@@ -168,7 +168,7 @@ function newFileCursor () {
dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2)
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
@@ -468,7 +468,7 @@ function newFileCursor () {
if (thisObject.files.get(dateString) === undefined) {
// We dont reload files we already have.
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived)
} else {
controlLoop()
}
+4 -4
View File
@@ -97,7 +97,7 @@ function newFileSequence () {
/* First we will get the sequence max number */
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived)
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived)
function onSequenceFileReceived (err, file) {
try {
@@ -142,7 +142,7 @@ function newFileSequence () {
/* Now we will get the sequence of files */
for (let i = 0; i <= maxSequence; i++) {
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
@@ -207,7 +207,7 @@ function newFileSequence () {
let currentMaxSequence = maxSequence
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived)
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived)
function onSequenceFileReceived (err, sequenceFile) {
try {
@@ -243,7 +243,7 @@ function newFileSequence () {
/* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */
for (let i = currentMaxSequence; i <= maxSequence; i++) {
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
+2 -2
View File
@@ -109,7 +109,7 @@ function newMarketFiles () {
let periodName = marketFilesPeriods[i][1]
if (dataset.config.validTimeFrames.includes(periodName) === true) {
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
@@ -161,7 +161,7 @@ function newMarketFiles () {
if (message.event.timeFrames !== undefined) {
if (message.event.timeFrames.includes(periodName) !== true) { continue }
}
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
+1 -1
View File
@@ -39,7 +39,7 @@ function newSingleFile () {
/* Now we will get the file */
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived)
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, undefined, onFileReceived)
function onFileReceived (err, pFile) {
try {