Preparing to read the Time Frames file
This commit is contained in:
1 parent
41d978212c
commit
6681c23a18
6 files changed
+91
-107
No files matched your search
+50
-58
@@ -50,40 +50,37 @@ function newDailyFiles () {
|
||||
callBackWhenFileReceived = callBackFunction
|
||||
|
||||
let exchange = pExchange
|
||||
let beginDateRange
|
||||
let endDateRange
|
||||
|
||||
fileCloud = newFileCloud()
|
||||
fileCloud.initialize(pBot, pHost, pPort)
|
||||
|
||||
/* Some Validations */
|
||||
/* Config Validations */
|
||||
if (pDataset.config.validTimeFrames === undefined) {
|
||||
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) }
|
||||
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Daily Files for bot ' + pBot.name) }
|
||||
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') }
|
||||
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
|
||||
return
|
||||
}
|
||||
|
||||
/* First we will get the Data Range */
|
||||
getDataRangeFile()
|
||||
|
||||
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived)
|
||||
|
||||
function onDataRangeReceived (err, pFile) {
|
||||
try {
|
||||
let beginDateRange
|
||||
let endDateRange
|
||||
function getDataRangeFile () {
|
||||
/* First we will get the Data Range */
|
||||
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived)
|
||||
|
||||
function onDataRangeReceived (err, pFile) {
|
||||
switch (err.result) {
|
||||
case GLOBAL.DEFAULT_OK_RESPONSE.result: {
|
||||
beginDateRange = pFile.begin
|
||||
endDateRange = pFile.end
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
case GLOBAL.DEFAULT_FAIL_RESPONSE.result: {
|
||||
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
|
||||
return
|
||||
}
|
||||
|
||||
case GLOBAL.CUSTOM_FAIL_RESPONSE.result: {
|
||||
if (err.message === 'File does not exist.') {
|
||||
beginDateRange = new Date()
|
||||
@@ -97,7 +94,6 @@ function newDailyFiles () {
|
||||
endDateRange = new Date()
|
||||
break
|
||||
}
|
||||
/* If none of the previous conditions are met, the we return the err to the caller. */
|
||||
callBackFunction(err)
|
||||
return
|
||||
}
|
||||
@@ -106,60 +102,56 @@ function newDailyFiles () {
|
||||
return
|
||||
}
|
||||
}
|
||||
/* Now we will get the daily files */
|
||||
for (i = 0; i < dailyFilePeriods.length; i++) {
|
||||
let periodTime = dailyFilePeriods[i][0]
|
||||
let periodName = dailyFilePeriods[i][1]
|
||||
createFileCursors()
|
||||
}
|
||||
}
|
||||
|
||||
if (pDataset.config.validTimeFrames.includes(periodName) === true) {
|
||||
let fileCursor = newFileCursor()
|
||||
fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s
|
||||
fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized)
|
||||
function onInitialized (err) {
|
||||
try {
|
||||
switch (err.result) {
|
||||
case GLOBAL.DEFAULT_OK_RESPONSE.result: {
|
||||
break
|
||||
}
|
||||
case GLOBAL.DEFAULT_FAIL_RESPONSE.result: {
|
||||
callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE)
|
||||
return
|
||||
}
|
||||
default: {
|
||||
callBackWhenFileReceived(err)
|
||||
return
|
||||
}
|
||||
function createFileCursors () {
|
||||
/* Now we will get the daily files */
|
||||
for (i = 0; i < dailyFilePeriods.length; i++) {
|
||||
let periodTime = dailyFilePeriods[i][0]
|
||||
let periodName = dailyFilePeriods[i][1]
|
||||
|
||||
if (pDataset.config.validTimeFrames.includes(periodName) === true) {
|
||||
let fileCursor = newFileCursor()
|
||||
fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s
|
||||
fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized)
|
||||
function onInitialized (err) {
|
||||
try {
|
||||
switch (err.result) {
|
||||
case GLOBAL.DEFAULT_OK_RESPONSE.result: {
|
||||
break
|
||||
}
|
||||
case GLOBAL.DEFAULT_FAIL_RESPONSE.result: {
|
||||
callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE)
|
||||
return
|
||||
}
|
||||
default: {
|
||||
callBackWhenFileReceived(err)
|
||||
return
|
||||
}
|
||||
fileCursors.set(periodTime, fileCursor)
|
||||
expectedFiles = expectedFiles + fileCursor.getExpectedFiles()
|
||||
} catch (err) {
|
||||
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) }
|
||||
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
|
||||
}
|
||||
fileCursors.set(periodTime, fileCursor)
|
||||
expectedFiles = expectedFiles + fileCursor.getExpectedFiles()
|
||||
} catch (err) {
|
||||
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) }
|
||||
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
|
||||
}
|
||||
}
|
||||
}
|
||||
loadThemAll()
|
||||
}
|
||||
loadThemAll()
|
||||
}
|
||||
|
||||
function loadThemAll () {
|
||||
try {
|
||||
for (i = 0; i < dailyFilePeriods.length; i++) {
|
||||
let periodTime = dailyFilePeriods[i][0]
|
||||
let periodName = dailyFilePeriods[i][1]
|
||||
function loadThemAll () {
|
||||
for (i = 0; i < dailyFilePeriods.length; i++) {
|
||||
let periodTime = dailyFilePeriods[i][0]
|
||||
let periodName = dailyFilePeriods[i][1]
|
||||
|
||||
if (pDataset.config.validTimeFrames.includes(periodName) === true) {
|
||||
let fileCursor = fileCursors.get(periodTime)
|
||||
fileCursor.reload(onFileReceived)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> loadThemAll -> err = ' + err.stack) }
|
||||
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
|
||||
}
|
||||
if (pDataset.config.validTimeFrames.includes(periodName) === true) {
|
||||
let fileCursor = fileCursors.get(periodTime)
|
||||
fileCursor.reload(onFileReceived)
|
||||
}
|
||||
} catch (err) {
|
||||
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) }
|
||||
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
|
||||
+32
-40
@@ -1,15 +1,12 @@
|
||||
|
||||
function newFileCloud () {
|
||||
const MODULE_NAME = 'File Cloud'
|
||||
const INFO_LOG = false
|
||||
const ERROR_LOG = true
|
||||
const logger = newWebDebugLog()
|
||||
logger.fileName = MODULE_NAME
|
||||
|
||||
/*
|
||||
|
||||
This is the module in the system that actually connects to the cloud storage and grabs from there the needed files.
|
||||
|
||||
*/
|
||||
|
||||
let thisObject = {
|
||||
@@ -18,47 +15,55 @@ function newFileCloud () {
|
||||
}
|
||||
|
||||
let fileStorage
|
||||
|
||||
return thisObject
|
||||
|
||||
function initialize (pBot, pHost, pPort) {
|
||||
fileStorage = newFileStorage(pHost, pPort)
|
||||
}
|
||||
|
||||
function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) {
|
||||
function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) {
|
||||
try {
|
||||
if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') }
|
||||
|
||||
const MAX_RETRIES = 3
|
||||
getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction)
|
||||
|
||||
getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction)
|
||||
|
||||
function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) {
|
||||
function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) {
|
||||
try {
|
||||
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') }
|
||||
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.config.codeName + '-' + pBot.config.codeName + '-' + pDataset.config.filePath + '-' + pDataset.config.fileName) }
|
||||
|
||||
let fileName
|
||||
let filePath
|
||||
|
||||
if (pDataRange === undefined) {
|
||||
if (pDataRange === undefined && pTimeFrames === undefined) {
|
||||
fileName = pDataset.config.fileName
|
||||
filePath = pDataset.config.filePath
|
||||
} else {
|
||||
if (pDataset.config.dataRange !== undefined) {
|
||||
fileName = pDataset.config.dataRange.fileName
|
||||
filePath = pDataset.config.dataRange.filePath
|
||||
} else {
|
||||
let customErr = {
|
||||
result: GLOBAL.CUSTOM_FAIL_RESPONSE.result,
|
||||
message: 'Missing Configuration.'
|
||||
if (pDataRange !== undefined) {
|
||||
if (pDataset.config.dataRange !== undefined) {
|
||||
fileName = pDataset.config.dataRange.fileName
|
||||
filePath = pDataset.config.dataRange.filePath
|
||||
} else {
|
||||
let customErr = {
|
||||
result: GLOBAL.CUSTOM_FAIL_RESPONSE.result,
|
||||
message: 'Missing Configuration.'
|
||||
}
|
||||
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) }
|
||||
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found at Dataset ' + pDataset.name) }
|
||||
callBackFunction(customErr)
|
||||
return
|
||||
}
|
||||
}
|
||||
if (pTimeFrames !== undefined) {
|
||||
if (pTimeFrames.config.timeFrames !== undefined) {
|
||||
fileName = pTimeFrames.config.timeFrames.fileName
|
||||
filePath = pTimeFrames.config.timeFrames.filePath
|
||||
} else {
|
||||
let customErr = {
|
||||
result: GLOBAL.CUSTOM_FAIL_RESPONSE.result,
|
||||
message: 'Missing Configuration.'
|
||||
}
|
||||
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) }
|
||||
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Time Frames configuration could not be found at Dataset ' + pDataset.name) }
|
||||
callBackFunction(customErr)
|
||||
return
|
||||
}
|
||||
|
||||
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) }
|
||||
if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found. ') }
|
||||
|
||||
callBackFunction(customErr)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,17 +123,10 @@ function newFileCloud () {
|
||||
fileName = fileName.replace('@Sequence', pSequence)
|
||||
}
|
||||
|
||||
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> filePath = ' + filePath) }
|
||||
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> fileName = ' + fileName) }
|
||||
|
||||
fileStorage.getFileFromHost(filePath + '/' + fileName, onFileReceived)
|
||||
|
||||
function onFileReceived (err, text, response) {
|
||||
try {
|
||||
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function.') }
|
||||
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) }
|
||||
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) }
|
||||
|
||||
let data
|
||||
|
||||
if (err && err.result !== GLOBAL.DEFAULT_OK_RESPONSE.result) {
|
||||
@@ -137,9 +135,6 @@ function newFileCloud () {
|
||||
result: GLOBAL.CUSTOM_FAIL_RESPONSE.result,
|
||||
message: 'File does not exist.'
|
||||
}
|
||||
|
||||
if (INFO_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) }
|
||||
|
||||
callBackFunction(customErr)
|
||||
return
|
||||
}
|
||||
@@ -171,9 +166,6 @@ function newFileCloud () {
|
||||
} else {
|
||||
try {
|
||||
data = JSON.parse(text)
|
||||
|
||||
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Data received is valid JSON.') }
|
||||
|
||||
callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data)
|
||||
return
|
||||
} catch (err) {
|
||||
|
||||
+2
-2
@@ -168,7 +168,7 @@ function newFileCursor () {
|
||||
|
||||
dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2)
|
||||
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived)
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived)
|
||||
|
||||
function onFileReceived (err, file) {
|
||||
try {
|
||||
@@ -468,7 +468,7 @@ function newFileCursor () {
|
||||
if (thisObject.files.get(dateString) === undefined) {
|
||||
// We dont reload files we already have.
|
||||
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived)
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived)
|
||||
} else {
|
||||
controlLoop()
|
||||
}
|
||||
|
||||
+4
-4
@@ -97,7 +97,7 @@ function newFileSequence () {
|
||||
|
||||
/* First we will get the sequence max number */
|
||||
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived)
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived)
|
||||
|
||||
function onSequenceFileReceived (err, file) {
|
||||
try {
|
||||
@@ -142,7 +142,7 @@ function newFileSequence () {
|
||||
/* Now we will get the sequence of files */
|
||||
|
||||
for (let i = 0; i <= maxSequence; i++) {
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived)
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived)
|
||||
|
||||
function onFileReceived (err, file) {
|
||||
try {
|
||||
@@ -207,7 +207,7 @@ function newFileSequence () {
|
||||
|
||||
let currentMaxSequence = maxSequence
|
||||
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived)
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived)
|
||||
|
||||
function onSequenceFileReceived (err, sequenceFile) {
|
||||
try {
|
||||
@@ -243,7 +243,7 @@ function newFileSequence () {
|
||||
/* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */
|
||||
|
||||
for (let i = currentMaxSequence; i <= maxSequence; i++) {
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived)
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived)
|
||||
|
||||
function onFileReceived (err, file) {
|
||||
try {
|
||||
|
||||
+2
-2
@@ -109,7 +109,7 @@ function newMarketFiles () {
|
||||
let periodName = marketFilesPeriods[i][1]
|
||||
|
||||
if (dataset.config.validTimeFrames.includes(periodName) === true) {
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived)
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived)
|
||||
|
||||
function onFileReceived (err, file) {
|
||||
try {
|
||||
@@ -161,7 +161,7 @@ function newMarketFiles () {
|
||||
if (message.event.timeFrames !== undefined) {
|
||||
if (message.event.timeFrames.includes(periodName) !== true) { continue }
|
||||
}
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived)
|
||||
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived)
|
||||
|
||||
function onFileReceived (err, file) {
|
||||
try {
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@ function newSingleFile () {
|
||||
|
||||
/* Now we will get the file */
|
||||
|
||||
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived)
|
||||
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, undefined, onFileReceived)
|
||||
|
||||
function onFileReceived (err, pFile) {
|
||||
try {
|
||||
|
||||
Reference in new issue
Block a user