fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Data Mine Refactoring

Browse Source
This commit is contained in:
Luis Fernando Molina committed 2019-12-26 10:45:42 +01:00
1 parent 9a329a0818
commit 66cdd3f3fd
6 files changed
+44 -44
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/DailyFiles.js
+4 -4
View File
@@ -45,14 +45,14 @@ function newDailyFiles () {
}
}
function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) {
function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) {
try {
callBackWhenFileReceived = callBackFunction
let exchange = ecosystem.getExchange(pProduct, pExchange)
if (exchange === undefined) {
throw 'Exchange not supoorted by this product of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange
throw 'Exchange not supoorted by this product of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange
}
fileCloud = newFileCloud()
@@ -60,7 +60,7 @@ function newDailyFiles () {
/* First we will get the Data Range */
fileCloud.getFile(pDevTeam, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived)
fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived)
function onDataRangeReceived (err, pFile) {
try {
@@ -110,7 +110,7 @@ function newDailyFiles () {
if (pSet.validPeriods.includes(periodName) === true) {
let fileCursor = newFileCursor()
fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s
fileCursor.initialize(fileCloud, pDevTeam, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized)
fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized)
function onInitialized (err) {
try {
switch (err.result) {
src/FileCloud.js
+10 -10
View File
@@ -46,18 +46,18 @@ function newFileCloud () {
}
}
function getFile (pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) {
function getFile (pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) {
try {
if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') }
const MAX_RETRIES = 3
getFileRecursively(0, pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction)
getFileRecursively(0, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction)
function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) {
function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) {
try {
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') }
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) }
if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) }
let fileName
let filePath
@@ -85,7 +85,7 @@ function newFileCloud () {
if (fileName === undefined) {
logger.write('[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: ')
logger.write('[ERROR] getFile -> getFileRecursively -> pDevTeam = ' + JSON.stringify(pDevTeam))
logger.write('[ERROR] getFile -> getFileRecursively -> pDataMine = ' + JSON.stringify(pDataMine))
logger.write('[ERROR] getFile -> getFileRecursively -> pBot = ' + JSON.stringify(pBot))
logger.write('[ERROR] getFile -> getFileRecursively -> pSet = ' + JSON.stringify(pSet))
logger.write('[ERROR] getFile -> getFileRecursively -> pExchange = ' + JSON.stringify(pExchange))
@@ -100,8 +100,8 @@ function newFileCloud () {
fileName = fileName.replace('@AssetB', pMarket.assetB)
}
if (pDevTeam !== undefined) {
filePath = filePath.replace('@DevTeam', pDevTeam.codeName)
if (pDataMine !== undefined) {
filePath = filePath.replace('@DataMine', pDataMine.codeName)
}
if (pBot !== undefined) {
@@ -151,9 +151,9 @@ function newFileCloud () {
let containerName
containerName = pDevTeam.codeName.toLowerCase()
containerName = pDataMine.codeName.toLowerCase()
blobService.getBlobToText(containerName, filePath + '/' + fileName, pDevTeam.host, onFileReceived)
blobService.getBlobToText(containerName, filePath + '/' + fileName, pDataMine.host, onFileReceived)
function onFileReceived (err, text, response) {
try {
@@ -190,7 +190,7 @@ function newFileCloud () {
if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = ' + MAX_RETRIES) }
if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = ' + pRetryCounter) }
getFileRecursively(pRetryCounter + 1, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction)
getFileRecursively(pRetryCounter + 1, pDataMine, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction)
return
} else {
if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. ') }
src/FileCursor.js
+8 -8
View File
@@ -25,7 +25,7 @@ function newFileCursor () {
let market
let exchange
let fileCloud
let devTeam
let dataMine
let bot
let session
let product
@@ -52,7 +52,7 @@ function newFileCursor () {
market = undefined
exchange = undefined
fileCloud = undefined
devTeam = undefined
dataMine = undefined
bot = undefined
session = undefined
product = undefined
@@ -73,12 +73,12 @@ function newFileCursor () {
}
}
function initialize (pFileCloud, pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) {
function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) {
try {
market = pMarket
exchange = pExchange
fileCloud = pFileCloud
devTeam = pDevTeam
dataMine = pDataMine
bot = pBot
session = pSession
product = pProduct
@@ -89,10 +89,10 @@ function newFileCursor () {
beginDateRange = pBeginDateRange
endDateRange = pEndDateRange
let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName
let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName
systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles)
key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + pPeriodName
key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + pPeriodName
systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange)
callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE)
@@ -147,7 +147,7 @@ function newFileCursor () {
dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2)
fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
@@ -447,7 +447,7 @@ function newFileCursor () {
if (thisObject.files.get(dateString) === undefined) {
// We dont reload files we already have.
fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived)
} else {
controlLoop()
}
src/FileSequence.js
+10 -10
View File
@@ -20,7 +20,7 @@ function newFileSequence () {
let files = new Map()
let maxSequence = -1 // This is replaced by the content of the sequence file, which contains an index that starts on zero. In the case that the sequence file is not found the default value is -1 so when you add 1 it gives you the amount of files in the sequence, zero.
let market
let devTeam
let dataMine
let bot
let session
let thisSet
@@ -47,7 +47,7 @@ function newFileSequence () {
files = undefined
maxSequence = undefined
market = undefined
devTeam = undefined
dataMine = undefined
bot = undefined
session = undefined
thisSet = undefined
@@ -59,16 +59,16 @@ function newFileSequence () {
}
}
function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) {
function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) {
try {
exchange = ecosystem.getExchange(pProduct, pExchange)
if (exchange === undefined) {
throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange
throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange
}
market = pMarket
devTeam = pDevTeam
dataMine = pDataMine
bot = pBot
session = pSession
thisSet = pSet
@@ -79,7 +79,7 @@ function newFileSequence () {
callerId = newUniqueId()
let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName
let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName
systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles)
function onResponse (message) {
@@ -88,7 +88,7 @@ function newFileSequence () {
/* First we will get the sequence max number */
fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived)
fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived)
function onSequenceFileReceived (err, file) {
try {
@@ -133,7 +133,7 @@ function newFileSequence () {
/* Now we will get the sequence of files */
for (let i = 0; i <= maxSequence; i++) {
fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
@@ -198,7 +198,7 @@ function newFileSequence () {
let currentMaxSequence = maxSequence
fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived)
fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived)
function onSequenceFileReceived (err, sequenceFile) {
try {
@@ -234,7 +234,7 @@ function newFileSequence () {
/* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */
for (let i = currentMaxSequence; i <= maxSequence; i++) {
fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
src/MarketFiles.js
+8 -8
View File
@@ -24,7 +24,7 @@ function newMarketFiles () {
let market
let exchange
let devTeam
let dataMine
let bot
let session
let thisSet
@@ -52,7 +52,7 @@ function newMarketFiles () {
market = undefined
exchange = undefined
devTeam = undefined
dataMine = undefined
bot = undefined
session = undefined
thisSet = undefined
@@ -65,16 +65,16 @@ function newMarketFiles () {
}
}
function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) {
function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) {
try {
exchange = ecosystem.getExchange(pProduct, pExchange)
if (exchange === undefined) {
throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange
throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange
}
market = pMarket
devTeam = pDevTeam
dataMine = pDataMine
bot = pBot
session = pSession
thisSet = pSet
@@ -90,7 +90,7 @@ function newMarketFiles () {
let periodName = marketFilesPeriods[i][1]
if (thisSet.validPeriods.includes(periodName) === true) {
fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
@@ -103,7 +103,7 @@ function newMarketFiles () {
}
if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) {
let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName
let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName
systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles)
callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject)
@@ -137,7 +137,7 @@ function newMarketFiles () {
let periodName = marketFilesPeriods[i][1]
if (thisSet.validPeriods.includes(periodName) === true) {
fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived)
fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
src/SingleFile.js
+4 -4
View File
@@ -16,17 +16,17 @@ function newSingleFile () {
return thisObject
function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) {
function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) {
try {
if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') }
if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) }
if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) }
let exchange = ecosystem.getExchange(pProduct, pExchange)
if (pExchange !== undefined && exchange === undefined) {
// We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product.
throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange
throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange
}
fileCloud = newFileCloud()
@@ -34,7 +34,7 @@ function newSingleFile () {
/* Now we will get the file */
fileCloud.getFile(pDevTeam, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived)
fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived)
function onFileReceived (err, pFile) {
try {