fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactoring dataMine

Browse Source
This commit is contained in:
Luis Fernando Molina committed 2020-11-03 09:31:41 +01:00
1 parent 431557dc92
commit 9d4c532a14
6 files changed
+39 -39
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/DailyFiles.js
+4 -4
View File
@@ -45,7 +45,7 @@ function newDailyFiles() {
}
}
function initialize(pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, pEventsServerClient, callBackFunction) {
function initialize(pMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, pEventsServerClient, callBackFunction) {
try {
callBackWhenFileReceived = callBackFunction
@@ -69,7 +69,7 @@ function newDailyFiles() {
function getDataRangeFile() {
/* First we will get the Data Range */
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived)
fileCloud.getFile(pMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived)
function onDataRangeReceived(err, pFile) {
switch (err.result) {
@@ -109,7 +109,7 @@ function newDailyFiles() {
function getTimeFramesFile() {
/* First we will get the Data Range */
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived)
fileCloud.getFile(pMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived)
function onTimeFramesReceived(err, pFile) {
switch (err.result) {
@@ -165,7 +165,7 @@ function newDailyFiles() {
let fileCursor = newFileCursor()
fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s
fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized)
fileCursor.initialize(fileCloud, pMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized)
function onInitialized(err) {
try {
switch (err.result) {
src/FileCloud.js
+7 -7
View File
@@ -21,12 +21,12 @@ function newFileCloud () {
fileStorage = newFileStorage(pHost, pPort)
}
function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) {
function getFile (pMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) {
try {
const MAX_RETRIES = 3
getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction)
getFileRecursively(0, pMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction)
function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) {
function getFileRecursively (pRetryCounter, pMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) {
try {
let fileName
let filePath
@@ -78,7 +78,7 @@ function newFileCloud () {
if (fileName === undefined) {
logger.write('[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: ')
logger.write('[ERROR] getFile -> getFileRecursively -> pDataMine = ' + JSON.stringify(pDataMine))
logger.write('[ERROR] getFile -> getFileRecursively -> pMine = ' + JSON.stringify(pMine))
logger.write('[ERROR] getFile -> getFileRecursively -> pBot = ' + JSON.stringify(pBot))
logger.write('[ERROR] getFile -> getFileRecursively -> pDataset = ' + JSON.stringify(pDataset))
logger.write('[ERROR] getFile -> getFileRecursively -> pExchange = ' + JSON.stringify(pExchange))
@@ -93,8 +93,8 @@ function newFileCloud () {
filePath = filePath.replace('@QuotedAsset', pMarket.quotedAsset)
}
if (pDataMine !== undefined) {
filePath = filePath.replace('@DataMine', pDataMine.config.codeName)
if (pMine !== undefined) {
filePath = filePath.replace('@DataMine', pMine.config.codeName)
}
if (pBot !== undefined) {
@@ -161,7 +161,7 @@ function newFileCloud () {
if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = ' + MAX_RETRIES) }
if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = ' + pRetryCounter) }
getFileRecursively(pRetryCounter + 1, pDataMine, pBot, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction)
getFileRecursively(pRetryCounter + 1, pMine, pBot, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction)
return
} else {
if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. ') }
src/FileCursor.js
+8 -8
View File
@@ -25,7 +25,7 @@ function newFileCursor() {
let market
let exchange
let fileCloud
let dataMine
let mine
let bot
let session
let product
@@ -53,7 +53,7 @@ function newFileCursor() {
market = undefined
exchange = undefined
fileCloud = undefined
dataMine = undefined
mine = undefined
bot = undefined
session = undefined
product = undefined
@@ -77,7 +77,7 @@ function newFileCursor() {
function initialize(
pFileCloud,
pDataMine,
pMine,
pBot,
pSession,
pProduct,
@@ -96,7 +96,7 @@ function newFileCursor() {
market = pMarket
exchange = pExchange
fileCloud = pFileCloud
dataMine = pDataMine
mine = pMine
bot = pBot
session = pSession
product = pProduct
@@ -108,11 +108,11 @@ function newFileCursor() {
endDateRange = pEndDateRange
eventsServerClient = pEventsServerClient
let key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset
let key = mine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset
let callerId = key + '-' + periodName + newUniqueId()
eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponseDataSet, updateFiles)
key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset
key = mine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset
callerId = key + '-' + periodName + newUniqueId()
eventsServerClient.listenToEvent(key, 'Data Range Updated', undefined, callerId, onResponseDataRange, updateDataRange)
@@ -168,7 +168,7 @@ function newFileCursor() {
dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2)
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived)
fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived)
function onFileReceived(err, file) {
try {
@@ -468,7 +468,7 @@ function newFileCursor() {
if (thisObject.files.get(dateString) === undefined) {
// We dont reload files we already have.
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived)
fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived)
} else {
controlLoop()
}
src/FileSequence.js
+9 -9
View File
@@ -20,7 +20,7 @@ function newFileSequence () {
let files = new Map()
let maxSequence = -1 // This is replaced by the content of the sequence file, which contains an index that starts on zero. In the case that the sequence file is not found the default value is -1 so when you add 1 it gives you the amount of files in the sequence, zero.
let market
let dataMine
let mine
let bot
let session
let dataset
@@ -48,7 +48,7 @@ function newFileSequence () {
files = undefined
maxSequence = undefined
market = undefined
dataMine = undefined
mine = undefined
bot = undefined
session = undefined
dataset = undefined
@@ -62,7 +62,7 @@ function newFileSequence () {
}
function initialize (
pDataMine,
pMine,
pBot,
pSession,
pProduct,
@@ -76,7 +76,7 @@ function newFileSequence () {
try {
exchange = pExchange
market = pMarket
dataMine = pDataMine
mine = pMine
bot = pBot
session = pSession
dataset = pDataset
@@ -88,7 +88,7 @@ function newFileSequence () {
callerId = newUniqueId()
let key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset
let key = mine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset
eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles)
function onResponse (message) {
@@ -97,7 +97,7 @@ function newFileSequence () {
/* First we will get the sequence max number */
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived)
fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived)
function onSequenceFileReceived (err, file) {
try {
@@ -142,7 +142,7 @@ function newFileSequence () {
/* Now we will get the sequence of files */
for (let i = 0; i <= maxSequence; i++) {
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived)
fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
@@ -207,7 +207,7 @@ function newFileSequence () {
let currentMaxSequence = maxSequence
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived)
fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived)
function onSequenceFileReceived (err, sequenceFile) {
try {
@@ -243,7 +243,7 @@ function newFileSequence () {
/* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */
for (let i = currentMaxSequence; i <= maxSequence; i++) {
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived)
fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived)
function onFileReceived (err, file) {
try {
src/MarketFiles.js
+8 -8
View File
@@ -25,7 +25,7 @@ function newMarketFiles() {
let market
let exchange
let dataMine
let mine
let bot
let session
let dataset
@@ -55,7 +55,7 @@ function newMarketFiles() {
market = undefined
exchange = undefined
dataMine = undefined
mine = undefined
bot = undefined
session = undefined
dataset = undefined
@@ -70,7 +70,7 @@ function newMarketFiles() {
}
function initialize(
pDataMine,
pMine,
pBot,
pSession,
pProduct,
@@ -86,7 +86,7 @@ function newMarketFiles() {
exchange = pExchange
market = pMarket
dataMine = pDataMine
mine = pMine
bot = pBot
session = pSession
dataset = pDataset
@@ -108,7 +108,7 @@ function newMarketFiles() {
function getTimeFramesFile() {
/* First we will get the Data Range */
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived)
fileCloud.getFile(pMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived)
function onTimeFramesReceived(err, pFile) {
switch (err.result) {
@@ -164,7 +164,7 @@ function newMarketFiles() {
}
filesExpected++
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived)
fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived)
function onFileReceived(err, file) {
try {
@@ -177,7 +177,7 @@ function newMarketFiles() {
}
if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) {
let key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset
let key = mine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset
let callerId = key + '-' + periodName + newUniqueId()
eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles)
@@ -219,7 +219,7 @@ function newMarketFiles() {
if (message.event.timeFrames.includes(periodName) !== true) { continue }
}
}
fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived)
fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived)
function onFileReceived(err, file) {
try {
src/SingleFile.js
+3 -3
View File
@@ -17,7 +17,7 @@ function newSingleFile () {
return thisObject
function initialize (
pDataMine,
pMine,
pBot,
pSession,
pProduct,
@@ -30,7 +30,7 @@ function newSingleFile () {
callBackFunction) {
try {
if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') }
if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.config.codeName + '-' + pBot.config.codeName + '-' + pProduct.config.codeName) }
if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pMine.config.codeName + '-' + pBot.config.codeName + '-' + pProduct.config.codeName) }
let exchange = pExchange
@@ -39,7 +39,7 @@ function newSingleFile () {
/* Now we will get the file */
fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, undefined, onFileReceived)
fileCloud.getFile(pMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, undefined, onFileReceived)
function onFileReceived (err, pFile) {
try {