Merge pull request #154 from Superalgos/develop

Docs Moved Out
This commit is contained in:
Luis Fernando Molina authored and GitHub committed 2020-02-28 10:56:25 +01:00
commit 2219755419
16 files changed
+618 -35

No files matched your search

+1 -1
View File
@@ -90,7 +90,7 @@ function newCodeEditor () {
'box-shadow: none;' +
'overflow:hidden;' +
'font-family: ' + UI_FONT.PRIMARY + ';' +
'font-size: 12px;' +
'font-size: 14px;' +
'background-color: rgb(' + UI_COLOR.RUSTED_RED + ');' +
'color:rgb(255, 255, 255);' +
'width: ' + thisObject.container.frame.width + 'px;' +
+1 -1
View File
@@ -90,7 +90,7 @@ function newConditionEditor () {
'box-shadow: none;' +
'overflow:hidden;' +
'font-family: ' + UI_FONT.PRIMARY + ';' +
'font-size: 12px;' +
'font-size: 14px;' +
'background-color: rgb(' + UI_COLOR.GREEN + ');' +
'color:rgb(255, 255, 255);' +
'width: ' + thisObject.container.frame.width + 'px;' +
+2 -2
View File
@@ -34,7 +34,7 @@ function newConfigEditor () {
thisObject.container.frame.position.y = 0
let isMouseOver = false
let SIZE_FACTOR = 0.6
let SIZE_FACTOR = 0.8
return thisObject
@@ -97,7 +97,7 @@ function newConfigEditor () {
'box-shadow: none;' +
'overflow:hidden;' +
'font-family: ' + UI_FONT.PRIMARY + ';' +
'font-size: 12px;' +
'font-size: 14px;' +
'background-color: rgb(' + UI_COLOR.DARK_TURQUOISE + ');' +
'color:rgb(255, 255, 255);' +
'width: ' + thisObject.container.frame.width + 'px;' +
+1 -1
View File
@@ -90,7 +90,7 @@ function newFormulaEditor () {
'box-shadow: none;' +
'overflow:hidden;' +
'font-family: ' + UI_FONT.PRIMARY + ';' +
'font-size: 12px;' +
'font-size: 14px;' +
'background-color: rgb(' + UI_COLOR.MANGANESE_PURPLE + ');' +
'color:rgb(255, 255, 255);' +
'width: ' + thisObject.container.frame.width + 'px;' +
+2 -1
View File
@@ -92,7 +92,8 @@ function newFloatingSpace () {
payload.node.type === 'Network' ||
payload.node.type === 'Crypto Ecosystem' ||
payload.node.type === 'Charting Space' ||
payload.node.type === 'Data Mine'
payload.node.type === 'Data Mine' ||
payload.node.type === 'Super Scripts'
) {
return false
}
+8
View File
@@ -54,6 +54,14 @@ function newDailyFiles () {
fileCloud = newFileCloud()
fileCloud.initialize(pBot)
/* Some Validations */
if (pDataset.code.validTimeFrames === undefined) {
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) }
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') }
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
return
}
/* First we will get the Data Range */
fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived)
+8
View File
@@ -78,6 +78,14 @@ function newMarketFiles () {
fileCloud = newFileCloud()
fileCloud.initialize(pBot)
/* Some Validations */
if (dataset.code.validTimeFrames === undefined) {
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) }
if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') }
callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE)
return
}
/* Now we will get the market files */
for (let i = 0; i < marketFilesPeriods.length; i++) {
@@ -0,0 +1,563 @@
exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, fileStorage, STATUS_REPORT, EXCHANGE_API) {
const FULL_LOG = true;
const LOG_FILE_CONTENT = false;
const GMT_SECONDS = ':00.000 GMT+0000';
const GMT_MILI_SECONDS = '.000 GMT+0000';
const MODULE_NAME = "User Bot";
const CANDLES_FOLDER_NAME = "Candles/One-Min";
const VOLUMES_FOLDER_NAME = "Volumes/One-Min";
thisObject = {
initialize: initialize,
start: start
};
let utilities = UTILITIES.newCloudUtilities(bot, logger)
let statusDependencies
const ONE_MIN = 60000
const ONE_DAY = ONE_MIN * 60 * 24
const MAX_OHLCVs_PER_EXECUTION = 100000
const symbol = bot.market.baseAsset + '/' + bot.market.quotedAsset
const ccxt = require('ccxt')
let allOHLCVs = []
let thisReport;
let since
let initialProcessTimestamp
let beginingOfMarket
let lastFile
let exchangeId
let options = {}
let firstId
let rateLimit
let exchange
let uiStartDate = new Date(bot.uiStartDate)
let fisrtTimeThisProcessRun = false
let limit = 1000 // This is the default value
return thisObject;
function initialize(pStatusDependencies, callBackFunction) {
try {
logger.fileName = MODULE_NAME;
logger.initialize();
statusDependencies = pStatusDependencies;
exchangeId = bot.exchange.toLowerCase()
/* Applying the parameters defined by the user at the Exchange node */
if (bot.exchangeNode.code.API !== undefined) {
for (let i = 0; i < bot.exchangeNode.code.API.length; i++) {
if (bot.exchangeNode.code.API[i].method === 'fetchOHLCVs') {
if (bot.exchangeNode.code.API[i].class !== undefined) {
exchangeId = bot.exchangeNode.code.API[i].class
}
if (bot.exchangeNode.code.API[i].fetchOHLCVsMethod !== undefined) {
options = {
'fetchOHLCVsMethod': bot.exchangeNode.code.API[i].fetchOHLCVsMethod
}
}
if (bot.exchangeNode.code.API[i].firstId !== undefined) {
firstId = bot.exchangeNode.code.API[i].firstId
}
if (bot.exchangeNode.code.API[i].rateLimit !== undefined) {
rateLimit = bot.exchangeNode.code.API[i].rateLimit
}
}
}
}
if (bot.code.fetchLimit !== undefined) {
limit = bot.code.fetchLimit
}
let key = process.env.KEY
let secret = process.env.SECRET
if (key === "undefined") { key = undefined }
if (secret === "undefined") { secret = undefined }
const exchangeClass = ccxt[exchangeId]
const exchangeConstructorParams = {
'apiKey': key,
'secret': secret,
'timeout': 30000,
'enableRateLimit': true,
verbose: false,
options: options
}
if (rateLimit !== undefined) {
exchangeConstructorParams.rateLimit = rateLimit
}
exchange = new exchangeClass(exchangeConstructorParams)
callBackFunction(global.DEFAULT_OK_RESPONSE);
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function start(callBackFunction) {
try {
if (global.STOP_TASK_GRACEFULLY === true) {
callBackFunction(global.DEFAULT_OK_RESPONSE);
return
}
let abort = false
begin()
async function begin() {
getContextVariables()
if (abort === true) { return }
await getOHLCVs()
if (abort === true) { return }
if (global.STOP_TASK_GRACEFULLY === true) {
callBackFunction(global.DEFAULT_OK_RESPONSE);
return
}
await saveOHLCVs()
}
function getContextVariables() {
try {
let reportKey;
reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Historic-OHLCVs" + "-" + "dataSet.V1";
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
if (statusDependencies.statusReports.get(reportKey).status === "Status Report is corrupt.") {
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
return;
}
thisReport = statusDependencies.statusReports.get(reportKey)
if (thisReport.file.beginingOfMarket !== undefined) { // This means this is not the first time this process run.
beginingOfMarket = new Date(thisReport.file.beginingOfMarket.year + "-" + thisReport.file.beginingOfMarket.month + "-" + thisReport.file.beginingOfMarket.days + " " + thisReport.file.beginingOfMarket.hours + ":" + thisReport.file.beginingOfMarket.minutes + GMT_SECONDS);
lastFile = new Date(thisReport.file.lastFile.year + "-" + thisReport.file.lastFile.month + "-" + thisReport.file.lastFile.days + " " + thisReport.file.lastFile.hours + ":" + thisReport.file.lastFile.minutes + GMT_SECONDS);
} else { // This means this is the first time this process run.
fisrtTimeThisProcessRun = true
beginingOfMarket = new Date(uiStartDate.valueOf())
}
defineSince()
function defineSince() {
if (thisReport.file.uiStartDate === undefined) {
thisReport.file.uiStartDate = uiStartDate
} else {
thisReport.file.uiStartDate = new Date(thisReport.file.uiStartDate)
}
if (uiStartDate.valueOf() !== thisReport.file.uiStartDate.valueOf()) {
since = uiStartDate.valueOf()
initialProcessTimestamp = since
beginingOfMarket = new Date(uiStartDate.valueOf())
} else {
if (lastFile !== undefined) {
since = lastFile.valueOf()
initialProcessTimestamp = lastFile.valueOf()
} else {
since = uiStartDate.valueOf()
initialProcessTimestamp = uiStartDate.valueOf()
}
}
}
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> err = " + err.stack);
if (err.message === "Cannot read property 'file' of undefined") {
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Check the bot Status Dependencies. ");
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys));
}
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
abort = true
}
}
async function getOHLCVs() {
try {
let lastOHLCVKey = ''
let params = undefined
let previousSince
while (true) {
/* Reporting we are doing well */
let processingDate = new Date(since)
processingDate = processingDate.getUTCFullYear() + '-' + utilities.pad(processingDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(processingDate.getUTCDate(), 2);
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getOHLCVs -> Fetching OHLCVs @ " + processingDate + "-> exchange = " + bot.exchange + " -> symbol = " + symbol + " -> since = " + since + " -> limit = " + limit) }
bot.processHeartBeat("Fetching " + allOHLCVs.length.toFixed(0) + " OHLCVs from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
/* Fetching the OHLCVs from the exchange.*/
const OHLCVs = await exchange.fetchOHLCV(symbol, '1m', since, limit, params)
/*
OHLCV Structure
The fetchOHLCV method shown above returns a list (a flat array) of OHLCV candles represented by the following structure:
[
[
1504541580000, // UTC timestamp in milliseconds, integer
4235.4, // (O)pen price, float
4240.6, // (H)ighest price, float
4230.0, // (L)owest price, float
4230.7, // (C)losing price, float
37.72941911 // (V)olume (in terms of the base currency), float
],
...
]
*/
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getOHLCVs -> OHLCVs Fetched = " + OHLCVs.length) }
if (OHLCVs.length > 0) {
let beginDate = new Date(OHLCVs[0].timestamp)
let endDate = new Date(OHLCVs[OHLCVs.length - 1].timestamp)
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getOHLCVs -> OHLCVs Fetched From " + beginDate) }
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getOHLCVs -> OHLCVs Fetched to " + endDate) }
if (fisrtTimeThisProcessRun === true) {
let OHLCV = OHLCVs[0]
initialProcessTimestamp = OHLCV[0] // 'timestamp'
beginingOfMarket = new Date(Math.trunc (OHLCV[0] / ONE_DAY) * ONE_DAY) // 'timestamp'
fisrtTimeThisProcessRun = false
}
}
if (OHLCVs.length > 1) {
previousSince = since
since = OHLCVs[OHLCVs.length - 1][0] // 'timestamp'
if (since === previousSince) {
since++ // this prevents requesting in a loop OHLCVs with the same timestamp, that can happen when all the records fetched comes with exactly the same timestamp.
}
for (let i = 0; i < OHLCVs.length; i++) {
let OHLCV = OHLCVs[i]
let OHLCVKey = OHLCV[0] + '-' + OHLCV[1].toFixed(16) + '-' + OHLCV[2].toFixed(16) + '-' + OHLCV[3].toFixed(16) + '-' + OHLCV[4].toFixed(16) + '-' + OHLCV[5].toFixed(16)
if (OHLCVKey !== lastOHLCVKey) {
allOHLCVs.push(OHLCV)
}
lastOHLCVKey = OHLCVKey
}
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getOHLCVs -> Fetching OHLCVs @ " + processingDate + "-> exchange = " + bot.exchange + " -> symbol = " + symbol + " -> since = " + since + " -> limit = " + limit) }
bot.processHeartBeat("Fetching " + allOHLCVs.length.toFixed(0) + " OHLCVs from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
}
if (
OHLCVs.length < limit - 1 ||
global.STOP_TASK_GRACEFULLY === true ||
allOHLCVs.length >= MAX_OHLCVs_PER_EXECUTION
) {
break
}
}
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> getOHLCVs -> Retrying Later -> err = " + err.stack);
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
abort = true
}
}
async function saveOHLCVs() {
try {
let candlesFileContent = '['
let volumesFileContent = '['
let previousDay = Math.trunc((initialProcessTimestamp - ONE_DAY) / ONE_DAY)
let currentDay = Math.trunc((initialProcessTimestamp - ONE_DAY) / ONE_DAY)
let needSeparator = false
let error
let separator
let heartBeatCounter = 0
let lastCandle = {
begin: 0,
end: 0,
open: 0,
close: 0,
min: 0,
max: 0
}
let lastVolume = {
begin: 0,
end: 0,
buy: 0,
sell: 0
}
let i = 0
controlLoop()
function loop() {
let filesToCreate = 0
let filesCreated = 0
/* Go through all possible 1 min candles of a day. */
for (let j = 0; j < 60 * 24; j++) {
let candle = {
begin: currentDay * ONE_DAY + ONE_MIN * j,
end: currentDay * ONE_DAY + ONE_MIN * j + ONE_MIN - 1,
open: lastCandle.close,
close: lastCandle.close,
min: lastCandle.close,
max: lastCandle.close
}
let volume = {
begin: currentDay * ONE_DAY + ONE_MIN * j,
end: currentDay * ONE_DAY + ONE_MIN * j + ONE_MIN - 1,
buy: lastVolume.buy,
sell: lastVolume.sell
}
let record = allOHLCVs[i]
let OHLCV = {
timestamp: record[0],
open: record[1],
hight: record[2],
low: record[3],
close: record[4],
volume: record[5]
}
if (OHLCV.open === 11478 || OHLCV.hight === 11478 || OHLCV.low === 11478 || OHLCV.close === 11478) {
console.log('FROM HERE')
}
let candleMinute = Math.trunc(candle.begin / ONE_MIN)
let OHLCVMinute
checkOHLCVMinute()
function checkOHLCVMinute() {
/*
Some exchanges return inconsistent data. It is not guaranteed that each candle will have a timeStamp exactly at the begining of an
UTC minute. It is also not guaranteed that the distance between timestamps will be the same. To fix this, we will do this.
*/
OHLCVMinute = Math.trunc(OHLCV.timestamp / ONE_MIN)
if (OHLCVMinute < candleMinute) {
i++
record = allOHLCVs[i]
OHLCV = {
timestamp: record[0],
open: record[1],
hight: record[2],
low: record[3],
close: record[4],
volume: record[5]
}
checkOHLCVMinute()
}
}
/* Reporting we are doing well */
let processingDate = new Date(candle.begin)
processingDate = processingDate.getUTCFullYear() + '-' + utilities.pad(processingDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(processingDate.getUTCDate(), 2);
heartBeatCounter--
if (heartBeatCounter <= 0) {
heartBeatCounter = 1440
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> saveOHLCVs -> Saving OHLCVs @ " + processingDate + " -> i = " + i + " -> total = " + allOHLCVs.length) }
bot.processHeartBeat("Saving " + i.toFixed(0) + " / " + allOHLCVs.length + " OHLCVs from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
}
/* End Reporting */
if (candleMinute === OHLCVMinute) {
candle.open = OHLCV.open
candle.close = OHLCV.close
candle.min = OHLCV.low
candle.max = OHLCV.hight
volume.buy = OHLCV.volume / 2
volume.sell = OHLCV.volume / 2
i++
}
lastCandle = candle
if (needSeparator === false) {
needSeparator = true;
separator = '';
} else {
separator = ',';
}
/* Add the candle to the file content.*/
candlesFileContent = candlesFileContent + separator + '[' + candle.min + "," + candle.max + "," + candle.open + "," + candle.close + "," + candle.begin + "," + candle.end + "]";
volumesFileContent = volumesFileContent + separator + '[' + volume.buy + "," + volume.sell + "," + volume.begin + "," + volume.end + "]";
if (i === allOHLCVs.length) {
/* This was the last OHLCV, so we save the file.*/
saveFile(currentDay)
/* It might happen that there are several days after the last OHLCV without OHLCVs. We need to record empty files for them.*/
if (allOHLCVs.length < MAX_OHLCVs_PER_EXECUTION) {
let currentTimeDay = Math.trunc((new Date()).valueOf() / ONE_DAY)
if (currentTimeDay - currentDay > 1) {
createMissingEmptyFiles(currentDay, currentTimeDay)
}
}
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> saveOHLCVs -> Saving OHLCVs @ " + processingDate + " -> i = " + i + " -> total = " + allOHLCVs.length) }
bot.processHeartBeat("Saving " + i.toFixed(0) + " / " + allOHLCVs.length + " OHLCVs from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
return
}
}
saveFile(currentDay)
previousDay = currentDay
if (error) {
callBackFunction(error);
return;
}
function saveFile(day) {
candlesFileContent = candlesFileContent + ']'
volumesFileContent = volumesFileContent + ']'
if (currentDay - previousDay > 1) {
createMissingEmptyFiles(previousDay, currentDay)
}
let fileName = bot.market.baseAsset + '_' + bot.market.quotedAsset + '.json'
filesToCreate++
fileStorage.createTextFile(getFilePath(day * ONE_DAY, CANDLES_FOLDER_NAME) + '/' + fileName, candlesFileContent + '\n', onFileCreated);
filesToCreate++
fileStorage.createTextFile(getFilePath(day * ONE_DAY, VOLUMES_FOLDER_NAME) + '/' + fileName, volumesFileContent + '\n', onFileCreated);
candlesFileContent = '['
volumesFileContent = '['
needSeparator = false
}
function createMissingEmptyFiles(begin, end) {
/*
If this range is too wide, we will consider this means that the begin is before the begining of this market at this exchange.
In that case we will change the begin and the beginingOfMarket
*/
if ((end - begin) > 7) {
begin = end
beginingOfMarket = new Date(end * ONE_DAY)
}
for (let j = begin + 1; j < end; j++) {
let fileName = bot.market.baseAsset + '_' + bot.market.quotedAsset + '.json'
filesToCreate++
fileStorage.createTextFile(getFilePath(j * ONE_DAY, CANDLES_FOLDER_NAME) + '/' + fileName, "[]" + '\n', onFileCreated);
filesToCreate++
fileStorage.createTextFile(getFilePath(j * ONE_DAY, VOLUMES_FOLDER_NAME) + '/' + fileName, "[]" + '\n', onFileCreated);
}
}
function onFileCreated(err) {
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] start -> OHLCVsReadyToBeSaved -> onFileBCreated -> err = " + JSON.stringify(err));
error = err // This allows the loop to be breaked.
return;
}
filesCreated++
lastFile = new Date((currentDay * ONE_DAY))
if (filesCreated === filesToCreate) {
controlLoop()
}
}
function getFilePath(timestamp, folderName) {
let datetime = new Date(timestamp)
let dateForPath = datetime.getUTCFullYear() + '/' +
utilities.pad(datetime.getUTCMonth() + 1, 2) + '/' +
utilities.pad(datetime.getUTCDate(), 2)
let filePath = bot.filePathRoot + "/Output/" + folderName + '/' + dateForPath;
return filePath
}
}
function controlLoop() {
if (global.STOP_TASK_GRACEFULLY === true) {
callBackFunction(global.DEFAULT_OK_RESPONSE);
return
}
currentDay++
if (i < allOHLCVs.length) {
setImmediate(loop)
} else {
writeStatusReport()
}
}
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> saveOHLCVs -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
abort = true
}
}
function writeStatusReport() {
try {
if (lastFile === undefined) { return }
thisReport.file = {
lastFile: {
year: lastFile.getUTCFullYear(),
month: (lastFile.getUTCMonth() + 1),
days: lastFile.getUTCDate(),
hours: lastFile.getUTCHours(),
minutes: lastFile.getUTCMinutes()
},
beginingOfMarket: {
year: beginingOfMarket.getUTCFullYear(),
month: (beginingOfMarket.getUTCMonth() + 1),
days: beginingOfMarket.getUTCDate(),
hours: beginingOfMarket.getUTCHours(),
minutes: beginingOfMarket.getUTCMinutes()
},
uiStartDate: uiStartDate.toUTCString()
};
thisReport.save(onSaved);
function onSaved(err) {
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> onSaved -> err = " + err.stack);
callBackFunction(err);
return;
}
callBackFunction(global.DEFAULT_OK_RESPONSE);
}
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
} catch (err) {
logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
};
@@ -52,7 +52,7 @@
This process is going to do the following:
Read the candles and volumes from Bruce and produce a file for each day and for each period with candles and volumes.
Read the candles and volumes from Charly and produce a file for each day and for each period with candles and volumes.
*/
@@ -86,7 +86,7 @@ Read the candles and volumes from Bruce and produce a file for each day and for
/* We look first for Charly in order to get when the market starts. */
reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Historic-Trades" + "-" + "dataSet.V1";
reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Historic-OHLCVs" + "-" + "dataSet.V1";
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
statusReport = statusDependencies.statusReports.get(reportKey);
@@ -120,9 +120,9 @@ Read the candles and volumes from Bruce and produce a file for each day and for
contextVariables.firstTradeFile = new Date(thisReport.beginingOfMarket.year + "-" + thisReport.beginingOfMarket.month + "-" + thisReport.beginingOfMarket.days + " " + thisReport.beginingOfMarket.hours + ":" + thisReport.beginingOfMarket.minutes + GMT_SECONDS);
/* Second, we get the report from Bruce, to know when the marted ends. */
/* Second, we get the report from Charly, to know when the marted ends. */
reportKey = "AAMasters" + "-" + "AABruce" + "-" + "Single-Period-Daily" + "-" + "dataSet.V1"
reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Historic-OHLCVs" + "-" + "dataSet.V1"
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
statusReport = statusDependencies.statusReports.get(reportKey);
@@ -304,7 +304,7 @@ Read the candles and volumes from Bruce and produce a file for each day and for
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
let fileName = market.baseAsset + '_' + market.quotedAsset + ".json"
let filePathRoot = bot.dataMine + "/" + "AABruce" + "/" + bot.exchange;
let filePathRoot = bot.dataMine + "/" + "AACharly" + "/" + bot.exchange;
let filePath = filePathRoot + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
filePath += '/' + fileName
@@ -430,7 +430,7 @@ Read the candles and volumes from Bruce and produce a file for each day and for
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
let fileName = market.baseAsset + '_' + market.quotedAsset + ".json"
let filePathRoot = bot.dataMine + "/" + "AABruce" + "/" + bot.exchange;
let filePathRoot = bot.dataMine + "/" + "AACharly" + "/" + bot.exchange;
let filePath = filePathRoot + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath;
filePath += '/' + fileName
@@ -50,16 +50,16 @@
This process is going to do the following:
Read the candles and volumes from Bruce and produce a single Index File for Market Period. But this is the situation:
Read the candles and volumes from Charly and produce a single Index File for Market Period. But this is the situation:
Bruce has a dataset organized with daily files with candles of 1 min. Olivia is writting in this process a single file for each timeFrame for the whole market.
Charly has a dataset organized with daily files with candles of 1 min. Olivia is writting in this process a single file for each timeFrame for the whole market.
Everytime this process run, must be able to resume its job and process everything pending until reaching the head of the market. So the tactic to do this is the
following:
1. First we need to read the last file written by this process, and load all the information into in-memory arrays. We will then append to this arrays the new
information we will get from Bruce.
information we will get from Charly.
2. We know from out status report which was the last DAY we processed from Bruce, but we must be carefull, because that day mightn not have been complete, if the
2. We know from out status report which was the last DAY we processed from Charly, but we must be carefull, because that day mightn not have been complete, if the
last run found the head of the market. That means that we have to be carefull not to append candles that are already there. To simplify what we do is to discard
all candles of the last processed day, and then we can process that full day again adding all the candles.
@@ -95,7 +95,7 @@
/* We look first for Charly in order to get when the market starts. */
reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Historic-Trades" + "-" + "dataSet.V1";
reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Historic-OHLCVs" + "-" + "dataSet.V1";
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
statusReport = statusDependencies.statusReports.get(reportKey);
@@ -129,9 +129,9 @@
contextVariables.firstTradeFile = new Date(thisReport.beginingOfMarket.year + "-" + thisReport.beginingOfMarket.month + "-" + thisReport.beginingOfMarket.days + " " + thisReport.beginingOfMarket.hours + ":" + thisReport.beginingOfMarket.minutes + GMT_SECONDS);
/* Second, we get the report from Bruce, to know when the marted ends. */
/* Second, we get the report from Charly, to know when the marted ends. */
reportKey = "AAMasters" + "-" + "AABruce" + "-" + "Single-Period-Daily" + "-" + "dataSet.V1"
reportKey = "AAMasters" + "-" + "AACharly" + "-" + "Historic-OHLCVs" + "-" + "dataSet.V1"
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
statusReport = statusDependencies.statusReports.get(reportKey);
@@ -403,6 +403,8 @@
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() + ONE_DAY_IN_MILISECONDS);
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> advanceTime -> New processing time @ " + contextVariables.lastCandleFile.getUTCFullYear() + "/" + (contextVariables.lastCandleFile.getUTCMonth() + 1) + "/" + contextVariables.lastCandleFile.getUTCDate() + "."); }
logger.newInternalLoop(bot.codeName, bot.process, contextVariables.lastCandleFile);
/* Validation that we are not going past the head of the market. */
@@ -450,12 +452,12 @@
const timeFrame = global.marketFilesPeriods[n][1];
/*
Here we are inside a Loop that is going to advance 1 day at the time, at each pass, will ready one of Bruce's daily files and
Here we are inside a Loop that is going to advance 1 day at the time, at each pass, will ready one of Charly's daily files and
add all its candles to our in memory arrays. At the first iteration of this loop, we will add the candles that we are carrying
from our previous run, the ones we already have in-memory. You can see below how we discard from those candles the ones that
are belonging to the first day we are processing at this run, that it is exactly the same as the last day processed the privious
run. By discarding these candles, we are ready to run after that standard function that will just add ALL the candles found each
day at Bruce.
day at Charly.
*/
if (previousCandles !== undefined) {
@@ -505,7 +507,7 @@
}
/*
From here on is where every iteration of the loop fully runs. Here is where we read Bruce's files and add their content to whatever
From here on is where every iteration of the loop fully runs. Here is where we read Charly's files and add their content to whatever
we already have in our arrays in-memory. In this way the process will run as many days needed and it should only stop when it reaches
the head of the market.
*/
@@ -518,7 +520,7 @@
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
let fileName = market.baseAsset + '_' + market.quotedAsset + ".json"
let filePathRoot = bot.dataMine + "/" + "AABruce" + "/" + bot.exchange;
let filePathRoot = bot.dataMine + "/" + "AACharly" + "/" + bot.exchange;
let filePath = filePathRoot + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
filePath += '/' + fileName
@@ -568,7 +570,7 @@
let beginingOutputTime = contextVariables.lastCandleFile.valueOf();
/*
The algorithm that follows is going to agregate candles of 1 min timeFrame read from Bruce, into candles of each timeFrame
The algorithm that follows is going to agregate candles of 1 min timeFrame read from Charly, into candles of each timeFrame
that Olivia generates. For market files those timePediods goes from 1h to 24hs.
*/
@@ -649,7 +651,7 @@
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
let fileName = market.baseAsset + '_' + market.quotedAsset + ".json"
let filePathRoot = bot.dataMine + "/" + "AABruce" + "/" + bot.exchange;
let filePathRoot = bot.dataMine + "/" + "AACharly" + "/" + bot.exchange;
let filePath = filePathRoot + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath;
filePath += '/' + fileName
@@ -56,17 +56,17 @@
"major": 1,
"minor": 0
},
"process": "Historic-Trades",
"process": "Historic-OHLCVs",
"dataSetVersion": "dataSet.V1"
},
{
"dataMine": "AAMasters",
"bot": "AABruce",
"bot": "AACharly",
"botVersion": {
"major": 1,
"minor": 0
},
"process": "Single-Period-Daily",
"process": "Historic-OHLCVs",
"dataSetVersion": "dataSet.V1",
"dataSetSection": "Month",
"waitUntilNextUpdate": true
@@ -128,17 +128,17 @@
"major": 1,
"minor": 0
},
"process": "Historic-Trades",
"process": "Historic-OHLCVs",
"dataSetVersion": "dataSet.V1"
},
{
"dataMine": "AAMasters",
"bot": "AABruce",
"bot": "AACharly",
"botVersion": {
"major": 1,
"minor": 0
},
"process": "Single-Period-Daily",
"process": "Historic-OHLCVs",
"dataSetVersion": "dataSet.V1",
"dataSetSection": "Month",
"waitUntilNextUpdate": true
+1 -1
View File
@@ -25,7 +25,7 @@ The current version opens up the door to working with multiple exchanges and mul
Please refer to the [new documentation site](https://docs.superalgos.org/).
Developers testing the version in the ```develop``` branch may consult the [develop documentation version](https://superalgos.github.io/Documentation/). This version of the docs is in a permanent work-in-progress state.
<!--Developers testing the version in the ```develop``` branch may consult the [develop documentation version](https://superalgos.github.io/Documentation/). This version of the docs is in a permanent work-in-progress state.-->
## Codebase and Development Environment
+1 -1
View File
@@ -454,7 +454,7 @@
previousDay = new Date(bot.multiPeriodProcessDatetime.valueOf() - ONE_DAY_IN_MILISECONDS);
if (global.WRITE_LOGS_TO_FILES === 'true') {
logger.newInternalLoop(bot.codeName, bot.process);
logger.newInternalLoop(bot.codeName, bot.process, bot.multiPeriodProcessDatetime);
}
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFramesDailyFiles -> advanceTime -> bot.multiPeriodProcessDatetime = " + bot.multiPeriodProcessDatetime); }
+1 -1
View File
@@ -320,7 +320,7 @@
previousDay = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() - ONE_DAY_IN_MILISECONDS);
if (global.WRITE_LOGS_TO_FILES === 'true') {
logger.newInternalLoop(bot.codeName, bot.process);
logger.newInternalLoop(bot.codeName, bot.process, bot.multiPeriodDailyProcessDatetime);
}
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimeFrames -> advanceTime -> bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime); }
+1
View File
@@ -212,6 +212,7 @@
quotedAsset: global.TASK_NODE.bot.processes[processIndex].marketReference.referenceParent.quotedAsset.referenceParent.code.codeName
}
botConfig.uiStartDate = global.TASK_NODE.bot.code.startDate
botConfig.code = global.TASK_NODE.bot.code
/* This stuff is still hardcoded and unresolved. */
botConfig.version = {
+1 -1
View File
@@ -1 +1 @@
{"type":"Workspace","name":"My Workspace","rootNodes":[{"type":"Network","name":"Superalgos","networkNodes":[{"type":"Network Node","name":"My Computer","dataStorage":{"type":"Data Storage","name":"New Data Storage","sessionIndependentData":{"type":"Session Independent Data","name":"New Session Independent Data","singleMarketData":[{"type":"Single Market Data","name":"Poloniex BTC/USDT","dataProducts":[{"type":"Data Product","name":"Candles","id":"55e735a5-60c9-4e2d-a582-a6c6eddf3502","savedPayload":{"position":{"x":21869.40956443714,"y":29946.274763371603},"targetPosition":{"x":21481.757830574268,"y":29392.650712345214},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":3,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Candles","id":"36ae2d5d-6734-4526-b2f8-31c89a0c4fc1"}}},{"type":"Data Product","name":"Volumes","id":"ffaf5a8e-12d6-4040-922d-57845eb3377c","savedPayload":{"position":{"x":21767.384455977735,"y":30005.17898720474},"targetPosition":{"x":21481.757830574268,"y":29392.650712345214},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":3,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Volumes","id":"b2d5ba69-67ac-490f-9e74-28239b35a92e"}}},{"type":"Data Product","name":"Candle-Stairs","id":"e1c46e1f-09d1-46a2-b738-e761b4e04cd9","savedPayload":{"position":{"x":21656.680727039493,"y":30045.471849360536},"targetPosition":{"x":21481.757830574268,"y":29392.650712345214},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":3,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Candle-Stairs","id":"642474e9-da61-4168-97dc-7f7869bebc02"}}},{"type":"Data Product","name":"Volume-Stairs","id":"eefaf182-ffe4-457e-a7fa-f43dfabdc086","savedPayload":{"position":{"x":21540.662054407407,"y":30065.929071611557},"targetPosition":{"x":21481.757830574268,"y":29392.650712345214},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":3,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Volume-Stairs","id":"846e9572-c4d6-4c7e-942e-c82402c5ecea"}}},{"type":"Data Product","name":"Bollinger Bands","id":"3576cfd0-1774-4094-b8a0-66e182fc4c41","savedPayload":{"position":{"x":21422.85360674113,"y":30065.929071611557},"targetPosition":{"x":21481.757830574268,"y":29392.650712345214},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":3,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Bollinger Bands","id":"29088877-8d63-4217-8d34-8742afc262f4"}}},{"type":"Data Product","name":"Percentage Bandwidth","id":"4dfde44c-0bbf-4204-91de-ce99ecba4cf2","savedPayload":{"position":{"x":21306.834934109043,"y":30045.471849360536},"targetPosition":{"x":21481.757830574268,"y":29392.650712345214},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":3,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Percentage Bandwidth","id":"3b7885a0-cd72-4a3f-a38b-3dc8c47bd068"}}},{"type":"Data Product","name":"Bollinger Channels Objects","id":"6986bcb1-119c-48c9-a00b-d046e9e07951","savedPayload":{"position":{"x":21196.1312051708,"y":30005.17898720474},"targetPosition":{"x":21481.757830574268,"y":29392.650712345214},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":3,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Bollinger Channels Objects","id":"1f111f72-624f-4166-9f99-ab8d785d5267"}}},{"type":"Data Product","name":"Bollinger Sub-Channels Objects","id":"85098ab8-a125-49f1-b24d-416c2090c7a0","savedPayload":{"position":{"x":21094.106096711395,"y":29946.274763371603},"targetPosition":{"x":21481.757830574268,"y":29392.650712345214},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":3,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Bollinger Sub-Channels Objects","id":"080c87fc-4546-4482-ab11-180a33f6d9ab"}}}],"id":"49a747b4-2eb7-41d6-b0e2-493f43ceed44","savedPayload":{"position":{"x":21481.757830574268,"y":29392.650712345214},"targetPosition":{"x":21481.757830574268,"y":28716.800539125023},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":2,"distanceToParent":4},"uiObject":{},"referenceParent":{"type":"Market","name":"BTC/USDT","id":"d4098b73-7e25-434e-8fd9-342393cbb400"}}},{"type":"Single Market Data","name":"Kraken ETH/BTC","dataProducts":[{"type":"Data Product","name":"Candles","id":"fce50098-d674-4006-94ad-cce46f3d59a4","savedPayload":{"position":{"x":21134.859543298608,"y":30011.44257229641},"targetPosition":{"x":21223.08725836212,"y":29341.286542746402},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":trLine truncated
{"type":"Workspace","name":"My Workspace","rootNodes":[{"type":"Network","name":"Superalgos","networkNodes":[{"type":"Network Node","name":"My Computer","dataStorage":{"type":"Data Storage","name":"New Data Storage","sessionIndependentData":{"type":"Session Independent Data","name":"New Session Independent Data","singleMarketData":[{"type":"Single Market Data","name":"Bitfinex BTC/USDT","dataProducts":[{"type":"Data Product","name":"Candles","id":"55a6b1fd-aed5-47d3-aecc-ad0de070be4c","savedPayload":{"position":{"x":24303.60676224509,"y":29850.452119703175},"targetPosition":{"x":24484.32803619672,"y":29416.045717149693},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":2,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Candles","id":"fe6065ec-4d2d-47a0-bae0-8647f0e195d7"}}},{"type":"Data Product","name":"Volumes","id":"d1d62165-35ea-49ec-90a4-43cadc0bd3a9","savedPayload":{"position":{"x":24226.861919149702,"y":29809.848075903006},"targetPosition":{"x":24484.32803619672,"y":29416.045717149693},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":2,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Volumes","id":"1a366aef-efef-470e-88e0-2a6fbe4dd91f"}}},{"type":"Data Product","name":"Candle-Stairs","id":"545afc08-f41f-4903-8264-82025a294487","savedPayload":{"position":{"x":24158.884776317158,"y":29755.833565195368},"targetPosition":{"x":24484.32803619672,"y":29416.045717149693},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":2,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Candle-Stairs","id":"642474e9-da61-4168-97dc-7f7869bebc02"}}},{"type":"Data Product","name":"Volume-Stairs","id":"b560fe82-8a81-4bd2-82b7-e1150b77cb76","savedPayload":{"position":{"x":24101.990213817022,"y":29690.247986959144},"targetPosition":{"x":24484.32803619672,"y":29416.045717149693},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":2,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Volume-Stairs","id":"846e9572-c4d6-4c7e-942e-c82402c5ecea"}}},{"type":"Data Product","name":"Bollinger Bands","id":"4625a5d7-a127-4cbd-99e8-cd7ce0397144","savedPayload":{"position":{"x":24058.115707737732,"y":29615.324779399933},"targetPosition":{"x":24484.32803619672,"y":29416.045717149693},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":2,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Bollinger Bands","id":"29088877-8d63-4217-8d34-8742afc262f4"}}},{"type":"Data Product","name":"Percentage Bandwidth","id":"a73f83a6-2331-4a05-a9ae-c58f42d833c5","savedPayload":{"position":{"x":24028.75535176215,"y":29533.61536249069},"targetPosition":{"x":24484.32803619672,"y":29416.045717149693},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":2,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Percentage Bandwidth","id":"3b7885a0-cd72-4a3f-a38b-3dc8c47bd068"}}},{"type":"Data Product","name":"Bollinger Channels Objects","id":"f60ea12a-fb58-49a5-a611-08eef941fd5a","savedPayload":{"position":{"x":24014.908977599152,"y":29447.90225242461},"targetPosition":{"x":24484.32803619672,"y":29416.045717149693},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":2,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Bollinger Channels Objects","id":"1f111f72-624f-4166-9f99-ab8d785d5267"}}},{"type":"Data Product","name":"Bollinger Sub-Channels Objects","id":"8779ef49-1579-41a4-8780-6bc6ee623f47","savedPayload":{"position":{"x":24017.04810691397,"y":29361.104306363446},"targetPosition":{"x":24484.32803619672,"y":29416.045717149693},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":2,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Bollinger Sub-Channels Objects","id":"080c87fc-4546-4482-ab11-180a33f6d9ab"}}},{"type":"Data Product","name":"Popular SMAs","id":"294a25cc-2520-41fc-ad01-eff64573c145","savedPayload":{"position":{"x":24035.09989422339,"y":29276.177324257656},"targetPosition":{"x":24484.32803619672,"y":29416.045717149693},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3},"uiObject":{},"referenceParent":{"type":"Product Definition","name":"Popular SMAs","id":"4029d5bf-a8ad-4d80-b874-2af506e0ab34"}}},{"type":"Data Product","name":"Base7 SMAs","id":"b8672127-0703-455b-b0c4-0ed33b5150e8","savedPayload":{"position":{"x":24068.449607561317,"y":29196.01339262416},"targetPosition":{"x":24484.32803619672,"y":29416.045717149693},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToPaLine truncated