Merge remote-tracking branch 'RawData/develop' into develop
This commit is contained in:
Luis Fernando Molina committed
commit
cef3d925fa
13 files changed
+1551
No files matched your search
@@ -0,0 +1,2 @@
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
|
||||
.vs/*
|
||||
obj/*
|
||||
|
||||
bin/*
|
||||
|
||||
*.config
|
||||
*.log
|
||||
*.cache
|
||||
node_modules/*
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
exports.newCommons = function newCommons(bot, logger, UTILITIES) {
|
||||
|
||||
let thisObject = {
|
||||
|
||||
};
|
||||
|
||||
return thisObject;
|
||||
};
|
||||
@@ -0,0 +1,25 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 15
|
||||
VisualStudioVersion = 15.0.27130.2027
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}") = "Project", "Project.njsproj", "{AE0A3FE2-7021-4CC2-9922-A887CA5CDD7C}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Release|Any CPU = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{AE0A3FE2-7021-4CC2-9922-A887CA5CDD7C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{AE0A3FE2-7021-4CC2-9922-A887CA5CDD7C}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{AE0A3FE2-7021-4CC2-9922-A887CA5CDD7C}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{AE0A3FE2-7021-4CC2-9922-A887CA5CDD7C}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {9128DF06-8CA5-497C-8BF7-B924571C5394}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
@@ -0,0 +1,694 @@
|
||||
|
||||
exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, FILE_STORAGE, STATUS_REPORT, EXCHANGE_API) {
|
||||
|
||||
const FULL_LOG = true;
|
||||
const GMT_SECONDS = ':00.000 GMT+0000';
|
||||
const MODULE_NAME = "User Bot";
|
||||
const CANDLES_FOLDER_NAME = "Candles/One-Min";
|
||||
const VOLUMES_FOLDER_NAME = "Volumes/One-Min";
|
||||
|
||||
thisObject = {
|
||||
initialize: initialize,
|
||||
start: start
|
||||
};
|
||||
|
||||
let utilities = UTILITIES.newCloudUtilities(bot, logger)
|
||||
let fileStorage = FILE_STORAGE.newFileStorage(logger);
|
||||
let statusDependencies
|
||||
|
||||
const ONE_MIN = 60000
|
||||
const ONE_DAY = ONE_MIN * 60 * 24
|
||||
|
||||
const MAX_OHLCVs_PER_EXECUTION = 10000000
|
||||
const symbol = bot.market.baseAsset + '/' + bot.market.quotedAsset
|
||||
const ccxt = require('ccxt')
|
||||
|
||||
let fetchType = "by Time"
|
||||
let lastId
|
||||
let firstId
|
||||
let allOHLCVs = []
|
||||
let thisReport;
|
||||
let since
|
||||
let initialProcessTimestamp
|
||||
let beginingOfMarket
|
||||
let lastFile
|
||||
let exchangeId
|
||||
let options = {}
|
||||
let rateLimit = 500
|
||||
let exchange
|
||||
let uiStartDate = new Date(bot.uiStartDate)
|
||||
let fisrtTimeThisProcessRun = false
|
||||
let limit = 1000 // This is the default value
|
||||
let hostname
|
||||
let lastCandleOfTheDay
|
||||
|
||||
return thisObject;
|
||||
|
||||
function initialize(pStatusDependencies, callBackFunction) {
|
||||
let exchangeClass
|
||||
|
||||
try {
|
||||
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.initialize();
|
||||
|
||||
statusDependencies = pStatusDependencies;
|
||||
|
||||
exchangeId = bot.exchange.toLowerCase().replace(' ', '')
|
||||
|
||||
/* Applying the parameters defined by the user at the Exchange Node Config */
|
||||
if (bot.exchangeNode.config.API !== undefined) {
|
||||
for (let i = 0; i < bot.exchangeNode.config.API.length; i++) {
|
||||
if (bot.exchangeNode.config.API[i].method === 'fetch_ohlcv') {
|
||||
if (bot.exchangeNode.config.API[i].class !== undefined) {
|
||||
exchangeId = bot.exchangeNode.config.API[i].class
|
||||
}
|
||||
if (bot.exchangeNode.config.API[i].fetchOHLCVsMethod !== undefined) {
|
||||
options = {
|
||||
'fetchOHLCVsMethod': bot.exchangeNode.config.API[i].fetchOHLCVsMethod
|
||||
}
|
||||
}
|
||||
if (bot.exchangeNode.config.API[i].firstId !== undefined) {
|
||||
firstId = bot.exchangeNode.config.API[i].firstId
|
||||
}
|
||||
if (bot.exchangeNode.config.API[i].rateLimit !== undefined) {
|
||||
rateLimit = bot.exchangeNode.config.API[i].rateLimit
|
||||
}
|
||||
if (bot.exchangeNode.config.API[i].hostname !== undefined) {
|
||||
hostname = bot.exchangeNode.config.API[i].hostname
|
||||
}
|
||||
if (bot.exchangeNode.config.API[i].fetchType !== undefined) {
|
||||
fetchType = bot.exchangeNode.config.API[i].fetchType
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bot.config.fetchLimit !== undefined) {
|
||||
limit = bot.config.fetchLimit
|
||||
}
|
||||
|
||||
let key = process.env.KEY
|
||||
let secret = process.env.SECRET
|
||||
|
||||
if (key === "undefined") { key = undefined }
|
||||
if (secret === "undefined") { secret = undefined }
|
||||
|
||||
|
||||
exchangeClass = ccxt[exchangeId]
|
||||
const exchangeConstructorParams = {
|
||||
'apiKey': key,
|
||||
'secret': secret,
|
||||
'timeout': 30000,
|
||||
'enableRateLimit': true,
|
||||
verbose: false,
|
||||
options: options
|
||||
}
|
||||
if (rateLimit !== undefined) {
|
||||
exchangeConstructorParams.rateLimit = rateLimit
|
||||
}
|
||||
if (hostname !== undefined) {
|
||||
exchangeConstructorParams.hostname = hostname
|
||||
}
|
||||
|
||||
exchange = new exchangeClass(exchangeConstructorParams)
|
||||
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack);
|
||||
|
||||
/* CCXT Supported Exchanges */
|
||||
console.log('CCXT Library current supported exchanges:')
|
||||
for (const property in ccxt) {
|
||||
console.log(`${property}`);
|
||||
}
|
||||
console.log('For more info please check: https://github.com/ccxt/ccxt/wiki/Manual')
|
||||
console.log('Exchange Class ' + exchangeId)
|
||||
console.log(exchangeClass)
|
||||
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
function start(callBackFunction) {
|
||||
try {
|
||||
|
||||
if (global.STOP_TASK_GRACEFULLY === true) {
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
return
|
||||
}
|
||||
|
||||
let abort = false
|
||||
begin()
|
||||
|
||||
async function begin() {
|
||||
|
||||
getContextVariables()
|
||||
if (abort === true) { return }
|
||||
await getFirstId()
|
||||
await getOHLCVs()
|
||||
if (abort === true) { return }
|
||||
if (global.STOP_TASK_GRACEFULLY === true) {
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
return
|
||||
}
|
||||
await saveOHLCVs()
|
||||
}
|
||||
|
||||
function getContextVariables() {
|
||||
|
||||
try {
|
||||
let reportKey;
|
||||
|
||||
reportKey = "Masters" + "-" + "Exchange-Raw-Data" + "-" + "Historic-OHLCVs"
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
|
||||
|
||||
if (statusDependencies.statusReports.get(reportKey).status === "Status Report is corrupt.") {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
thisReport = statusDependencies.statusReports.get(reportKey)
|
||||
|
||||
if (thisReport.file.beginingOfMarket !== undefined) { // This means this is not the first time this process run.
|
||||
beginingOfMarket = new Date(thisReport.file.beginingOfMarket.year + "-" + thisReport.file.beginingOfMarket.month + "-" + thisReport.file.beginingOfMarket.days + " " + thisReport.file.beginingOfMarket.hours + ":" + thisReport.file.beginingOfMarket.minutes + GMT_SECONDS);
|
||||
lastFile = new Date(thisReport.file.lastFile.year + "-" + thisReport.file.lastFile.month + "-" + thisReport.file.lastFile.days + " " + thisReport.file.lastFile.hours + ":" + thisReport.file.lastFile.minutes + GMT_SECONDS);
|
||||
lastId = thisReport.file.lastId
|
||||
lastCandleOfTheDay = thisReport.file.lastCandleOfTheDay
|
||||
} else { // This means this is the first time this process run.
|
||||
fisrtTimeThisProcessRun = true
|
||||
beginingOfMarket = new Date(uiStartDate.valueOf())
|
||||
}
|
||||
|
||||
defineSince()
|
||||
function defineSince() {
|
||||
if (thisReport.file.uiStartDate === undefined) {
|
||||
thisReport.file.uiStartDate = uiStartDate
|
||||
} else {
|
||||
thisReport.file.uiStartDate = new Date(thisReport.file.uiStartDate)
|
||||
}
|
||||
|
||||
if (uiStartDate.valueOf() !== thisReport.file.uiStartDate.valueOf()) {
|
||||
since = uiStartDate.valueOf()
|
||||
initialProcessTimestamp = since
|
||||
fisrtTimeThisProcessRun = true
|
||||
beginingOfMarket = new Date(uiStartDate.valueOf())
|
||||
} else {
|
||||
if (lastFile !== undefined) {
|
||||
since = lastFile.valueOf()
|
||||
initialProcessTimestamp = lastFile.valueOf()
|
||||
} else {
|
||||
since = uiStartDate.valueOf()
|
||||
initialProcessTimestamp = uiStartDate.valueOf()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> err = " + err.stack);
|
||||
if (err.message === "Cannot read property 'file' of undefined") {
|
||||
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Check the bot Status Dependencies. ");
|
||||
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys));
|
||||
}
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
abort = true
|
||||
}
|
||||
}
|
||||
|
||||
async function getFirstId() {
|
||||
try {
|
||||
/* We need the first id only when we are going to fetch trades based on id and it is the first time the process runs*/
|
||||
if (fetchType !== "by Id") { return }
|
||||
if (lastId !== undefined) { return }
|
||||
lastId = 0
|
||||
|
||||
return
|
||||
/*
|
||||
const limit = 1
|
||||
const exchangeClass = ccxt[exchangeId]
|
||||
const exchange = new exchangeClass({
|
||||
'timeout': 30000,
|
||||
'enableRateLimit': true,
|
||||
verbose: false
|
||||
})
|
||||
|
||||
const OHLCVs = await exchange.fetchOHLCV(symbol, '1m', since, limit, undefined)
|
||||
|
||||
let lastRecord = OHLCVs[OHLCVs.length - 1]
|
||||
lastId = lastRecord.info[firstId]
|
||||
*/
|
||||
} catch (err) {
|
||||
/* If something fails trying to get an id close to since, we just will continue without an id.*/
|
||||
}
|
||||
}
|
||||
|
||||
async function getOHLCVs() {
|
||||
|
||||
try {
|
||||
let lastOHLCVKey = ''
|
||||
let params = undefined
|
||||
let previousSince
|
||||
let fromDate = new Date(since)
|
||||
let lastDate = new Date()
|
||||
|
||||
while (true) {
|
||||
|
||||
/* Reporting we are doing well */
|
||||
function heartBeat(noNewInternalLoop) {
|
||||
let processingDate = new Date(since)
|
||||
processingDate = processingDate.getUTCFullYear() + '-' + utilities.pad(processingDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(processingDate.getUTCDate(), 2);
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getOHLCVs -> Fetching OHLCVs @ " + processingDate + "-> exchange = " + bot.exchange + " -> symbol = " + symbol + " -> since = " + since + " -> limit = " + limit) }
|
||||
let heartBeatText = "Fetching " + allOHLCVs.length.toFixed(0) + " OHLCVs from " + bot.exchange + " " + symbol + " @ " + processingDate
|
||||
let currentDate = new Date(since)
|
||||
let percentage = global.getPercentage(fromDate, currentDate, lastDate)
|
||||
bot.processHeartBeat(heartBeatText, percentage) // tell the world we are alive and doing well
|
||||
if (global.areEqualDates(currentDate, new Date()) === false) {
|
||||
if (noNewInternalLoop !== true) {
|
||||
logger.newInternalLoop(bot.codeName, bot.process, currentDate, percentage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
heartBeat()
|
||||
|
||||
/* Defining if we will query the exchange by Date or Id */
|
||||
if (fetchType === "by Id") {
|
||||
/*
|
||||
params = {
|
||||
'fromId': lastId
|
||||
}
|
||||
*/
|
||||
since = lastId
|
||||
}
|
||||
|
||||
/* Fetching the OHLCVs from the exchange.*/
|
||||
await new Promise(resolve => setTimeout(resolve, rateLimit)) // rate limit
|
||||
const OHLCVs = await exchange.fetchOHLCV(symbol, '1m', since, limit, params)
|
||||
|
||||
/*
|
||||
OHLCV Structure
|
||||
The fetchOHLCV method shown above returns a list (a flat array) of OHLCV candles represented by the following structure:
|
||||
|
||||
[
|
||||
[
|
||||
1504541580000, // UTC timestamp in milliseconds, integer
|
||||
4235.4, // (O)pen price, float
|
||||
4240.6, // (H)ighest price, float
|
||||
4230.0, // (L)owest price, float
|
||||
4230.7, // (C)losing price, float
|
||||
37.72941911 // (V)olume (in terms of the base currency), float
|
||||
],
|
||||
...
|
||||
]
|
||||
*/
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getOHLCVs -> OHLCVs Fetched = " + OHLCVs.length) }
|
||||
if (OHLCVs.length > 0) {
|
||||
let beginDate = new Date(OHLCVs[0].timestamp)
|
||||
let endDate = new Date(OHLCVs[OHLCVs.length - 1].timestamp)
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getOHLCVs -> OHLCVs Fetched From " + beginDate + " -> timestamp = " + OHLCVs[0].timestamp) }
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getOHLCVs -> OHLCVs Fetched to " + endDate + " -> timestamp = " + OHLCVs[OHLCVs.length - 1].timestamp) }
|
||||
|
||||
if (fisrtTimeThisProcessRun === true) {
|
||||
let OHLCV = OHLCVs[0]
|
||||
|
||||
initialProcessTimestamp = OHLCV[0] // 'timestamp'
|
||||
beginingOfMarket = new Date(Math.trunc(OHLCV[0] / ONE_DAY) * ONE_DAY) // 'timestamp'
|
||||
fromDate = new Date(beginingOfMarket.valueOf())
|
||||
fisrtTimeThisProcessRun = false
|
||||
}
|
||||
}
|
||||
|
||||
if (OHLCVs.length > 1) {
|
||||
previousSince = since
|
||||
since = OHLCVs[OHLCVs.length - 1][0] // 'timestamp'
|
||||
if (since === previousSince) {
|
||||
since++ // this prevents requesting in a loop OHLCVs with the same timestamp, that can happen when all the records fetched comes with exactly the same timestamp.
|
||||
}
|
||||
|
||||
|
||||
lastId = OHLCVs[OHLCVs.length - 1]['id']
|
||||
|
||||
for (let i = 0; i < OHLCVs.length; i++) {
|
||||
|
||||
let OHLCV = OHLCVs[i]
|
||||
|
||||
let OHLCVKey = OHLCV[0] + '-' + OHLCV[1].toFixed(16) + '-' + OHLCV[2].toFixed(16) + '-' + OHLCV[3].toFixed(16) + '-' + OHLCV[4].toFixed(16) + '-' + OHLCV[5].toFixed(16)
|
||||
if (OHLCVKey !== lastOHLCVKey) {
|
||||
allOHLCVs.push(OHLCV)
|
||||
}
|
||||
lastOHLCVKey = OHLCVKey
|
||||
}
|
||||
|
||||
heartBeat(true)
|
||||
}
|
||||
|
||||
if (
|
||||
OHLCVs.length < limit - 1 ||
|
||||
global.STOP_TASK_GRACEFULLY === true ||
|
||||
allOHLCVs.length >= MAX_OHLCVs_PER_EXECUTION
|
||||
) {
|
||||
break
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err.stack.toString().indexOf('ERR_RATE_LIMIT') >= 0) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getOHLCVs -> Retrying Later -> The Exchange " + bot.exchange + " is saying you are requesting data too often. I will retry the request later, no action is required. To avoid this happening again please increase the rateLimit at the Exchange node config. You might continue seeing this if you are retrieving data from multiple markets at the same time. In this case I tried to get 1 min OHLCVs from " + symbol);
|
||||
return
|
||||
}
|
||||
|
||||
if (err.stack.toString().indexOf('RequestTimeout') >= 0) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getOHLCVs -> Retrying Later -> The Exchange " + bot.exchange + " is not responding at the moment. I will save the data already fetched and try to reconnect later to fetch the rest of the missing data.");
|
||||
return
|
||||
}
|
||||
|
||||
if (err.stack.toString().indexOf('ExchangeNotAvailable') >= 0) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getOHLCVs -> Retrying Later -> The Exchange " + bot.exchange + " is not available at the moment. I will save the data already fetched and try to reconnect later to fetch the rest of the missing data.");
|
||||
return
|
||||
}
|
||||
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getOHLCVs -> Retrying Later -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
abort = true
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
async function saveOHLCVs() {
|
||||
|
||||
try {
|
||||
|
||||
let candlesFileContent = '['
|
||||
let volumesFileContent = '['
|
||||
let previousDay = Math.trunc((initialProcessTimestamp - ONE_DAY) / ONE_DAY)
|
||||
let currentDay = Math.trunc((initialProcessTimestamp - ONE_DAY) / ONE_DAY)
|
||||
let needSeparator = false
|
||||
let error
|
||||
let separator
|
||||
let heartBeatCounter = 0
|
||||
let headOfTheMarketReached = false
|
||||
|
||||
let lastCandle = {
|
||||
begin: 0,
|
||||
end: 0,
|
||||
open: 0,
|
||||
close: 0,
|
||||
min: 0,
|
||||
max: 0
|
||||
}
|
||||
|
||||
let lastVolume = {
|
||||
begin: 0,
|
||||
end: 0,
|
||||
buy: 0,
|
||||
sell: 0
|
||||
}
|
||||
|
||||
let i = 0
|
||||
lastId = undefined
|
||||
|
||||
if (lastCandleOfTheDay !== undefined) {
|
||||
lastCandle = JSON.parse(JSON.stringify(lastCandleOfTheDay))
|
||||
}
|
||||
|
||||
controlLoop()
|
||||
|
||||
function loop() {
|
||||
|
||||
let filesToCreate = 0
|
||||
let filesCreated = 0
|
||||
|
||||
/* Go through all possible 1 min candles of a day. */
|
||||
for (let j = 0; j < 60 * 24; j++) {
|
||||
|
||||
let candle = {
|
||||
begin: currentDay * ONE_DAY + ONE_MIN * j,
|
||||
end: currentDay * ONE_DAY + ONE_MIN * j + ONE_MIN - 1,
|
||||
open: lastCandle.close,
|
||||
close: lastCandle.close,
|
||||
min: lastCandle.close,
|
||||
max: lastCandle.close
|
||||
}
|
||||
let volume = {
|
||||
begin: currentDay * ONE_DAY + ONE_MIN * j,
|
||||
end: currentDay * ONE_DAY + ONE_MIN * j + ONE_MIN - 1,
|
||||
buy: lastVolume.buy,
|
||||
sell: lastVolume.sell
|
||||
}
|
||||
|
||||
let processingDate = new Date(candle.begin)
|
||||
processingDate = processingDate.getUTCFullYear() + '-' + utilities.pad(processingDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(processingDate.getUTCDate(), 2);
|
||||
|
||||
if (candle.begin > (new Date()).valueOf()) {
|
||||
/* We stop when the current candle is pointing to a time in the future.*/
|
||||
headOfTheMarketReached = true
|
||||
saveFile(currentDay)
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> saveOHLCVs -> Before Fetch -> Saving OHLCVs @ " + processingDate + " -> i = " + i + " -> total = " + allOHLCVs.length) }
|
||||
bot.processHeartBeat("Saving " + i.toFixed(0) + " / " + allOHLCVs.length + " OHLCVs from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
|
||||
|
||||
/* We exit the loop and we aint comming back*/
|
||||
return
|
||||
}
|
||||
|
||||
let OHLCV = { // these values should be overrided unless there are no OHLVCs fetched from the exchange.
|
||||
timestamp: (new Date()).valueOf() + ONE_MIN,
|
||||
open: 0,
|
||||
hight: 0,
|
||||
low: 0,
|
||||
close: 0,
|
||||
volume: 0
|
||||
}
|
||||
|
||||
let record = allOHLCVs[i]
|
||||
|
||||
if (record !== undefined) { // can be undefined if there is no candle fetched from the exchange
|
||||
OHLCV = {
|
||||
timestamp: record[0],
|
||||
open: record[1],
|
||||
hight: record[2],
|
||||
low: record[3],
|
||||
close: record[4],
|
||||
volume: record[5]
|
||||
}
|
||||
}
|
||||
|
||||
let candleMinute = Math.trunc(candle.begin / ONE_MIN)
|
||||
let OHLCVMinute
|
||||
|
||||
checkOHLCVMinute()
|
||||
|
||||
function checkOHLCVMinute() {
|
||||
/*
|
||||
Some exchanges return inconsistent data. It is not guaranteed that each candle will have a timeStamp exactly at the begining of an
|
||||
UTC minute. It is also not guaranteed that the distance between timestamps will be the same. To fix this, we will do this.
|
||||
*/
|
||||
|
||||
OHLCVMinute = Math.trunc(OHLCV.timestamp / ONE_MIN)
|
||||
|
||||
if (OHLCVMinute < candleMinute) {
|
||||
if (i >= allOHLCVs.length - 1) {
|
||||
/* We run out of OHLCVs, we can not move to the next OHLCV, we leave this function. */
|
||||
return
|
||||
}
|
||||
|
||||
i++
|
||||
record = allOHLCVs[i]
|
||||
OHLCV = {
|
||||
timestamp: record[0],
|
||||
open: record[1],
|
||||
hight: record[2],
|
||||
low: record[3],
|
||||
close: record[4],
|
||||
volume: record[5],
|
||||
id: record[6]
|
||||
}
|
||||
checkOHLCVMinute()
|
||||
}
|
||||
}
|
||||
|
||||
/* Reporting we are doing well */
|
||||
heartBeatCounter--
|
||||
if (heartBeatCounter <= 0) {
|
||||
heartBeatCounter = 1440
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> saveOHLCVs -> After Fetch -> Saving OHLCVs @ " + processingDate + " -> i = " + i + " -> total = " + allOHLCVs.length) }
|
||||
bot.processHeartBeat("Saving " + i.toFixed(0) + " / " + allOHLCVs.length + " OHLCVs from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
|
||||
}
|
||||
/* End Reporting */
|
||||
|
||||
if (candleMinute === OHLCVMinute) {
|
||||
candle.open = OHLCV.open
|
||||
candle.close = OHLCV.close
|
||||
candle.min = OHLCV.low
|
||||
candle.max = OHLCV.hight
|
||||
volume.buy = OHLCV.volume / 2
|
||||
volume.sell = OHLCV.volume / 2
|
||||
|
||||
if (i < allOHLCVs.length - 1) {
|
||||
i++
|
||||
}
|
||||
|
||||
lastId = OHLCV.id
|
||||
|
||||
}
|
||||
lastCandle = candle
|
||||
|
||||
if (needSeparator === false) {
|
||||
needSeparator = true;
|
||||
separator = '';
|
||||
} else {
|
||||
separator = ',';
|
||||
}
|
||||
|
||||
/* Add the candle to the file content.*/
|
||||
candlesFileContent = candlesFileContent + separator + '[' + candle.min + "," + candle.max + "," + candle.open + "," + candle.close + "," + candle.begin + "," + candle.end + "]";
|
||||
volumesFileContent = volumesFileContent + separator + '[' + volume.buy + "," + volume.sell + "," + volume.begin + "," + volume.end + "]";
|
||||
|
||||
/* We store the last candle of the day in order to have a previous candles during next execution. */
|
||||
if (j === 1440 - 1) {
|
||||
lastCandleOfTheDay = JSON.parse(JSON.stringify(candle))
|
||||
}
|
||||
}
|
||||
|
||||
previousDay = currentDay
|
||||
|
||||
if (i > 0) { // Only start saving at the day of the first candle.
|
||||
saveFile(currentDay)
|
||||
return
|
||||
}
|
||||
|
||||
controlLoop()
|
||||
|
||||
function saveFile(day) {
|
||||
candlesFileContent = candlesFileContent + ']'
|
||||
volumesFileContent = volumesFileContent + ']'
|
||||
|
||||
let fileName = 'Data.json'
|
||||
|
||||
filesToCreate++
|
||||
fileStorage.createTextFile(getFilePath(day * ONE_DAY, CANDLES_FOLDER_NAME) + '/' + fileName, candlesFileContent + '\n', onFileCreated);
|
||||
|
||||
filesToCreate++
|
||||
fileStorage.createTextFile(getFilePath(day * ONE_DAY, VOLUMES_FOLDER_NAME) + '/' + fileName, volumesFileContent + '\n', onFileCreated);
|
||||
|
||||
candlesFileContent = '['
|
||||
volumesFileContent = '['
|
||||
needSeparator = false
|
||||
|
||||
}
|
||||
|
||||
function onFileCreated(err) {
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> OHLCVsReadyToBeSaved -> onFileBCreated -> err = " + JSON.stringify(err));
|
||||
error = err // This allows the loop to be breaked.
|
||||
return;
|
||||
}
|
||||
filesCreated++
|
||||
lastFile = new Date((currentDay * ONE_DAY))
|
||||
if (filesCreated === filesToCreate) {
|
||||
controlLoop()
|
||||
}
|
||||
}
|
||||
|
||||
function getFilePath(timestamp, folderName) {
|
||||
let datetime = new Date(timestamp)
|
||||
let dateForPath = datetime.getUTCFullYear() + '/' +
|
||||
utilities.pad(datetime.getUTCMonth() + 1, 2) + '/' +
|
||||
utilities.pad(datetime.getUTCDate(), 2)
|
||||
let filePath = bot.filePathRoot + "/Output/" + folderName + '/' + dateForPath;
|
||||
return filePath
|
||||
}
|
||||
}
|
||||
|
||||
function controlLoop() {
|
||||
if (global.STOP_TASK_GRACEFULLY === true) {
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
return
|
||||
}
|
||||
|
||||
if (error) {
|
||||
callBackFunction(error);
|
||||
return;
|
||||
}
|
||||
|
||||
currentDay++
|
||||
|
||||
/*
|
||||
One possible exit is when we reached the amount of candles downloaded. This not necesary happens at the end of the market
|
||||
if the process was canceled for any reason at the middle.
|
||||
*/
|
||||
|
||||
if (i >= allOHLCVs.length - 1) {
|
||||
writeStatusReport()
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> saveOHLCVs -> controlLoop -> Exit because i reached the end of the candles array. ") }
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
/* The natural exit is at the end of the market */
|
||||
if (headOfTheMarketReached === false) {
|
||||
setImmediate(loop)
|
||||
} else {
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> saveOHLCVs -> controlLoop -> Exit because we reached the end of the market. ") }
|
||||
writeStatusReport()
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> saveOHLCVs -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
abort = true
|
||||
}
|
||||
}
|
||||
|
||||
function writeStatusReport() {
|
||||
try {
|
||||
if (lastFile === undefined) { return }
|
||||
thisReport.file = {
|
||||
lastFile: {
|
||||
year: lastFile.getUTCFullYear(),
|
||||
month: (lastFile.getUTCMonth() + 1),
|
||||
days: lastFile.getUTCDate(),
|
||||
hours: lastFile.getUTCHours(),
|
||||
minutes: lastFile.getUTCMinutes()
|
||||
},
|
||||
beginingOfMarket: {
|
||||
year: beginingOfMarket.getUTCFullYear(),
|
||||
month: (beginingOfMarket.getUTCMonth() + 1),
|
||||
days: beginingOfMarket.getUTCDate(),
|
||||
hours: beginingOfMarket.getUTCHours(),
|
||||
minutes: beginingOfMarket.getUTCMinutes()
|
||||
},
|
||||
uiStartDate: uiStartDate.toUTCString(),
|
||||
lastCandleOfTheDay: lastCandleOfTheDay
|
||||
};
|
||||
|
||||
if (fetchType === "by Id") {
|
||||
thisReport.file.lastId = lastId
|
||||
}
|
||||
|
||||
thisReport.save(onSaved);
|
||||
|
||||
function onSaved(err) {
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> onSaved -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,501 @@
|
||||
|
||||
exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, FILE_STORAGE, STATUS_REPORT, EXCHANGE_API) {
|
||||
|
||||
const FULL_LOG = true;
|
||||
const LOG_FILE_CONTENT = false;
|
||||
const GMT_SECONDS = ':00.000 GMT+0000';
|
||||
const GMT_MILI_SECONDS = '.000 GMT+0000';
|
||||
const MODULE_NAME = "User Bot";
|
||||
const TRADES_FOLDER_NAME = "Trades";
|
||||
|
||||
thisObject = {
|
||||
initialize: initialize,
|
||||
start: start
|
||||
};
|
||||
|
||||
let utilities = UTILITIES.newCloudUtilities(bot, logger)
|
||||
let fileStorage = FILE_STORAGE.newFileStorage(logger);
|
||||
let statusDependencies
|
||||
|
||||
const ONE_MINUTE = 60000
|
||||
const MAX_TRADES_PER_EXECUTION = 100000
|
||||
const symbol = bot.market.baseAsset + '/' + bot.market.quotedAsset
|
||||
const ccxt = require('ccxt')
|
||||
|
||||
let allTrades = []
|
||||
let thisReport;
|
||||
let since
|
||||
let initialProcessTimestamp
|
||||
let beginingOfMarket
|
||||
let lastFileSaved
|
||||
let exchangeId
|
||||
let options = {}
|
||||
let fetchType = "by Time"
|
||||
let lastId
|
||||
let firstId
|
||||
let rateLimit
|
||||
let exchange
|
||||
let uiStartDate = new Date(bot.uiStartDate)
|
||||
|
||||
const limit = 1000
|
||||
|
||||
return thisObject;
|
||||
|
||||
function initialize(pStatusDependencies, callBackFunction) {
|
||||
try {
|
||||
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.initialize();
|
||||
|
||||
statusDependencies = pStatusDependencies;
|
||||
|
||||
exchangeId = bot.exchange.toLowerCase()
|
||||
|
||||
/* Applying the parameters defined by the user at the Exchange node */
|
||||
if (bot.exchangeNode.config.API !== undefined) {
|
||||
for (let i = 0; i < bot.exchangeNode.config.API.length; i++) {
|
||||
if (bot.exchangeNode.config.API[i].method === 'fetchTrades') {
|
||||
if (bot.exchangeNode.config.API[i].class !== undefined) {
|
||||
exchangeId = bot.exchangeNode.config.API[i].class
|
||||
}
|
||||
if (bot.exchangeNode.config.API[i].fetchType !== undefined) {
|
||||
fetchType = bot.exchangeNode.config.API[i].fetchType
|
||||
}
|
||||
if (bot.exchangeNode.config.API[i].fetchTradesMethod !== undefined) {
|
||||
options = {
|
||||
'fetchTradesMethod': bot.exchangeNode.config.API[i].fetchTradesMethod
|
||||
}
|
||||
}
|
||||
if (bot.exchangeNode.config.API[i].firstId !== undefined) {
|
||||
firstId = bot.exchangeNode.config.API[i].firstId
|
||||
}
|
||||
if (bot.exchangeNode.config.API[i].rateLimit !== undefined) {
|
||||
rateLimit = bot.exchangeNode.config.API[i].rateLimit
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let key = process.env.KEY
|
||||
let secret = process.env.SECRET
|
||||
|
||||
if (key === "undefined") { key = undefined }
|
||||
if (secret === "undefined") { secret = undefined }
|
||||
|
||||
|
||||
const exchangeClass = ccxt[exchangeId]
|
||||
const exchangeConstructorParams = {
|
||||
'apiKey': key,
|
||||
'secret': secret,
|
||||
'timeout': 30000,
|
||||
'enableRateLimit': true,
|
||||
verbose: false,
|
||||
options: options
|
||||
}
|
||||
if (rateLimit !== undefined) {
|
||||
exchangeConstructorParams.rateLimit = rateLimit
|
||||
}
|
||||
|
||||
exchange = new exchangeClass(exchangeConstructorParams)
|
||||
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
function start(callBackFunction) {
|
||||
try {
|
||||
|
||||
if (global.STOP_TASK_GRACEFULLY === true) {
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
return
|
||||
}
|
||||
|
||||
let abort = false
|
||||
begin()
|
||||
|
||||
async function begin() {
|
||||
|
||||
getContextVariables()
|
||||
if (abort === true) { return }
|
||||
await getFirstId()
|
||||
await getTrades()
|
||||
if (abort === true) { return}
|
||||
if (global.STOP_TASK_GRACEFULLY === true) {
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
return
|
||||
}
|
||||
await saveTrades()
|
||||
}
|
||||
|
||||
function getContextVariables() {
|
||||
|
||||
try {
|
||||
let reportKey;
|
||||
|
||||
reportKey = "Masters" + "-" + "Exchange-Raw-Data" + "-" + "Historic-Trades"
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
|
||||
|
||||
if (statusDependencies.statusReports.get(reportKey).status === "Status Report is corrupt.") {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
thisReport = statusDependencies.statusReports.get(reportKey)
|
||||
|
||||
if (thisReport.file.beginingOfMarket !== undefined) { // This means this is not the first time this process run.
|
||||
beginingOfMarket = new Date(thisReport.file.beginingOfMarket.year + "-" + thisReport.file.beginingOfMarket.month + "-" + thisReport.file.beginingOfMarket.days + " " + thisReport.file.beginingOfMarket.hours + ":" + thisReport.file.beginingOfMarket.minutes + GMT_SECONDS);
|
||||
lastFileSaved = new Date(thisReport.file.lastFileSaved.year + "-" + thisReport.file.lastFileSaved.month + "-" + thisReport.file.lastFileSaved.days + " " + thisReport.file.lastFileSaved.hours + ":" + thisReport.file.lastFileSaved.minutes + GMT_SECONDS);
|
||||
lastId = thisReport.file.lastId
|
||||
} else { // This means this is the first time this process run.
|
||||
beginingOfMarket = new Date(uiStartDate.valueOf())
|
||||
}
|
||||
|
||||
defineSince()
|
||||
function defineSince() {
|
||||
if (thisReport.file.uiStartDate === undefined) {
|
||||
thisReport.file.uiStartDate = uiStartDate
|
||||
} else {
|
||||
thisReport.file.uiStartDate = new Date(thisReport.file.uiStartDate)
|
||||
}
|
||||
|
||||
if (uiStartDate.valueOf() !== thisReport.file.uiStartDate.valueOf()) {
|
||||
since = uiStartDate.valueOf()
|
||||
initialProcessTimestamp = since
|
||||
beginingOfMarket = new Date(uiStartDate.valueOf())
|
||||
} else {
|
||||
if (lastFileSaved !== undefined) {
|
||||
since = lastFileSaved.valueOf()
|
||||
initialProcessTimestamp = lastFileSaved.valueOf()
|
||||
} else {
|
||||
since = uiStartDate.valueOf()
|
||||
initialProcessTimestamp = uiStartDate.valueOf()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> err = " + err.stack);
|
||||
if (err.message === "Cannot read property 'file' of undefined") {
|
||||
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Check the bot Status Dependencies. ");
|
||||
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys));
|
||||
}
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
abort = true
|
||||
}
|
||||
}
|
||||
|
||||
async function getFirstId() {
|
||||
try {
|
||||
/* We need the first id only when we are going to fetch trades based on id and it is the first time the process runs*/
|
||||
if (fetchType !== "by Id") { return }
|
||||
if (lastId !== undefined) { return }
|
||||
|
||||
const limit = 1
|
||||
const exchangeClass = ccxt[exchangeId]
|
||||
const exchange = new exchangeClass({
|
||||
'timeout': 30000,
|
||||
'enableRateLimit': true,
|
||||
verbose: false
|
||||
})
|
||||
|
||||
const trades = await exchange.fetchTrades(symbol, since, limit, undefined)
|
||||
|
||||
let lastRecord = trades[trades.length - 1]
|
||||
lastId = lastRecord.info[firstId]
|
||||
} catch (err) {
|
||||
/* If something fails trying to get an id close to since, we just will continue without an id.*/
|
||||
}
|
||||
}
|
||||
|
||||
async function getTrades() {
|
||||
|
||||
try {
|
||||
let lastTradeKey = ''
|
||||
let params = undefined
|
||||
let previousSince
|
||||
|
||||
while (true) {
|
||||
|
||||
/* Reporting we are doing well */
|
||||
let processingDate = new Date(since)
|
||||
processingDate = processingDate.getUTCFullYear() + '-' + utilities.pad(processingDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(processingDate.getUTCDate(), 2);
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getTrades -> Fetching Trades @ " + processingDate + "-> exchange = " + bot.exchange + " -> symbol = " + symbol + " -> since = " + since + " -> limit = " + limit ) }
|
||||
bot.processHeartBeat("Fetching " + allTrades.length.toFixed(0) + " trades from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
|
||||
|
||||
if (fetchType === "by Id") {
|
||||
params = {
|
||||
'fromId': lastId
|
||||
}
|
||||
since = undefined
|
||||
}
|
||||
|
||||
/* Fetching the trades from the exchange.*/
|
||||
const trades = await exchange.fetchTrades(symbol, since, limit, params)
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getTrades -> Trades Fetched = " + trades.length) }
|
||||
if (trades.length > 0) {
|
||||
let beginDate = new Date(trades[0].timestamp)
|
||||
let endDate = new Date(trades[trades.length - 1].timestamp)
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getTrades -> Trades Fetched From " + beginDate) }
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getTrades -> Trades Fetched to " + endDate) }
|
||||
}
|
||||
|
||||
if (trades.length > 1) {
|
||||
previousSince = since
|
||||
since = trades[trades.length - 1]['timestamp']
|
||||
if (since === previousSince) {
|
||||
since++ // this prevents requesting in a loop trades with the same timestamp, that can happen when all the records fetched comes with exactly the same timestamp.
|
||||
}
|
||||
|
||||
lastId = trades[trades.length - 1]['id']
|
||||
|
||||
for (let i = 0; i < trades.length; i++) {
|
||||
let trade = trades[i]
|
||||
let tradeKey = trade.id + '-' + trade.timestamp + '-' + trade.side + '-' + trade.price.toFixed(16) + '-' + trade.amount.toFixed(16)
|
||||
if (tradeKey !== lastTradeKey) {
|
||||
allTrades.push([trade.timestamp, trade.side, trade.price, trade.amount, trade.id])
|
||||
}
|
||||
lastTradeKey = tradeKey
|
||||
}
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getTrades -> Fetching Trades @ " + processingDate + "-> exchange = " + bot.exchange + " -> symbol = " + symbol + " -> since = " + since + " -> limit = " + limit) }
|
||||
bot.processHeartBeat("Fetching " + allTrades.length.toFixed(0) + " trades from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
|
||||
|
||||
}
|
||||
|
||||
if (
|
||||
trades.length < limit - 1 ||
|
||||
global.STOP_TASK_GRACEFULLY === true ||
|
||||
allTrades.length >= MAX_TRADES_PER_EXECUTION
|
||||
) {
|
||||
break
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getTrades -> Retrying Later -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
abort = true
|
||||
}
|
||||
}
|
||||
|
||||
async function saveTrades() {
|
||||
|
||||
try {
|
||||
|
||||
let fileContent = '['
|
||||
let previousRecordMinute = Math.trunc((initialProcessTimestamp - ONE_MINUTE) / ONE_MINUTE)
|
||||
let currentRecordMinute
|
||||
let needSeparator = false
|
||||
let error
|
||||
let separator
|
||||
let heartBeatCounter = 0
|
||||
|
||||
let i = -1
|
||||
lastId = undefined
|
||||
controlLoop()
|
||||
|
||||
function loop() {
|
||||
|
||||
let filesToCreate = 0
|
||||
let filesCreated = 0
|
||||
|
||||
let record = allTrades[i]
|
||||
let trade = {
|
||||
timestamp: record[0],
|
||||
side: record[1],
|
||||
price: record[2],
|
||||
amount: record[3]
|
||||
}
|
||||
|
||||
if (lastId === undefined) {
|
||||
lastId = record[4]
|
||||
}
|
||||
|
||||
let processingDate = new Date(trade.timestamp)
|
||||
processingDate = processingDate.getUTCFullYear() + '-' + utilities.pad(processingDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(processingDate.getUTCDate(), 2);
|
||||
|
||||
/* Reporting we are doing well */
|
||||
heartBeatCounter--
|
||||
if (heartBeatCounter <= 0) {
|
||||
heartBeatCounter = 1000
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> saveTrades -> Saving Trades @ " + processingDate + " -> i = " + i + " -> total = " + allTrades.length) }
|
||||
bot.processHeartBeat("Saving " + i.toFixed(0) + " trades from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
|
||||
}
|
||||
|
||||
/* Saving the trades in Files*/
|
||||
currentRecordMinute = Math.trunc(trade.timestamp / ONE_MINUTE)
|
||||
|
||||
if (
|
||||
currentRecordMinute !== previousRecordMinute
|
||||
) {
|
||||
/* There are no more trades at this minute or it is the last trade, so we save the file.*/
|
||||
saveFile(previousRecordMinute)
|
||||
lastId = undefined
|
||||
}
|
||||
|
||||
if (needSeparator === false) {
|
||||
needSeparator = true;
|
||||
separator = '';
|
||||
} else {
|
||||
separator = ',';
|
||||
}
|
||||
|
||||
/* Add the trade to the file content.*/
|
||||
fileContent = fileContent + separator + '[' + trade.timestamp + ',"' + trade.side + '",' + trade.price + ',' + trade.amount + ']';
|
||||
|
||||
if (i === allTrades.length - 1) {
|
||||
/* This is the last trade, so we save the file.*/
|
||||
saveFile(currentRecordMinute)
|
||||
/* It might happen that there are several minutes after the last trade without trades. We need to record empty files for them.*/
|
||||
if (allTrades.length < MAX_TRADES_PER_EXECUTION) {
|
||||
let currentTimeMinute = Math.trunc((new Date()).valueOf() / ONE_MINUTE)
|
||||
if (currentTimeMinute - currentRecordMinute > 1) {
|
||||
createMissingEmptyFiles(currentRecordMinute, currentTimeMinute)
|
||||
}
|
||||
}
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> saveTrades -> Saving Trades @ " + processingDate + " -> i = " + i + " -> total = " + allTrades.length) }
|
||||
bot.processHeartBeat("Saving " + i.toFixed(0) + " trades from " + bot.exchange + " " + symbol + " @ " + processingDate) // tell the world we are alive and doing well
|
||||
|
||||
return
|
||||
}
|
||||
previousRecordMinute = currentRecordMinute
|
||||
if (error) {
|
||||
callBackFunction(error);
|
||||
return;
|
||||
}
|
||||
if (filesToCreate === 0) {
|
||||
controlLoop()
|
||||
}
|
||||
function saveFile(minute) {
|
||||
fileContent = fileContent + ']'
|
||||
if (currentRecordMinute - previousRecordMinute > 1) {
|
||||
createMissingEmptyFiles(previousRecordMinute, currentRecordMinute)
|
||||
}
|
||||
let fileName = 'Data.json'
|
||||
if (previousRecordMinute >= initialProcessTimestamp / ONE_MINUTE) {
|
||||
filesToCreate++
|
||||
fileStorage.createTextFile(getFilePath(minute * ONE_MINUTE) + '/' + fileName, fileContent + '\n', onFileCreated);
|
||||
}
|
||||
fileContent = '['
|
||||
needSeparator = false
|
||||
|
||||
}
|
||||
function createMissingEmptyFiles(begin, end) {
|
||||
|
||||
/*
|
||||
If this range is too wide, we will consider this means that the begin is before the begining of this market at this exchange.
|
||||
In that case we will change the begin and the beginingOfMarket
|
||||
*/
|
||||
|
||||
if ((end - begin) / 60 / 24 > 7) {
|
||||
begin = end
|
||||
beginingOfMarket = new Date(end * ONE_MINUTE)
|
||||
}
|
||||
|
||||
for (let j = begin + 1; j < end; j++) {
|
||||
let fileName = 'Data.json'
|
||||
filesToCreate++
|
||||
fileStorage.createTextFile(getFilePath(j * ONE_MINUTE) + '/' + fileName, "[]" + '\n', onFileCreated);
|
||||
|
||||
}
|
||||
}
|
||||
function onFileCreated(err) {
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> tradesReadyToBeSaved -> onFileBCreated -> err = " + JSON.stringify(err));
|
||||
error = err // This allows the loop to be breaked.
|
||||
return;
|
||||
}
|
||||
filesCreated++
|
||||
lastFileSaved = new Date((currentRecordMinute * ONE_MINUTE))
|
||||
if (filesCreated === filesToCreate) {
|
||||
controlLoop()
|
||||
}
|
||||
}
|
||||
function getFilePath(timestamp) {
|
||||
let datetime = new Date(timestamp)
|
||||
let dateForPath = datetime.getUTCFullYear() + '/' +
|
||||
utilities.pad(datetime.getUTCMonth() + 1, 2) + '/' +
|
||||
utilities.pad(datetime.getUTCDate(), 2) + '/' +
|
||||
utilities.pad(datetime.getUTCHours(), 2) + '/' +
|
||||
utilities.pad(datetime.getUTCMinutes(), 2)
|
||||
let filePath = bot.filePathRoot + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath;
|
||||
return filePath
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
function controlLoop() {
|
||||
if (global.STOP_TASK_GRACEFULLY === true) {
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
return
|
||||
}
|
||||
|
||||
i++
|
||||
if (i < allTrades.length) {
|
||||
setImmediate(loop)
|
||||
} else {
|
||||
writeStatusReport()
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> saveTrades -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
abort = true
|
||||
}
|
||||
}
|
||||
|
||||
function writeStatusReport() {
|
||||
try {
|
||||
if (lastFileSaved === undefined) {return}
|
||||
thisReport.file = {
|
||||
lastFileSaved: {
|
||||
year: lastFileSaved.getUTCFullYear(),
|
||||
month: (lastFileSaved.getUTCMonth() + 1),
|
||||
days: lastFileSaved.getUTCDate(),
|
||||
hours: lastFileSaved.getUTCHours(),
|
||||
minutes: lastFileSaved.getUTCMinutes()
|
||||
},
|
||||
beginingOfMarket: {
|
||||
year: beginingOfMarket.getUTCFullYear(),
|
||||
month: (beginingOfMarket.getUTCMonth() + 1),
|
||||
days: beginingOfMarket.getUTCDate(),
|
||||
hours: beginingOfMarket.getUTCHours(),
|
||||
minutes: beginingOfMarket.getUTCMinutes()
|
||||
},
|
||||
uiStartDate: uiStartDate.toUTCString()
|
||||
};
|
||||
|
||||
if (fetchType === "by Id") {
|
||||
thisReport.file.lastId = lastId
|
||||
}
|
||||
|
||||
thisReport.save(onSaved);
|
||||
|
||||
function onSaved(err) {
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> onSaved -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,24 @@
|
||||
Financial Beings License
|
||||
|
||||
Collective Copyright by the Advanced Algos Community
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software including without limitation the rights to use, copy, modify,
|
||||
merge, publish, distribute and sublicense copies of the Software, and to permit
|
||||
persons to whom the Software is furnished to do so, subject to the following
|
||||
conditions:
|
||||
|
||||
1. Using, copying, modifying, publishing, distributing and sublicensing rights
|
||||
are restricted to the Advanced Algos Platform, meaning the rights granted can
|
||||
only be exercised through the Advanced Algos Service.
|
||||
|
||||
2. The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
||||
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -0,0 +1,73 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<ProjectGuid>{ae0a3fe2-7021-4cc2-9922-a887ca5cdd7c}</ProjectGuid>
|
||||
<ProjectHome />
|
||||
<ProjectView>ShowAllFiles</ProjectView>
|
||||
<StartupFile>
|
||||
</StartupFile>
|
||||
<WorkingDirectory>c:\users\luis\source\repos\AACharly\Poloniex-Hole-Fixing</WorkingDirectory>
|
||||
<OutputPath>.</OutputPath>
|
||||
<ProjectTypeGuids>{3AF33F2E-1136-4D97-BBB7-1795711AC8B8};{349c5851-65df-11da-9384-00065b846f21};{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}</ProjectTypeGuids>
|
||||
<VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion>
|
||||
<VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
|
||||
<Name>Project</Name>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)' == 'Debug'" />
|
||||
<PropertyGroup Condition="'$(Configuration)' == 'Release'" />
|
||||
<ItemGroup>
|
||||
<Compile Include="Historic-OHLCVs\User.Bot.js">
|
||||
<SubType>Code</SubType>
|
||||
</Compile>
|
||||
<Compile Include="Historic-Trades\User.Bot.js" />
|
||||
<Content Include="this.bot.config.json" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Folder Include="Historic-Trades\" />
|
||||
<Folder Include="Historic-OHLCVs\" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.Common.targets" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<!--Do not delete the following Import Project. While this appears to do nothing it is a marker for setting TypeScript properties before our import that depends on them.-->
|
||||
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\TypeScript\Microsoft.TypeScript.targets" Condition="False" />
|
||||
<Import Project="$(VSToolsPath)\Node.js Tools\Microsoft.NodejsTools.targets" />
|
||||
<ProjectExtensions>
|
||||
<VisualStudio>
|
||||
<FlavorProperties GUID="{349c5851-65df-11da-9384-00065b846f21}">
|
||||
<WebProjectProperties>
|
||||
<UseIIS>False</UseIIS>
|
||||
<AutoAssignPort>True</AutoAssignPort>
|
||||
<DevelopmentServerPort>0</DevelopmentServerPort>
|
||||
<DevelopmentServerVPath>/</DevelopmentServerVPath>
|
||||
<IISUrl>http://localhost:48022/</IISUrl>
|
||||
<NTLMAuthentication>False</NTLMAuthentication>
|
||||
<UseCustomServer>True</UseCustomServer>
|
||||
<CustomServerUrl>http://localhost:1337</CustomServerUrl>
|
||||
<SaveServerSettingsInUserFile>False</SaveServerSettingsInUserFile>
|
||||
</WebProjectProperties>
|
||||
</FlavorProperties>
|
||||
<FlavorProperties GUID="{349c5851-65df-11da-9384-00065b846f21}" User="">
|
||||
<WebProjectProperties>
|
||||
<StartPageUrl>
|
||||
</StartPageUrl>
|
||||
<StartAction>CurrentPage</StartAction>
|
||||
<AspNetDebugging>True</AspNetDebugging>
|
||||
<SilverlightDebugging>False</SilverlightDebugging>
|
||||
<NativeDebugging>False</NativeDebugging>
|
||||
<SQLDebugging>False</SQLDebugging>
|
||||
<ExternalProgram>
|
||||
</ExternalProgram>
|
||||
<StartExternalURL>
|
||||
</StartExternalURL>
|
||||
<StartCmdLineArguments>
|
||||
</StartCmdLineArguments>
|
||||
<StartWorkingDirectory>
|
||||
</StartWorkingDirectory>
|
||||
<EnableENC>False</EnableENC>
|
||||
<AlwaysStartWebServerOnDebug>False</AlwaysStartWebServerOnDebug>
|
||||
</WebProjectProperties>
|
||||
</FlavorProperties>
|
||||
</VisualStudio>
|
||||
</ProjectExtensions>
|
||||
</Project>
|
||||
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<LastActiveSolutionConfig>Debug|Any CPU</LastActiveSolutionConfig>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,51 @@
|
||||
# Charly
|
||||
|
||||
Charly is a Sensor bot that is capable of extracting trades from exchanges. It has 3 main processes which together creates an ongoing dataset of trades guaranteed to be reliable.
|
||||
|
||||
## Live Trades
|
||||
|
||||
This process extracts the latest 2 minutes of trade records and save 2 files of 1 minute each. The last file written is overwritten at the next run, allowing the process to complete with the trades that happened after the previous execution retrieval of data.
|
||||
|
||||
### Start Mode
|
||||
|
||||
The process need to run every 1 minute in noTime mode.
|
||||
|
||||
```
|
||||
"startMode": {
|
||||
"noTime": {
|
||||
"run": "false"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Hole Fixing
|
||||
|
||||
When the Live Trades process goes down, the exchange goes down or anything in the middle goes down, data from the exchange can not be retrieved. Once Live Trades is running normally again a whole on the dataset appears between the last files saved and the new files after the restart.
|
||||
|
||||
The Hole Fixing process is there to detect those holes and retrieve the missing information, allowing the dataset to be 100% reliable.
|
||||
|
||||
### Start Mode
|
||||
|
||||
This process runs every 1 minute under allMonths start mode, where a yearly range must be specified (initial processing year and final processing year)
|
||||
|
||||
```
|
||||
"startMode": {
|
||||
"allMonths": {
|
||||
"run": "true",
|
||||
"minYear": "2019",
|
||||
"maxYear": "2021"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Historic Trades
|
||||
|
||||
This process is designed to grabb historical trades information from an exchange, starting from the latest trades, and going all the way back to the begining of the market, respecting the exchange limits on trades per batch extracted.
|
||||
|
||||
This process is currently not being mantained, meaning that there is no guarantee that it can still run.
|
||||
|
||||
# Disclaimer
|
||||
|
||||
THE AA MASTERS BOTS AND THEIR ASSOCIATED PRODUCTS AND SERVICES ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, SUITABILITY FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT.
|
||||
|
||||
IN NO EVENT WILL ADVANCED ALGOS BE LIABLE TO ANY PARTY FOR ANY DIRECT, INDIRECT, SPECIAL OR OTHER CONSEQUENTIAL DAMAGES FOR ANY USE OF THE AACLOUD, THE AA ARENA COMPETITION, THE AA MASTERS BOTS, OR ANY OTHER ASSOCIATED SERVICE OR WEB SITE, INCLUDING, WITHOUT LIMITATION, ANY LOST PROFITS, BUSINESS INTERRUPTION, LOSS OF FUNDS, PROGRAMS OR OTHER DATA OR OTHERWISE, EVEN IF WE ARE EXPRESSLY ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
|
||||
Generated
+5
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"name": "AACharly",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 1
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "AACharly",
|
||||
"version": "1.0.0",
|
||||
"description": "Processes that get the trades from Poloniex.",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "Luis-Fernando-Molina",
|
||||
"license": "ISC",
|
||||
"dependencies": {}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
{
|
||||
"displayName": "Exchange Raw Data",
|
||||
"codeName": "Exchange-Raw-Data",
|
||||
"type": "Sensor Bot",
|
||||
"dataMine": "Masters",
|
||||
"profilePicture": "Exchange Raw Data.png",
|
||||
"processes": [
|
||||
{
|
||||
"name": "Live-Trades",
|
||||
"description": "Retrieves the trades done at the current and the previous minute and saves them at the storage account.",
|
||||
"startMode": {
|
||||
"allMonths": {
|
||||
"run": "false",
|
||||
"minYear": "",
|
||||
"maxYear": ""
|
||||
},
|
||||
"oneMonth": {
|
||||
"run": "false",
|
||||
"year": "",
|
||||
"month": ""
|
||||
},
|
||||
"noTime": {
|
||||
"run": "false"
|
||||
},
|
||||
"fixedInterval": {
|
||||
"run": "true",
|
||||
"interval": 60000
|
||||
}
|
||||
},
|
||||
"deadWaitTime": 0,
|
||||
"normalWaitTime": 60000,
|
||||
"retryWaitTime": 10000,
|
||||
"sleepWaitTime": 3600000,
|
||||
"comaWaitTime": 86400000,
|
||||
"statusDependencies": [
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Exchange-Raw-Data",
|
||||
"process": "Live-Trades"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Historic-Trades",
|
||||
"description": "Retrieves and saves the historical trades in batches going backwards from the current time until reaching the begining of the market.",
|
||||
"startMode": {
|
||||
"allMonths": {
|
||||
"run": "false",
|
||||
"minYear": "2014",
|
||||
"maxYear": "2020"
|
||||
},
|
||||
"oneMonth": {
|
||||
"run": "true",
|
||||
"year": "2018",
|
||||
"month": "01"
|
||||
},
|
||||
"noTime": {
|
||||
"run": "false"
|
||||
},
|
||||
"fixedInterval": {
|
||||
"run": "false",
|
||||
"interval": 0
|
||||
}
|
||||
},
|
||||
"deadWaitTime": 0,
|
||||
"normalWaitTime": 60000,
|
||||
"retryWaitTime": 10000,
|
||||
"sleepWaitTime": 3600000,
|
||||
"comaWaitTime": 86400000,
|
||||
"statusDependencies": [
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Exchange-Raw-Data",
|
||||
"process": "Live-Trades"
|
||||
},
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Exchange-Raw-Data",
|
||||
"process": "Historic-Trades"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Hole-Fixing",
|
||||
"description": "Scans the trades saved by the Live Trades and Historic Trades processes searching for missing records. Once a hole is found on the data set, it patches it by retrieving the missing records from the exchange.",
|
||||
"startMode": {
|
||||
"allMonths": {
|
||||
"run": "true",
|
||||
"minYear": "2019",
|
||||
"maxYear": "2019"
|
||||
},
|
||||
"oneMonth": {
|
||||
"run": "false",
|
||||
"year": "2019",
|
||||
"month": "10"
|
||||
},
|
||||
"noTime": {
|
||||
"run": "false"
|
||||
},
|
||||
"fixedInterval": {
|
||||
"run": "false",
|
||||
"interval": 0
|
||||
}
|
||||
},
|
||||
"deadWaitTime": 0,
|
||||
"normalWaitTime": 0,
|
||||
"retryWaitTime": 10000,
|
||||
"sleepWaitTime": 3600000,
|
||||
"comaWaitTime": 86400000,
|
||||
"statusDependencies": [
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Exchange-Raw-Data",
|
||||
"process": "Live-Trades",
|
||||
"waitUntilNextUpdate": true
|
||||
},
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Exchange-Raw-Data",
|
||||
"process": "Historic-Trades"
|
||||
},
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Exchange-Raw-Data",
|
||||
"process": "Hole-Fixing",
|
||||
"dataSetSection": "Month"
|
||||
},
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Exchange-Raw-Data",
|
||||
"process": "Hole-Fixing"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user