Merge remote-tracking branch 'CandlesVolumes/develop' into develop
This commit is contained in:
Luis Fernando Molina committed
commit
9f1ed8b7b8
12 files changed
+2232
No files matched your search
@@ -0,0 +1,2 @@
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
|
||||
.vs/*
|
||||
obj/*
|
||||
|
||||
bin/*
|
||||
|
||||
*.config
|
||||
*.log
|
||||
*.cache
|
||||
node_modules/*
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 15
|
||||
VisualStudioVersion = 15.0.27130.2027
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}") = "Project", "Project.njsproj", "{AA2901BC-8F81-40F0-A402-5CA79A3F22AD}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Release|Any CPU = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{AA2901BC-8F81-40F0-A402-5CA79A3F22AD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{AA2901BC-8F81-40F0-A402-5CA79A3F22AD}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{AA2901BC-8F81-40F0-A402-5CA79A3F22AD}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{AA2901BC-8F81-40F0-A402-5CA79A3F22AD}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {B4BDAEAF-E863-458A-87C1-CC6D8C490784}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
@@ -0,0 +1 @@
|
||||
// Nothing here to run
|
||||
@@ -0,0 +1,24 @@
|
||||
Financial Beings License
|
||||
|
||||
Collective Copyright by the Advanced Algos Community
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software including without limitation the rights to use, copy, modify,
|
||||
merge, publish, distribute and sublicense copies of the Software, and to permit
|
||||
persons to whom the Software is furnished to do so, subject to the following
|
||||
conditions:
|
||||
|
||||
1. Using, copying, modifying, publishing, distributing and sublicensing rights
|
||||
are restricted to the Advanced Algos Platform, meaning the rights granted can
|
||||
only be exercised through the Advanced Algos Service.
|
||||
|
||||
2. The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
||||
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -0,0 +1,812 @@
|
||||
exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, FILE_STORAGE) {
|
||||
|
||||
const FULL_LOG = true;
|
||||
const LOG_FILE_CONTENT = false;
|
||||
|
||||
const GMT_SECONDS = ':00.000 GMT+0000';
|
||||
const GMT_MILI_SECONDS = '.000 GMT+0000';
|
||||
const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000;
|
||||
|
||||
const MODULE_NAME = "User Bot";
|
||||
|
||||
const TRADES_FOLDER_NAME = "Trades";
|
||||
|
||||
const CANDLES_FOLDER_NAME = "Candles";
|
||||
const CANDLES_ONE_MIN = "One-Min";
|
||||
|
||||
const VOLUMES_FOLDER_NAME = "Volumes";
|
||||
const VOLUMES_ONE_MIN = "One-Min";
|
||||
|
||||
thisObject = {
|
||||
initialize: initialize,
|
||||
start: start
|
||||
};
|
||||
|
||||
let utilities = UTILITIES.newCloudUtilities(bot, logger);
|
||||
let fileStorage = FILE_STORAGE.newFileStorage(logger);
|
||||
|
||||
let statusDependencies;
|
||||
let beginingOfMarket
|
||||
|
||||
return thisObject;
|
||||
|
||||
function initialize(pStatusDependencies, callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.initialize();
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
|
||||
|
||||
statusDependencies = pStatusDependencies;
|
||||
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
This process is going to do the following:
|
||||
|
||||
Read the candles and volumes from Exchange Raw Data and produce a file for each day and for each period with candles and volumes.
|
||||
|
||||
*/
|
||||
|
||||
function start(callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); }
|
||||
|
||||
let market = bot.market;
|
||||
|
||||
/* Context Variables */
|
||||
|
||||
let contextVariables = {
|
||||
lastCandleFile: undefined, // Datetime of the last file files sucessfully produced by this process.
|
||||
firstTradeFile: undefined, // Datetime of the first trade file in the whole market history.
|
||||
maxCandleFile: undefined // Datetime of the last file available to be used as an input of this process.
|
||||
};
|
||||
|
||||
getContextVariables();
|
||||
|
||||
function getContextVariables() {
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Entering function."); }
|
||||
|
||||
let thisReport;
|
||||
let reportKey;
|
||||
let statusReport;
|
||||
|
||||
/* We look first for Exchange Raw Data in order to get when the market starts. */
|
||||
|
||||
reportKey = "Masters" + "-" + "Exchange-Raw-Data" + "-" + "Historic-OHLCVs"
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
|
||||
|
||||
statusReport = statusDependencies.statusReports.get(reportKey);
|
||||
|
||||
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
if (statusReport.status === "Status Report is corrupt.") {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
thisReport = statusDependencies.statusReports.get(reportKey).file;
|
||||
|
||||
if (thisReport.beginingOfMarket === undefined) {
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
|
||||
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet.");
|
||||
|
||||
let customOK = {
|
||||
result: global.CUSTOM_OK_RESPONSE.result,
|
||||
message: "Dependency does not exist."
|
||||
}
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
|
||||
callBackFunction(customOK);
|
||||
return;
|
||||
}
|
||||
|
||||
contextVariables.firstTradeFile = new Date(thisReport.beginingOfMarket.year + "-" + thisReport.beginingOfMarket.month + "-" + thisReport.beginingOfMarket.days + " " + thisReport.beginingOfMarket.hours + ":" + thisReport.beginingOfMarket.minutes + GMT_SECONDS);
|
||||
|
||||
/* Second, we get the report from Exchange Raw Data, to know when the marted ends. */
|
||||
|
||||
reportKey = "Masters" + "-" + "Exchange-Raw-Data" + "-" + "Historic-OHLCVs"
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
|
||||
|
||||
statusReport = statusDependencies.statusReports.get(reportKey);
|
||||
|
||||
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
if (statusReport.status === "Status Report is corrupt.") {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
thisReport = statusDependencies.statusReports.get(reportKey).file;
|
||||
|
||||
if (thisReport.lastFile === undefined) {
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
|
||||
|
||||
let customOK = {
|
||||
result: global.CUSTOM_OK_RESPONSE.result,
|
||||
message: "Dependency not ready."
|
||||
}
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
|
||||
callBackFunction(customOK);
|
||||
return;
|
||||
}
|
||||
|
||||
contextVariables.maxCandleFile = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + "00:00" + GMT_SECONDS);
|
||||
|
||||
/* Finally we get our own Status Report. */
|
||||
|
||||
reportKey = "Masters" + "-" + "Candles-Volumes" + "-" + "Multi-Period-Daily"
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
|
||||
|
||||
statusReport = statusDependencies.statusReports.get(reportKey);
|
||||
|
||||
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
if (statusReport.status === "Status Report is corrupt.") {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process.");
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
thisReport = statusDependencies.statusReports.get(reportKey).file;
|
||||
|
||||
if (thisReport.lastFile !== undefined) {
|
||||
logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Process Running not for the very first time. -> reportKey = " + reportKey);
|
||||
|
||||
beginingOfMarket = new Date(thisReport.beginingOfMarket);
|
||||
|
||||
if (beginingOfMarket.valueOf() !== contextVariables.firstTradeFile.valueOf()) { // Reset Mechanism for Begining of the Market
|
||||
logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Reset Mechanism for Begining of the Market Activated. -> reportKey = " + reportKey);
|
||||
|
||||
beginingOfMarket = new Date(contextVariables.firstTradeFile.getUTCFullYear() + "-" + (contextVariables.firstTradeFile.getUTCMonth() + 1) + "-" + contextVariables.firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.firstTradeFile.getUTCFullYear() + "-" + (contextVariables.firstTradeFile.getUTCMonth() + 1) + "-" + contextVariables.firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well.
|
||||
|
||||
buildCandles();
|
||||
return;
|
||||
}
|
||||
|
||||
contextVariables.lastCandleFile = new Date(thisReport.lastFile);
|
||||
|
||||
/*
|
||||
Here we assume that the last day written might contain incomplete information. This actually happens every time the head of the market is reached.
|
||||
For that reason we go back one day, the partial information is discarded and added again with whatever new info is available.
|
||||
*/
|
||||
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS);
|
||||
|
||||
buildCandles();
|
||||
return;
|
||||
|
||||
} else {
|
||||
logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Process Running for the very first time. -> reportKey = " + reportKey);
|
||||
|
||||
beginingOfMarket = new Date(contextVariables.firstTradeFile.getUTCFullYear() + "-" + (contextVariables.firstTradeFile.getUTCMonth() + 1) + "-" + contextVariables.firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.firstTradeFile.getUTCFullYear() + "-" + (contextVariables.firstTradeFile.getUTCMonth() + 1) + "-" + contextVariables.firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well.
|
||||
|
||||
buildCandles();
|
||||
return;
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> err = " + err.stack);
|
||||
if (err.message === "Cannot read property 'file' of undefined") {
|
||||
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Check the bot configuration to see if all of its statusDependencies declarations are correct. ");
|
||||
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys));
|
||||
}
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
function buildCandles() {
|
||||
|
||||
try {
|
||||
let fromDate = new Date(contextVariables.lastCandleFile.valueOf())
|
||||
let lastDate = new Date()
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> Entering function."); }
|
||||
|
||||
let outputCandles;
|
||||
let outputVolumes;
|
||||
|
||||
advanceTime();
|
||||
|
||||
function advanceTime() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> advanceTime -> Entering function."); }
|
||||
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() + ONE_DAY_IN_MILISECONDS);
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> advanceTime -> New processing time @ " + contextVariables.lastCandleFile.getUTCFullYear() + "/" + (contextVariables.lastCandleFile.getUTCMonth() + 1) + "/" + contextVariables.lastCandleFile.getUTCDate() + "."); }
|
||||
|
||||
/* Validation that we are not going past the head of the market. */
|
||||
|
||||
if (contextVariables.lastCandleFile.valueOf() > contextVariables.maxCandleFile.valueOf()) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> advanceTime -> Head of the market found @ " + contextVariables.lastCandleFile.getUTCFullYear() + "/" + (contextVariables.lastCandleFile.getUTCMonth() + 1) + "/" + contextVariables.lastCandleFile.getUTCDate() + "."); }
|
||||
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE); // Here is where we finish processing and wait for the platform to run this module again.
|
||||
return;
|
||||
}
|
||||
|
||||
/* Telling the world we are alive and doing well */
|
||||
let currentDateString = contextVariables.lastCandleFile.getUTCFullYear() + '-' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '-' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
|
||||
let currentDate = new Date(contextVariables.lastCandleFile)
|
||||
let percentage = global.getPercentage(fromDate, currentDate, lastDate)
|
||||
bot.processHeartBeat(currentDateString, percentage)
|
||||
|
||||
if (global.areEqualDates(currentDate, new Date()) === false) {
|
||||
logger.newInternalLoop(bot.codeName, bot.process, currentDate, percentage);
|
||||
}
|
||||
/*
|
||||
|
||||
We prepere the arrays that will accumulate all the information for each output file.
|
||||
|
||||
*/
|
||||
|
||||
outputCandles = [];
|
||||
outputVolumes = [];
|
||||
|
||||
for (let n = 0; n < global.dailyFilePeriods.length; n++) {
|
||||
|
||||
const emptyArray1 = [];
|
||||
const emptyArray2 = [];
|
||||
|
||||
outputCandles.push(emptyArray1);
|
||||
outputVolumes.push(emptyArray2);
|
||||
|
||||
}
|
||||
|
||||
periodsLoop();
|
||||
|
||||
}
|
||||
|
||||
function periodsLoop() {
|
||||
|
||||
/*
|
||||
|
||||
We will iterate through all posible periods.
|
||||
|
||||
*/
|
||||
|
||||
let n = 0 // loop Variable representing each possible period as defined at the periods array.
|
||||
|
||||
loopBody();
|
||||
|
||||
function loopBody() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> Entering function."); }
|
||||
|
||||
const outputPeriod = global.dailyFilePeriods[n][0];
|
||||
const timeFrame = global.dailyFilePeriods[n][1];
|
||||
|
||||
nextCandleFile();
|
||||
|
||||
function nextCandleFile() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> Entering function."); }
|
||||
|
||||
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
|
||||
let fileName = "Data.json"
|
||||
|
||||
let filePathRoot = 'Project/' + bot.project + "/" + bot.mineType + "/" + bot.dataMine + "/" + "Exchange-Raw-Data" + '/' + bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset
|
||||
let filePath = filePathRoot + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
|
||||
filePath += '/' + fileName
|
||||
|
||||
fileStorage.getTextFile(filePath, onFileReceived);
|
||||
|
||||
logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> getting file at dateForPath = " + dateForPath);
|
||||
|
||||
function onFileReceived(err, text) {
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Entering function."); }
|
||||
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> text = " + text); }
|
||||
|
||||
let candlesFile;
|
||||
|
||||
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
|
||||
try {
|
||||
candlesFile = JSON.parse(text);
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Error Parsing JSON -> err = " + err.stack);
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
|
||||
if (err.message === 'File does not exist.' || err.code === 'The specified key does not exist.') {
|
||||
|
||||
logger.write(MODULE_NAME, "[WARN] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Dependency Not Ready -> err = " + err.stack);
|
||||
logger.write(MODULE_NAME, "[WARN] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
|
||||
} else {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Error Received -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const inputCandlesPerdiod = 60 * 1000; // 1 min
|
||||
const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs
|
||||
|
||||
let totalOutputCandles = inputFilePeriod / outputPeriod;
|
||||
let beginingOutputTime = contextVariables.lastCandleFile.valueOf();
|
||||
|
||||
for (let i = 0; i < totalOutputCandles; i++) {
|
||||
|
||||
let outputCandle = {
|
||||
open: 0,
|
||||
close: 0,
|
||||
min: 0,
|
||||
max: 0,
|
||||
begin: 0,
|
||||
end: 0
|
||||
};
|
||||
|
||||
let saveCandle = false;
|
||||
|
||||
outputCandle.begin = beginingOutputTime + i * outputPeriod;
|
||||
outputCandle.end = beginingOutputTime + (i + 1) * outputPeriod - 1;
|
||||
|
||||
for (let j = 0; j < candlesFile.length; j++) {
|
||||
|
||||
let candle = {
|
||||
open: candlesFile[j][2],
|
||||
close: candlesFile[j][3],
|
||||
min: candlesFile[j][0],
|
||||
max: candlesFile[j][1],
|
||||
begin: candlesFile[j][4],
|
||||
end: candlesFile[j][5]
|
||||
};
|
||||
|
||||
/* Here we discard all the candles out of range. */
|
||||
|
||||
if (candle.begin >= outputCandle.begin && candle.end <= outputCandle.end) {
|
||||
|
||||
if (saveCandle === false) { // this will set the value only once.
|
||||
|
||||
outputCandle.open = candle.open;
|
||||
outputCandle.min = candle.min;
|
||||
outputCandle.max = candle.max;
|
||||
}
|
||||
|
||||
saveCandle = true;
|
||||
|
||||
outputCandle.close = candle.close; // only the last one will be saved
|
||||
|
||||
if (candle.min < outputCandle.min) {
|
||||
|
||||
outputCandle.min = candle.min;
|
||||
}
|
||||
|
||||
if (candle.max > outputCandle.max) {
|
||||
|
||||
outputCandle.max = candle.max;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (saveCandle === true) { // then we have a valid candle, otherwise it means there were no candles to fill this one in its time range.
|
||||
|
||||
outputCandles[n].push(outputCandle);
|
||||
}
|
||||
}
|
||||
|
||||
nextVolumeFile();
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function nextVolumeFile() {
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> Entering function."); }
|
||||
|
||||
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
|
||||
let fileName = "Data.json"
|
||||
|
||||
let filePathRoot = 'Project/' + bot.project + "/" + bot.mineType + "/" + bot.dataMine + "/" + "Exchange-Raw-Data" + '/' + bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset
|
||||
let filePath = filePathRoot + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath;
|
||||
filePath += '/' + fileName
|
||||
|
||||
fileStorage.getTextFile(filePath, onFileReceived);
|
||||
|
||||
logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> getting file at dateForPath = " + dateForPath);
|
||||
|
||||
function onFileReceived(err, text) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Entering function."); }
|
||||
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> text = " + text); }
|
||||
|
||||
let volumesFile;
|
||||
|
||||
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
|
||||
try {
|
||||
volumesFile = JSON.parse(text);
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Error Parsing JSON -> err = " + err.stack);
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Error Received -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
|
||||
const inputVolumesPerdiod = 60 * 1000; // 1 min
|
||||
const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs
|
||||
|
||||
let totalOutputVolumes = inputFilePeriod / outputPeriod; // this should be 2 in this case.
|
||||
let beginingOutputTime = contextVariables.lastCandleFile.valueOf();
|
||||
|
||||
for (let i = 0; i < totalOutputVolumes; i++) {
|
||||
|
||||
let outputVolume = {
|
||||
buy: 0,
|
||||
sell: 0,
|
||||
begin: 0,
|
||||
end: 0
|
||||
};
|
||||
|
||||
let saveVolume = false;
|
||||
|
||||
outputVolume.begin = beginingOutputTime + i * outputPeriod;
|
||||
outputVolume.end = beginingOutputTime + (i + 1) * outputPeriod - 1;
|
||||
|
||||
for (let j = 0; j < volumesFile.length; j++) {
|
||||
|
||||
let volume = {
|
||||
buy: volumesFile[j][0],
|
||||
sell: volumesFile[j][1],
|
||||
begin: volumesFile[j][2],
|
||||
end: volumesFile[j][3]
|
||||
};
|
||||
|
||||
/* Here we discard all the Volumes out of range. */
|
||||
|
||||
if (volume.begin >= outputVolume.begin && volume.end <= outputVolume.end) {
|
||||
|
||||
saveVolume = true;
|
||||
|
||||
outputVolume.buy = outputVolume.buy + volume.buy;
|
||||
outputVolume.sell = outputVolume.sell + volume.sell;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if (saveVolume === true) {
|
||||
|
||||
outputVolumes[n].push(outputVolume);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
writeFiles(outputCandles[n], outputVolumes[n], timeFrame, controlLoop);
|
||||
|
||||
}
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function controlLoop() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> controlLoop -> Entering function."); }
|
||||
|
||||
n++;
|
||||
|
||||
if (n < global.dailyFilePeriods.length) {
|
||||
|
||||
loopBody();
|
||||
|
||||
} else {
|
||||
|
||||
writeDataRanges(onWritten);
|
||||
|
||||
function onWritten(err) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> controlLoop -> onWritten -> Entering function."); }
|
||||
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] writeDataRanges -> writeDataRanges -> onCandlesDataRangeWritten -> err = " + err.stack);
|
||||
callBack(err);
|
||||
return;
|
||||
}
|
||||
|
||||
writeStatusReport(contextVariables.lastCandleFile, advanceTime);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
function writeFiles(candles, volumes, timeFrame, callBack) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> Entering function."); }
|
||||
|
||||
/*
|
||||
|
||||
Here we will write the contents of the Candles and Volumens files.
|
||||
|
||||
*/
|
||||
|
||||
try {
|
||||
|
||||
writeCandles();
|
||||
|
||||
function writeCandles() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeCandles -> Entering function."); }
|
||||
|
||||
let separator = "";
|
||||
let fileRecordCounter = 0;
|
||||
let fileContent = "";
|
||||
|
||||
for (let i = 0; i < candles.length; i++) {
|
||||
|
||||
let candle = candles[i];
|
||||
|
||||
fileContent = fileContent + separator + '[' + candles[i].min + "," + candles[i].max + "," + candles[i].open + "," + candles[i].close + "," + candles[i].begin + "," + candles[i].end + "]";
|
||||
|
||||
if (separator === "") { separator = ","; }
|
||||
|
||||
fileRecordCounter++;
|
||||
|
||||
}
|
||||
|
||||
fileContent = "[" + fileContent + "]";
|
||||
|
||||
let fileName = 'Data.json';
|
||||
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
|
||||
let filePath = bot.filePathRoot + "/Output/" + CANDLES_FOLDER_NAME + "/" + bot.process + "/" + timeFrame + "/" + dateForPath;
|
||||
filePath += '/' + fileName
|
||||
|
||||
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated);
|
||||
|
||||
logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeCandles -> creating file at filePath = " + filePath);
|
||||
|
||||
function onFileCreated(err) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeCandles -> onFileCreated -> Entering function."); }
|
||||
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeFiles -> writeCandles -> onFileCreated -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
|
||||
if (LOG_FILE_CONTENT === true) {
|
||||
logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeCandles -> onFileCreated -> Content written = " + fileContent);
|
||||
}
|
||||
|
||||
logger.write(MODULE_NAME, "[WARN] start -> writeFiles -> writeCandles -> onFileCreated -> Finished with File @ " + market.baseAsset + "_" + market.quotedAsset + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName);
|
||||
|
||||
writeVolumes();
|
||||
}
|
||||
}
|
||||
|
||||
function writeVolumes() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeVolumes -> Entering function."); }
|
||||
|
||||
let separator = "";
|
||||
let fileRecordCounter = 0;
|
||||
let fileContent = "";
|
||||
|
||||
for (let i = 0; i < volumes.length; i++) {
|
||||
|
||||
let candle = volumes[i];
|
||||
|
||||
fileContent = fileContent + separator + '[' + volumes[i].buy + "," + volumes[i].sell + "," + volumes[i].begin + "," + volumes[i].end + "]";
|
||||
|
||||
if (separator === "") { separator = ","; }
|
||||
|
||||
fileRecordCounter++;
|
||||
|
||||
}
|
||||
|
||||
fileContent = "[" + fileContent + "]";
|
||||
|
||||
let fileName = 'Data.json';
|
||||
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
|
||||
let filePath = bot.filePathRoot + "/Output/" + VOLUMES_FOLDER_NAME + "/" + bot.process + "/" + timeFrame + "/" + dateForPath;
|
||||
filePath += '/' + fileName
|
||||
|
||||
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated);
|
||||
|
||||
logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeVolumes -> creating file at filePath = " + filePath);
|
||||
|
||||
function onFileCreated(err) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeVolumes -> onFileCreated -> Entering function."); }
|
||||
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeFiles -> writeVolumes -> onFileCreated -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
|
||||
if (LOG_FILE_CONTENT === true) {
|
||||
logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeVolumes -> onFileCreated -> Content written = " + fileContent);
|
||||
}
|
||||
|
||||
logger.write(MODULE_NAME, "[WARN] start -> writeFiles -> writeVolumes -> onFileCreated -> Finished with File @ " + market.baseAsset + "_" + market.quotedAsset + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName);
|
||||
|
||||
callBack();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeFiles -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
function writeDataRanges(callBack) {
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRanges -> Entering function."); }
|
||||
|
||||
writeDataRange(contextVariables.firstTradeFile, contextVariables.lastCandleFile, CANDLES_FOLDER_NAME, onCandlesDataRangeWritten);
|
||||
|
||||
function onCandlesDataRangeWritten(err) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRanges -> Entering function."); }
|
||||
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] writeDataRanges -> writeDataRanges -> onCandlesDataRangeWritten -> err = " + err.stack);
|
||||
callBack(err);
|
||||
return;
|
||||
}
|
||||
|
||||
writeDataRange(contextVariables.firstTradeFile, contextVariables.lastCandleFile, VOLUMES_FOLDER_NAME, onVolumeDataRangeWritten);
|
||||
|
||||
function onVolumeDataRangeWritten(err) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] writeDataRanges -> writeDataRanges -> onVolumeDataRangeWritten -> Entering function."); }
|
||||
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] writeDataRanges -> writeDataRanges -> onVolumeDataRangeWritten -> err = " + err.stack);
|
||||
callBack(err);
|
||||
return;
|
||||
}
|
||||
|
||||
callBack(global.DEFAULT_OK_RESPONSE);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeDataRanges -> err = " + err.stack);
|
||||
callBack(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function writeDataRange(pBegin, pEnd, pProductFolder, callBack) {
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> Entering function."); }
|
||||
|
||||
let dataRange = {
|
||||
begin: pBegin.valueOf(),
|
||||
end:pEnd.valueOf() + ONE_DAY_IN_MILISECONDS
|
||||
};
|
||||
|
||||
let fileContent = JSON.stringify(dataRange);
|
||||
|
||||
let fileName = 'Data.Range.json';
|
||||
let filePath = bot.filePathRoot + "/Output/" + pProductFolder + "/" + bot.process;
|
||||
filePath += '/' + fileName
|
||||
|
||||
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated);
|
||||
|
||||
function onFileCreated(err) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> onFileCreated -> Entering function."); }
|
||||
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeDataRange -> onFileCreated -> err = " + err.stack);
|
||||
callBack(err);
|
||||
return;
|
||||
}
|
||||
|
||||
if (LOG_FILE_CONTENT === true) {
|
||||
logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> onFileCreated -> Content written = " + fileContent);
|
||||
}
|
||||
|
||||
let key = bot.dataMine + "-" + bot.codeName + "-" + pProductFolder + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset
|
||||
let event = {
|
||||
dateRange: dataRange
|
||||
}
|
||||
|
||||
global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Data Range Updated', event)
|
||||
|
||||
callBack(global.DEFAULT_OK_RESPONSE);
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeDataRange -> err = " + err.stack);
|
||||
callBack(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
function writeStatusReport(lastFileDate, callBack) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> Entering function."); }
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> lastFileDate = " + lastFileDate); }
|
||||
|
||||
try {
|
||||
|
||||
let reportKey = "Masters" + "-" + "Candles-Volumes" + "-" + "Multi-Period-Daily"
|
||||
let thisReport = statusDependencies.statusReports.get(reportKey);
|
||||
|
||||
thisReport.file.lastExecution = bot.processDatetime;
|
||||
thisReport.file.lastFile = lastFileDate;
|
||||
thisReport.file.beginingOfMarket = beginingOfMarket.toUTCString()
|
||||
thisReport.save(callBack);
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] 'Start' - ERROR : " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,929 @@
|
||||
exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, FILE_STORAGE) {
|
||||
|
||||
const FULL_LOG = true;
|
||||
const LOG_FILE_CONTENT = false;
|
||||
|
||||
const GMT_SECONDS = ':00.000 GMT+0000';
|
||||
const GMT_MILI_SECONDS = '.000 GMT+0000';
|
||||
const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000;
|
||||
|
||||
const MODULE_NAME = "User Bot";
|
||||
|
||||
const TRADES_FOLDER_NAME = "Trades";
|
||||
|
||||
const CANDLES_FOLDER_NAME = "Candles";
|
||||
const CANDLES_ONE_MIN = "One-Min";
|
||||
|
||||
const VOLUMES_FOLDER_NAME = "Volumes";
|
||||
const VOLUMES_ONE_MIN = "One-Min";
|
||||
|
||||
thisObject = {
|
||||
initialize: initialize,
|
||||
start: start
|
||||
};
|
||||
|
||||
let utilities = UTILITIES.newCloudUtilities(bot, logger);
|
||||
let fileStorage = FILE_STORAGE.newFileStorage(logger);
|
||||
|
||||
let statusDependencies;
|
||||
let beginingOfMarket
|
||||
|
||||
return thisObject;
|
||||
|
||||
function initialize(pStatusDependencies, callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.initialize();
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
|
||||
|
||||
statusDependencies = pStatusDependencies;
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE);
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
This process is going to do the following:
|
||||
|
||||
Read the candles and volumes from Exchange Raw Data and produce a single Index File for Market Period. But this is the situation:
|
||||
|
||||
Exchange Raw Data has a dataset organized with daily files with candles of 1 min. Candles Volumes is writting in this process a single file for each timeFrame for the whole market.
|
||||
Everytime this process run, must be able to resume its job and process everything pending until reaching the head of the market. So the tactic to do this is the
|
||||
following:
|
||||
|
||||
1. First we need to read the last file written by this process, and load all the information into in-memory arrays. We will then append to this arrays the new
|
||||
information we will get from Exchange Raw Data.
|
||||
|
||||
2. We know from out status report which was the last DAY we processed from Exchange Raw Data, but we must be carefull, because that day mightn not have been complete, if the
|
||||
last run found the head of the market. That means that we have to be carefull not to append candles that are already there. To simplify what we do is to discard
|
||||
all candles of the last processed day, and then we can process that full day again adding all the candles.
|
||||
|
||||
*/
|
||||
|
||||
function start(callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); }
|
||||
|
||||
let market = bot.market;
|
||||
|
||||
/* Context Variables */
|
||||
|
||||
let contextVariables = {
|
||||
lastCandleFile: undefined, // Datetime of the last file files sucessfully produced by this process.
|
||||
firstTradeFile: undefined, // Datetime of the first trade file in the whole market history.
|
||||
maxCandleFile: undefined // Datetime of the last file available to be used as an input of this process.
|
||||
};
|
||||
|
||||
getContextVariables();
|
||||
|
||||
function getContextVariables() {
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Entering function."); }
|
||||
|
||||
let thisReport;
|
||||
let reportKey;
|
||||
let statusReport;
|
||||
|
||||
/* We look first for Exchange Raw Data in order to get when the market starts. */
|
||||
|
||||
reportKey = "Masters" + "-" + "Exchange-Raw-Data" + "-" + "Historic-OHLCVs"
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
|
||||
|
||||
statusReport = statusDependencies.statusReports.get(reportKey);
|
||||
|
||||
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
if (statusReport.status === "Status Report is corrupt.") {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
thisReport = statusDependencies.statusReports.get(reportKey).file;
|
||||
|
||||
if (thisReport.beginingOfMarket === undefined) {
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
|
||||
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet.");
|
||||
|
||||
let customOK = {
|
||||
result: global.CUSTOM_OK_RESPONSE.result,
|
||||
message: "Dependency does not exist."
|
||||
}
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
|
||||
callBackFunction(customOK);
|
||||
return;
|
||||
}
|
||||
|
||||
contextVariables.firstTradeFile = new Date(thisReport.beginingOfMarket.year + "-" + thisReport.beginingOfMarket.month + "-" + thisReport.beginingOfMarket.days + " " + thisReport.beginingOfMarket.hours + ":" + thisReport.beginingOfMarket.minutes + GMT_SECONDS);
|
||||
|
||||
/* Second, we get the report from Exchange Raw Data, to know when the marted ends. */
|
||||
|
||||
reportKey = "Masters" + "-" + "Exchange-Raw-Data" + "-" + "Historic-OHLCVs"
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
|
||||
|
||||
statusReport = statusDependencies.statusReports.get(reportKey);
|
||||
|
||||
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
if (statusReport.status === "Status Report is corrupt.") {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
thisReport = statusDependencies.statusReports.get(reportKey).file;
|
||||
|
||||
if (thisReport.lastFile === undefined) {
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
|
||||
|
||||
let customOK = {
|
||||
result: global.CUSTOM_OK_RESPONSE.result,
|
||||
message: "Dependency not ready."
|
||||
}
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message);
|
||||
callBackFunction(customOK);
|
||||
return;
|
||||
}
|
||||
|
||||
contextVariables.maxCandleFile = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + "00:00" + GMT_SECONDS);
|
||||
|
||||
/* Finally we get our own Status Report. */
|
||||
|
||||
reportKey = "Masters" + "-" + "Candles-Volumes" + "-" + "Multi-Period-Market"
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); }
|
||||
|
||||
statusReport = statusDependencies.statusReports.get(reportKey);
|
||||
|
||||
if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month.
|
||||
logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. ");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
if (statusReport.status === "Status Report is corrupt.") {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process.");
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
return;
|
||||
}
|
||||
|
||||
thisReport = statusDependencies.statusReports.get(reportKey).file;
|
||||
|
||||
if (thisReport.lastFile !== undefined) {
|
||||
logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Process Running not for the very first time. -> reportKey = " + reportKey);
|
||||
|
||||
beginingOfMarket = new Date(thisReport.beginingOfMarket);
|
||||
|
||||
if (beginingOfMarket.valueOf() !== contextVariables.firstTradeFile.valueOf()) { // Reset Mechanism for Begining of the Market
|
||||
logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Reset Mechanism for Begining of the Market Activated. -> reportKey = " + reportKey);
|
||||
|
||||
beginingOfMarket = new Date(contextVariables.firstTradeFile.getUTCFullYear() + "-" + (contextVariables.firstTradeFile.getUTCMonth() + 1) + "-" + contextVariables.firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.firstTradeFile.getUTCFullYear() + "-" + (contextVariables.firstTradeFile.getUTCMonth() + 1) + "-" + contextVariables.firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well.
|
||||
|
||||
buildCandles();
|
||||
return;
|
||||
}
|
||||
|
||||
contextVariables.lastCandleFile = new Date(thisReport.lastFile);
|
||||
|
||||
/*
|
||||
Here we assume that the last day written might contain incomplete information. This actually happens every time the head of the market is reached.
|
||||
For that reason we go back one day, the partial information is discarded and added again with whatever new info is available.
|
||||
*/
|
||||
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS);
|
||||
|
||||
findPreviousContent();
|
||||
return;
|
||||
|
||||
} else {
|
||||
logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Process Running for the very first time. -> reportKey = " + reportKey);
|
||||
|
||||
beginingOfMarket = new Date(contextVariables.firstTradeFile.getUTCFullYear() + "-" + (contextVariables.firstTradeFile.getUTCMonth() + 1) + "-" + contextVariables.firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.firstTradeFile.getUTCFullYear() + "-" + (contextVariables.firstTradeFile.getUTCMonth() + 1) + "-" + contextVariables.firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well.
|
||||
|
||||
buildCandles();
|
||||
return;
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> err = " + err.stack);
|
||||
if (err.message === "Cannot read property 'file' of undefined") {
|
||||
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Check the bot configuration to see if all of its statusDependencies declarations are correct. ");
|
||||
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys));
|
||||
logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Dependencies loaded -> Double check that you are not running a process that only can be run at noTime mode at a certain month when it is not prepared to do so.");
|
||||
}
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
function findPreviousContent() {
|
||||
|
||||
/*
|
||||
This is where we read the current files we have produced at previous runs of this same process. We just read all the content and organize it
|
||||
in arrays and keep them in memory.
|
||||
*/
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> Entering function."); }
|
||||
|
||||
let n = 0 // loop Variable representing each possible period as defined at the periods array.
|
||||
|
||||
let allPreviousCandles = [];
|
||||
let allPreviousVolumes = [];
|
||||
|
||||
loopBody();
|
||||
|
||||
function loopBody() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> loopBody -> Entering function."); }
|
||||
|
||||
let timeFrame = global.marketFilesPeriods[n][1];
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> loopBody -> timeFrame = " + timeFrame); }
|
||||
|
||||
let previousCandles;
|
||||
let previousVolumes;
|
||||
|
||||
getCandles();
|
||||
|
||||
function getCandles() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> loopBody -> getCandles -> Entering function."); }
|
||||
|
||||
let fileName = 'Data.json';
|
||||
let filePath = bot.filePathRoot + "/Output/" + CANDLES_FOLDER_NAME + "/" + bot.process + "/" + timeFrame;
|
||||
filePath += '/' + fileName
|
||||
|
||||
fileStorage.getTextFile(filePath, onFileReceived);
|
||||
|
||||
logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> loopBody -> getCandles -> getting file.");
|
||||
|
||||
function onFileReceived(err, text) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> loopBody -> getCandles -> onFileReceived -> Entering function."); }
|
||||
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> loopBody -> getCandles -> onFileReceived -> text = " + text); }
|
||||
|
||||
let candlesFile;
|
||||
|
||||
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
|
||||
try {
|
||||
candlesFile = JSON.parse(text);
|
||||
|
||||
previousCandles = candlesFile;
|
||||
|
||||
getVolumes();
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> findPreviousContent -> loopBody -> getCandles -> onFileReceived -> fileName = " + fileName);
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> findPreviousContent -> loopBody -> getCandles -> onFileReceived -> filePath = " + filePath);
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> findPreviousContent -> loopBody -> getCandles -> onFileReceived -> err = " + err.stack);
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> findPreviousContent -> loopBody -> getCandles -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
}
|
||||
} else {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> findPreviousContent -> loopBody -> getCandles -> onFileReceived -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getVolumes() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> loopBody -> getVolumes -> Entering function."); }
|
||||
|
||||
let fileName = 'Data.json';
|
||||
let filePath = bot.filePathRoot + "/Output/" + VOLUMES_FOLDER_NAME + "/" + bot.process + "/" + timeFrame;
|
||||
filePath += '/' + fileName
|
||||
|
||||
fileStorage.getTextFile(filePath, onFileReceived);
|
||||
|
||||
logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> loopBody -> getVolumes -> getting file.");
|
||||
|
||||
function onFileReceived(err, text) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> loopBody -> getVolumes -> onFileReceived -> Entering function."); }
|
||||
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> loopBody -> getVolumes -> onFileReceived -> text = " + text); }
|
||||
|
||||
let volumesFile;
|
||||
|
||||
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
|
||||
try {
|
||||
volumesFile = JSON.parse(text);
|
||||
|
||||
previousVolumes = volumesFile;
|
||||
|
||||
allPreviousCandles.push(previousCandles);
|
||||
allPreviousVolumes.push(previousVolumes);
|
||||
|
||||
controlLoop();
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> findPreviousContent -> loopBody -> getVolumes -> onFileReceived -> fileName = " + fileName);
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> findPreviousContent -> loopBody -> getVolumes -> onFileReceived -> filePath = " + filePath);
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> findPreviousContent -> loopBody -> getVolumes -> onFileReceived -> err = " + err.stack);
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> findPreviousContent -> loopBody -> getVolumes -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
}
|
||||
} else {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> findPreviousContent -> loopBody -> getVolumes -> onFileReceived -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function controlLoop() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findPreviousContent -> controlLoop -> Entering function."); }
|
||||
|
||||
n++;
|
||||
|
||||
if (n < global.marketFilesPeriods.length) {
|
||||
|
||||
loopBody();
|
||||
|
||||
} else {
|
||||
|
||||
buildCandles(allPreviousCandles, allPreviousVolumes);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> findPreviousContent -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
function buildCandles(allPreviousCandles, allPreviousVolumes) {
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> Entering function."); }
|
||||
|
||||
let fromDate = new Date(contextVariables.lastCandleFile.valueOf())
|
||||
let lastDate = new Date()
|
||||
|
||||
/*
|
||||
Firstly we prepere the arrays that will accumulate all the information for each output file.
|
||||
*/
|
||||
|
||||
let outputCandles = [];
|
||||
let outputVolumes = [];
|
||||
|
||||
for (let n = 0; n < global.marketFilesPeriods.length; n++) {
|
||||
|
||||
const emptyArray1 = [];
|
||||
const emptyArray2 = [];
|
||||
|
||||
outputCandles.push(emptyArray1);
|
||||
outputVolumes.push(emptyArray2);
|
||||
|
||||
}
|
||||
|
||||
advanceTime();
|
||||
|
||||
function advanceTime() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> advanceTime -> Entering function."); }
|
||||
|
||||
contextVariables.lastCandleFile = new Date(contextVariables.lastCandleFile.valueOf() + ONE_DAY_IN_MILISECONDS);
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> advanceTime -> New processing time @ " + contextVariables.lastCandleFile.getUTCFullYear() + "/" + (contextVariables.lastCandleFile.getUTCMonth() + 1) + "/" + contextVariables.lastCandleFile.getUTCDate() + "."); }
|
||||
|
||||
/* Validation that we are not going past the head of the market. */
|
||||
|
||||
if (contextVariables.lastCandleFile.valueOf() > contextVariables.maxCandleFile.valueOf()) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> advanceTime -> Head of the market found @ " + contextVariables.lastCandleFile.getUTCFullYear() + "/" + (contextVariables.lastCandleFile.getUTCMonth() + 1) + "/" + contextVariables.lastCandleFile.getUTCDate() + "."); }
|
||||
|
||||
callBackFunction(global.DEFAULT_OK_RESPONSE); // Here is where we finish processing and wait for the platform to run this module again.
|
||||
return;
|
||||
}
|
||||
|
||||
/* Telling the world we are alive and doing well */
|
||||
let currentDateString = contextVariables.lastCandleFile.getUTCFullYear() + '-' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '-' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
|
||||
let currentDate = new Date(contextVariables.lastCandleFile)
|
||||
let percentage = global.getPercentage(fromDate, currentDate, lastDate)
|
||||
bot.processHeartBeat(currentDateString, percentage)
|
||||
|
||||
if (global.areEqualDates(currentDate, new Date()) === false) {
|
||||
logger.newInternalLoop(bot.codeName, bot.process, currentDate, percentage);
|
||||
}
|
||||
|
||||
periodsLoop();
|
||||
|
||||
}
|
||||
|
||||
function periodsLoop() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> Entering function."); }
|
||||
|
||||
/*
|
||||
|
||||
We will iterate through all posible periods.
|
||||
|
||||
*/
|
||||
|
||||
let n = 0 // loop Variable representing each possible period as defined at the periods array.
|
||||
|
||||
loopBody();
|
||||
|
||||
function loopBody() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> Entering function."); }
|
||||
|
||||
let previousCandles;
|
||||
let previousVolumes;
|
||||
|
||||
if (allPreviousCandles !== undefined) {
|
||||
|
||||
previousCandles = allPreviousCandles[n];
|
||||
previousVolumes = allPreviousVolumes[n];
|
||||
|
||||
}
|
||||
|
||||
const outputPeriod = global.marketFilesPeriods[n][0];
|
||||
const timeFrame = global.marketFilesPeriods[n][1];
|
||||
|
||||
/*
|
||||
Here we are inside a Loop that is going to advance 1 day at the time, at each pass, will ready one of Exchange Raw Data's daily files and
|
||||
add all its candles to our in memory arrays. At the first iteration of this loop, we will add the candles that we are carrying
|
||||
from our previous run, the ones we already have in-memory. You can see below how we discard from those candles the ones that
|
||||
are belonging to the first day we are processing at this run, that it is exactly the same as the last day processed the privious
|
||||
run. By discarding these candles, we are ready to run after that standard function that will just add ALL the candles found each
|
||||
day at Exchange Raw Data.
|
||||
*/
|
||||
|
||||
if (previousCandles !== undefined) {
|
||||
|
||||
for (let i = 0; i < previousCandles.length; i++) {
|
||||
|
||||
let candle = {
|
||||
open: previousCandles[i][2],
|
||||
close: previousCandles[i][3],
|
||||
min: previousCandles[i][0],
|
||||
max: previousCandles[i][1],
|
||||
begin: previousCandles[i][4],
|
||||
end: previousCandles[i][5]
|
||||
};
|
||||
|
||||
if (candle.end < contextVariables.lastCandleFile.valueOf()) {
|
||||
|
||||
outputCandles[n].push(candle);
|
||||
|
||||
} else {
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> Candle # " + i + " @ " + timeFrame + " discarded for closing past the current process time."); }
|
||||
}
|
||||
}
|
||||
allPreviousCandles[n] = []; // erasing these so as not to duplicate them.
|
||||
}
|
||||
|
||||
if (previousVolumes !== undefined) {
|
||||
|
||||
for (let i = 0; i < previousVolumes.length; i++) {
|
||||
|
||||
let volume = {
|
||||
begin: previousVolumes[i][2],
|
||||
end: previousVolumes[i][3],
|
||||
buy: previousVolumes[i][0],
|
||||
sell: previousVolumes[i][1]
|
||||
};
|
||||
|
||||
if (volume.end < contextVariables.lastCandleFile.valueOf()) {
|
||||
|
||||
outputVolumes[n].push(volume);
|
||||
|
||||
} else {
|
||||
if (FULL_LOG === true) {logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> Volume # " + i + " @ " + timeFrame + " discarded for closing past the current process time."); }
|
||||
}
|
||||
}
|
||||
allPreviousVolumes[n] = []; // erasing these so as not to duplicate them.
|
||||
}
|
||||
|
||||
/*
|
||||
From here on is where every iteration of the loop fully runs. Here is where we read Exchange Raw Data's files and add their content to whatever
|
||||
we already have in our arrays in-memory. In this way the process will run as many days needed and it should only stop when it reaches
|
||||
the head of the market.
|
||||
*/
|
||||
|
||||
nextCandleFile();
|
||||
|
||||
function nextCandleFile() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> Entering function."); }
|
||||
|
||||
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
|
||||
let fileName = "Data.json"
|
||||
|
||||
let filePathRoot = 'Project/' + bot.project + "/" + bot.mineType + "/" + bot.dataMine + "/" + "Exchange-Raw-Data" + '/' + bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset
|
||||
let filePath = filePathRoot + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
|
||||
filePath += '/' + fileName
|
||||
|
||||
fileStorage.getTextFile(filePath, onFileReceived);
|
||||
|
||||
logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> getting file at dateForPath = " + dateForPath);
|
||||
|
||||
function onFileReceived(err, text) {
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Entering function."); }
|
||||
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> text = " + text); }
|
||||
|
||||
let candlesFile;
|
||||
|
||||
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
|
||||
try {
|
||||
candlesFile = JSON.parse(text);
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Error Parsing JSON -> err = " + err.stack);
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
|
||||
if (err.message === 'File does not exist.' || err.code === 'The specified key does not exist.') {
|
||||
|
||||
logger.write(MODULE_NAME, "[WARN] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Dependency Not Ready -> err = " + JSON.stringify(err));
|
||||
logger.write(MODULE_NAME, "[WARN] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
|
||||
} else {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> Error Received -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const inputCandlesPerdiod = 60 * 1000; // 1 min
|
||||
const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs
|
||||
|
||||
let totalOutputCandles = inputFilePeriod / outputPeriod; // this should be 2 in this case.
|
||||
let beginingOutputTime = contextVariables.lastCandleFile.valueOf();
|
||||
|
||||
/*
|
||||
The algorithm that follows is going to agregate candles of 1 min timeFrame read from Exchange Raw Data, into candles of each timeFrame
|
||||
that Candles Volumes generates. For market files those timePediods goes from 1h to 24hs.
|
||||
*/
|
||||
|
||||
for (let i = 0; i < totalOutputCandles; i++) {
|
||||
|
||||
let outputCandle = {
|
||||
open: 0,
|
||||
close: 0,
|
||||
min: 0,
|
||||
max: 0,
|
||||
begin: 0,
|
||||
end: 0
|
||||
};
|
||||
|
||||
let saveCandle = false;
|
||||
|
||||
outputCandle.begin = beginingOutputTime + i * outputPeriod;
|
||||
outputCandle.end = beginingOutputTime + (i + 1) * outputPeriod - 1;
|
||||
|
||||
for (let j = 0; j < candlesFile.length; j++) {
|
||||
|
||||
let candle = {
|
||||
open: candlesFile[j][2],
|
||||
close: candlesFile[j][3],
|
||||
min: candlesFile[j][0],
|
||||
max: candlesFile[j][1],
|
||||
begin: candlesFile[j][4],
|
||||
end: candlesFile[j][5]
|
||||
};
|
||||
|
||||
/* Here we discard all the candles out of range. */
|
||||
|
||||
if (candle.begin >= outputCandle.begin && candle.end <= outputCandle.end) {
|
||||
|
||||
if (saveCandle === false) { // this will set the value only once.
|
||||
|
||||
outputCandle.open = candle.open;
|
||||
outputCandle.min = candle.min;
|
||||
outputCandle.max = candle.max;
|
||||
}
|
||||
|
||||
saveCandle = true;
|
||||
|
||||
outputCandle.close = candle.close; // only the last one will be saved
|
||||
|
||||
if (candle.min < outputCandle.min) {
|
||||
|
||||
outputCandle.min = candle.min;
|
||||
}
|
||||
|
||||
if (candle.max > outputCandle.max) {
|
||||
|
||||
outputCandle.max = candle.max;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (saveCandle === true) { // then we have a valid candle, otherwise it means there were no candles to fill this one in its time range.
|
||||
|
||||
outputCandles[n].push(outputCandle);
|
||||
}
|
||||
}
|
||||
|
||||
nextVolumeFile();
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextCandleFile -> onFileReceived -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function nextVolumeFile() {
|
||||
|
||||
try {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> Entering function."); }
|
||||
|
||||
let dateForPath = contextVariables.lastCandleFile.getUTCFullYear() + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(contextVariables.lastCandleFile.getUTCDate(), 2);
|
||||
let fileName = "Data.json"
|
||||
|
||||
let filePathRoot = 'Project/' + bot.project + "/" + bot.mineType + "/" + bot.dataMine + "/" + "Exchange-Raw-Data" + '/' + bot.exchange + "/" + bot.market.baseAsset + "-" + bot.market.quotedAsset
|
||||
let filePath = filePathRoot + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath;
|
||||
filePath += '/' + fileName
|
||||
|
||||
fileStorage.getTextFile(filePath, onFileReceived);
|
||||
|
||||
logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> getting file at dateForPath = " + dateForPath);
|
||||
|
||||
function onFileReceived(err, text) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Entering function."); }
|
||||
if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> text = " + text); }
|
||||
|
||||
let volumesFile;
|
||||
|
||||
if (err.result === global.DEFAULT_OK_RESPONSE.result) {
|
||||
try {
|
||||
volumesFile = JSON.parse(text);
|
||||
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Error Parsing JSON -> err = " + err.stack);
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.");
|
||||
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> Error Received -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
|
||||
const inputVolumesPerdiod = 60 * 1000; // 1 min
|
||||
const inputFilePeriod = 24 * 60 * 60 * 1000; // 24 hs
|
||||
|
||||
let totalOutputVolumes = inputFilePeriod / outputPeriod; // this should be 2 in this case.
|
||||
let beginingOutputTime = contextVariables.lastCandleFile.valueOf();
|
||||
|
||||
for (let i = 0; i < totalOutputVolumes; i++) {
|
||||
|
||||
let outputVolume = {
|
||||
buy: 0,
|
||||
sell: 0,
|
||||
begin: 0,
|
||||
end: 0
|
||||
};
|
||||
|
||||
let saveVolume = false;
|
||||
|
||||
outputVolume.begin = beginingOutputTime + i * outputPeriod;
|
||||
outputVolume.end = beginingOutputTime + (i + 1) * outputPeriod - 1;
|
||||
|
||||
for (let j = 0; j < volumesFile.length; j++) {
|
||||
|
||||
let volume = {
|
||||
buy: volumesFile[j][0],
|
||||
sell: volumesFile[j][1],
|
||||
begin: volumesFile[j][2],
|
||||
end: volumesFile[j][3]
|
||||
};
|
||||
|
||||
/* Here we discard all the Volumes out of range. */
|
||||
|
||||
if (volume.begin >= outputVolume.begin && volume.end <= outputVolume.end) {
|
||||
|
||||
saveVolume = true;
|
||||
|
||||
outputVolume.buy = outputVolume.buy + volume.buy;
|
||||
outputVolume.sell = outputVolume.sell + volume.sell;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if (saveVolume === true) {
|
||||
|
||||
outputVolumes[n].push(outputVolume);
|
||||
}
|
||||
}
|
||||
|
||||
writeFiles(outputCandles[n], outputVolumes[n], timeFrame, controlLoop);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> periodsLoop -> loopBody -> nextVolumeFile -> onFileReceived -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function controlLoop() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> buildCandles -> periodsLoop -> controlLoop -> Entering function."); }
|
||||
|
||||
n++;
|
||||
|
||||
if (n < global.marketFilesPeriods.length) {
|
||||
|
||||
loopBody();
|
||||
|
||||
} else {
|
||||
|
||||
writeStatusReport(contextVariables.lastCandleFile, advanceTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> buildCandles -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
function writeFiles(candles, volumes, timeFrame, callBack) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> Entering function."); }
|
||||
|
||||
/*
|
||||
|
||||
Here we will write the contents of the Candles and Volumens files.
|
||||
|
||||
*/
|
||||
|
||||
try {
|
||||
|
||||
writeCandles();
|
||||
|
||||
function writeCandles() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeCandles -> Entering function."); }
|
||||
|
||||
let separator = "";
|
||||
let fileRecordCounter = 0;
|
||||
let fileContent = "";
|
||||
|
||||
for (let i = 0; i < candles.length; i++) {
|
||||
|
||||
let candle = candles[i];
|
||||
|
||||
fileContent = fileContent + separator + '[' + candles[i].min + "," + candles[i].max + "," + candles[i].open + "," + candles[i].close + "," + candles[i].begin + "," + candles[i].end + "]";
|
||||
|
||||
if (separator === "") { separator = ","; }
|
||||
|
||||
fileRecordCounter++;
|
||||
|
||||
}
|
||||
|
||||
fileContent = "[" + fileContent + "]";
|
||||
|
||||
let fileName = 'Data.json';
|
||||
let filePath = bot.filePathRoot + "/Output/" + CANDLES_FOLDER_NAME + "/" + bot.process + "/" + timeFrame;
|
||||
filePath += '/' + fileName
|
||||
|
||||
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated);
|
||||
|
||||
logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeCandles -> creating file at filePath = " + filePath);
|
||||
|
||||
function onFileCreated(err) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeCandles -> onFileCreated -> Entering function."); }
|
||||
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeFiles -> writeCandles -> onFileCreated -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
|
||||
if (LOG_FILE_CONTENT === true) {
|
||||
logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeCandles -> onFileCreated -> Content written = " + fileContent);
|
||||
}
|
||||
|
||||
logger.write(MODULE_NAME, "[WARN] start -> writeFiles -> writeCandles -> onFileCreated -> Finished with File @ " + market.baseAsset + "_" + market.quotedAsset + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName );
|
||||
|
||||
writeVolumes();
|
||||
}
|
||||
}
|
||||
|
||||
function writeVolumes() {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeVolumes -> Entering function."); }
|
||||
|
||||
let separator = "";
|
||||
let fileRecordCounter = 0;
|
||||
let fileContent = "";
|
||||
|
||||
for (let i = 0; i < volumes.length; i++) {
|
||||
|
||||
let candle = volumes[i];
|
||||
|
||||
fileContent = fileContent + separator + '[' + volumes[i].buy + "," + volumes[i].sell + "," + volumes[i].begin + "," + volumes[i].end + "]";
|
||||
|
||||
if (separator === "") { separator = ","; }
|
||||
|
||||
fileRecordCounter++;
|
||||
|
||||
}
|
||||
|
||||
fileContent = "[" + fileContent + "]";
|
||||
|
||||
let fileName = 'Data.json';
|
||||
let filePath = bot.filePathRoot + "/Output/" + VOLUMES_FOLDER_NAME + "/" + bot.process + "/" + timeFrame;
|
||||
filePath += '/' + fileName
|
||||
|
||||
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated);
|
||||
|
||||
logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeVolumes -> creating file at filePath = " + filePath);
|
||||
|
||||
function onFileCreated(err) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeVolumes -> onFileCreated -> Entering function."); }
|
||||
|
||||
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeFiles -> writeVolumes -> onFileCreated -> err = " + err.stack);
|
||||
callBackFunction(err);
|
||||
return;
|
||||
}
|
||||
|
||||
if (LOG_FILE_CONTENT === true) {
|
||||
logger.write(MODULE_NAME, "[INFO] start -> writeFiles -> writeVolumes -> onFileCreated -> Content written = " + fileContent);
|
||||
}
|
||||
|
||||
logger.write(MODULE_NAME, "[WARN] start -> writeFiles -> writeVolumes -> onFileCreated -> Finished with File @ " + market.baseAsset + "_" + market.quotedAsset + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName);
|
||||
|
||||
callBack();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeFiles -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
|
||||
function writeStatusReport(lastFileDate, callBack) {
|
||||
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> Entering function."); }
|
||||
if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> lastFileDate = " + lastFileDate); }
|
||||
|
||||
try {
|
||||
|
||||
let reportKey = "Masters" + "-" + "Candles-Volumes" + "-" + "Multi-Period-Market"
|
||||
let thisReport = statusDependencies.statusReports.get(reportKey);
|
||||
|
||||
thisReport.file.lastExecution = bot.processDatetime;
|
||||
thisReport.file.lastFile = lastFileDate;
|
||||
thisReport.file.beginingOfMarket = beginingOfMarket.toUTCString()
|
||||
thisReport.save(callBack);
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.stack);
|
||||
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,80 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<ProjectGuid>{aa2901bc-8f81-40f0-a402-5ca79a3f22ad}</ProjectGuid>
|
||||
<ProjectHome />
|
||||
<ProjectView>ShowAllFiles</ProjectView>
|
||||
<StartupFile>
|
||||
</StartupFile>
|
||||
<WorkingDirectory>c:\users\luis\source\repos\AAOlivia\Multi-Period-Market</WorkingDirectory>
|
||||
<OutputPath>.</OutputPath>
|
||||
<ProjectTypeGuids>{3AF33F2E-1136-4D97-BBB7-1795711AC8B8};{349c5851-65df-11da-9384-00065b846f21};{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}</ProjectTypeGuids>
|
||||
<VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion>
|
||||
<VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
|
||||
<Name>Project</Name>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)' == 'Debug'" />
|
||||
<PropertyGroup Condition="'$(Configuration)' == 'Release'" />
|
||||
<ItemGroup>
|
||||
<Compile Include="Commons.js">
|
||||
<SubType>Code</SubType>
|
||||
</Compile>
|
||||
<Compile Include="Multi-Period-Daily\User.Bot.js" />
|
||||
<Compile Include="Multi-Period-Market\User.Bot.js" />
|
||||
<Content Include="obj\Debug\AABruce.njsproj.FileListAbsolute.txt" />
|
||||
<Content Include="obj\Debug\AACarol.njsproj.FileListAbsolute.txt" />
|
||||
<Content Include="obj\Debug\AACharly.njsproj.FileListAbsolute.txt" />
|
||||
<Content Include="this.bot.config.json" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Folder Include="Multi-Period-Daily\" />
|
||||
<Folder Include="Multi-Period-Market\" />
|
||||
<Folder Include="bin" />
|
||||
<Folder Include="obj" />
|
||||
<Folder Include="obj\Debug" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.Common.targets" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<!--Do not delete the following Import Project. While this appears to do nothing it is a marker for setting TypeScript properties before our import that depends on them.-->
|
||||
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\TypeScript\Microsoft.TypeScript.targets" Condition="False" />
|
||||
<Import Project="$(VSToolsPath)\Node.js Tools\Microsoft.NodejsTools.targets" />
|
||||
<ProjectExtensions>
|
||||
<VisualStudio>
|
||||
<FlavorProperties GUID="{349c5851-65df-11da-9384-00065b846f21}">
|
||||
<WebProjectProperties>
|
||||
<UseIIS>False</UseIIS>
|
||||
<AutoAssignPort>True</AutoAssignPort>
|
||||
<DevelopmentServerPort>0</DevelopmentServerPort>
|
||||
<DevelopmentServerVPath>/</DevelopmentServerVPath>
|
||||
<IISUrl>http://localhost:48022/</IISUrl>
|
||||
<NTLMAuthentication>False</NTLMAuthentication>
|
||||
<UseCustomServer>True</UseCustomServer>
|
||||
<CustomServerUrl>http://localhost:1337</CustomServerUrl>
|
||||
<SaveServerSettingsInUserFile>False</SaveServerSettingsInUserFile>
|
||||
</WebProjectProperties>
|
||||
</FlavorProperties>
|
||||
<FlavorProperties GUID="{349c5851-65df-11da-9384-00065b846f21}" User="">
|
||||
<WebProjectProperties>
|
||||
<StartPageUrl>
|
||||
</StartPageUrl>
|
||||
<StartAction>CurrentPage</StartAction>
|
||||
<AspNetDebugging>True</AspNetDebugging>
|
||||
<SilverlightDebugging>False</SilverlightDebugging>
|
||||
<NativeDebugging>False</NativeDebugging>
|
||||
<SQLDebugging>False</SQLDebugging>
|
||||
<ExternalProgram>
|
||||
</ExternalProgram>
|
||||
<StartExternalURL>
|
||||
</StartExternalURL>
|
||||
<StartCmdLineArguments>
|
||||
</StartCmdLineArguments>
|
||||
<StartWorkingDirectory>
|
||||
</StartWorkingDirectory>
|
||||
<EnableENC>False</EnableENC>
|
||||
<AlwaysStartWebServerOnDebug>False</AlwaysStartWebServerOnDebug>
|
||||
</WebProjectProperties>
|
||||
</FlavorProperties>
|
||||
</VisualStudio>
|
||||
</ProjectExtensions>
|
||||
</Project>
|
||||
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<LastActiveSolutionConfig>Debug|Any CPU</LastActiveSolutionConfig>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,126 @@
|
||||
# Olivia
|
||||
|
||||
Olivia is a middle level Indicator bot that has as an input cnadles and volumes daily files and outputs 2 different datasets at different time-periods also of candles and volumnes.
|
||||
|
||||
## Multi Period Daily
|
||||
|
||||
This process generates files containing 1 day of candles or volumes each.
|
||||
|
||||
### Start Mode
|
||||
|
||||
This process runs every 1 minute under noTime start mode.
|
||||
|
||||
```
|
||||
"startMode": {
|
||||
"noTime": {
|
||||
"run": "true"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Multi Period Market
|
||||
|
||||
This process generates files containing candles or volumes for the whole history of the market.
|
||||
|
||||
### Start Mode
|
||||
|
||||
This process runs every 1 minute under noTime start mode.
|
||||
|
||||
```
|
||||
"startMode": {
|
||||
"noTime": {
|
||||
"run": "true"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Output
|
||||
Several datasets are produced:
|
||||
* One with candles at 1, 2, 3, 4, 5, 10, 15, 20, 30, 45 minutes resolution;
|
||||
* One with candles at 1, 2, 3, 4, 6, 8, 12, 24 hours resolution;
|
||||
* One with volumes at 1, 2, 3, 4, 5, 10, 15, 20, 30, 45 minutes resolution;
|
||||
* One with volumes at 1, 2, 3, 4, 6, 8, 12, 24 hours resolution.
|
||||
|
||||
### Ideal for
|
||||
Output datasets are the base for most common indicators and they become available at multiple resolution levels.
|
||||
|
||||
### Details
|
||||
|
||||
| **Name** | **Type** | **Version** | **Release Date** | **Current dataSet** |
|
||||
|----------|----------|----------|----------|----------|
|
||||
| Olivia | Indicator | 1.0 | 28 Feb 2018 | dataSet.V1 |
|
||||
|
||||
# Products
|
||||
|
||||
## Dataset 1: Daily Candles in Resolutions Below 1 hour
|
||||
|
||||
### Current Dataset
|
||||
|
||||
* **Folder Structure Tree**: Output > Candles > Multi-Period-Daily > (01-min | 02-min | 03-min | 04-min | 05-min | 10-min | 15-min | 20-min | 30-min | 45-min) > Year > Month > Day
|
||||
* **Files Structure**: One .json file stored at the Day level of the Folder Tree Structure
|
||||
* **In-File Record Structure**:
|
||||
* Minimum, decimal;
|
||||
* Maximum, decimal;
|
||||
* Open, decimal;
|
||||
* Close, decimal;
|
||||
* Begin, decimal;
|
||||
* End, decimal;
|
||||
* **In-File Record Example**:
|
||||
* [[10194.64038378,10308.25665115,10261.25665108,10236.90000135,1517443200000,1517443799999],[10236.90000135,10300.680924,10236.90000229,10288.10043292,1517443800000,1517444399999], ... ,[10288.10043292,10367.78412393,10288.10043293,10309.00000025,1517444400000,1517444999999],[10201.97490283,10367.78412393,10309.00000025,10201.97490283,1517445000000,1517445599999]]
|
||||
|
||||
## Dataset 2: All-time, Market-long Candles in Resolutions Between 1 and 24 Hours
|
||||
|
||||
### Current Dataset
|
||||
|
||||
* **Folder Structure Tree**: Output > Candles > Multi-Period-Market > (01-hs| 02-hs | 03-hs | 04-hs | 06-hs | 08-hs | 12-hs | 24-hs)
|
||||
* **Files Structure**: One .json file stored at the deepest level of the Folder Tree Structure
|
||||
* **In-File Record Structure**:
|
||||
* Minimum, decimal;
|
||||
* Maximum, decimal;
|
||||
* Open, decimal;
|
||||
* Close, decimal;
|
||||
* Begin, decimal;
|
||||
* End, decimal;
|
||||
* **In-File Record Example**:
|
||||
* [[10194.64038378,10308.25665115,10261.25665108,10236.90000135,1517443200000,1517443799999],[10236.90000135,10300.680924,10236.90000229,10288.10043292,1517443800000,1517444399999], ... ,[10288.10043292,10367.78412393,10288.10043293,10309.00000025,1517444400000,1517444999999],[10201.97490283,10367.78412393,10309.00000025,10201.97490283,1517445000000,1517445599999]]
|
||||
|
||||
## Dataset 3: Daily Volumes in Resolutions Below 1 hour
|
||||
|
||||
### Current Dataset
|
||||
|
||||
* **Folder Structure Tree**: Output > Volumes > Multi-Period-Daily > (01-min | 02-min | 03-min | 04-min | 05-min | 10-min | 15-min | 20-min | 30-min | 45-min) > Year > Month > Day
|
||||
* **Files Structure**: One .json file stored at the Day level of the Folder Tree Structure
|
||||
* **In-File Record Structure**:
|
||||
* Buy Volume, decimal;
|
||||
* Sell Volume, decimal;
|
||||
* Open Volume, decimal;
|
||||
* Close Volume, decimal
|
||||
* **In-File Record Example**:
|
||||
* [[33271.902671610005,13940.15380426,1514764800000,1514764859999],[15312.391007760001,18734.01756361,1514764860000,1514764919999], ... ,[16261.87486184,2788.53802902,1514764920000,1514764979999],[513.24710259,9896.328325810004,1514764980000,1514765039999]]
|
||||
|
||||
## Dataset 4: All-time, Market-long Volumes in Resolutions Between 1 and 24 Hours
|
||||
|
||||
### Current Dataset
|
||||
|
||||
* **Folder Structure Tree**: Output > Volumes > Multi-Period-Market > (01-hs| 02-hs | 03-hs | 04-hs | 06-hs | 08-hs | 12-hs | 24-hs) >
|
||||
* **Files Structure**: One .json file stored at the deepest level of the Folder Tree Structure
|
||||
* **In-File Record Structure**:
|
||||
* Buy Volume, decimal;
|
||||
* Sell Volume, decimal;
|
||||
* Open Volume, decimal;
|
||||
* Close Volume, decimal
|
||||
* **In-File Record Example**:
|
||||
* [[33271.902671610005,13940.15380426,1514764800000,1514764859999],[15312.391007760001,18734.01756361,1514764860000,1514764919999], ... ,[16261.87486184,2788.53802902,1514764920000,1514764979999],[513.24710259,9896.328325810004,1514764980000,1514765039999]]
|
||||
|
||||
### Compatible Plotters
|
||||
[AAMasters Plotters-Candles-Volumes](https://github.com/AAMasters/Plotters-Candles-Volumes)
|
||||
|
||||
### Dependencies
|
||||
[AAMasters Charly](https://github.com/AAMasters/AACharly-Extraction-Bot)
|
||||
[AAMasters Bruce](https://github.com/AAMasters/AABruce-Indicator-Bot)
|
||||
|
||||
# Disclaimer
|
||||
|
||||
THE AA MASTERS BOTS AND THEIR ASSOCIATED PRODUCTS AND SERVICES ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, SUITABILITY FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT.
|
||||
|
||||
IN NO EVENT WILL ADVANCED ALGOS BE LIABLE TO ANY PARTY FOR ANY DIRECT, INDIRECT, SPECIAL OR OTHER CONSEQUENTIAL DAMAGES FOR ANY USE OF THE AACLOUD, THE AA ARENA COMPETITION, THE AA MASTERS BOTS, OR ANY OTHER ASSOCIATED SERVICE OR WEB SITE, INCLUDING, WITHOUT LIMITATION, ANY LOST PROFITS, BUSINESS INTERRUPTION, LOSS OF FUNDS, PROGRAMS OR OTHER DATA OR OTHERWISE, EVEN IF WE ARE EXPRESSLY ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "AAOlivia",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "Luis-Fernando-Molina",
|
||||
"license": "ISC",
|
||||
"dependencies": {}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
{
|
||||
"displayName": "Candles Volumes",
|
||||
"codeName": "Candles-Volumes",
|
||||
"type": "Indicator Bot",
|
||||
"dataMine": "Masters",
|
||||
"processes": [
|
||||
{
|
||||
"name": "Multi-Period-Market",
|
||||
"description": "Produces different files with multi-period candles and volumes for entire markets.",
|
||||
"startMode": {
|
||||
"allMonths": {
|
||||
"run": "false",
|
||||
"minYear": "",
|
||||
"maxYear": ""
|
||||
},
|
||||
"oneMonth": {
|
||||
"run": "false",
|
||||
"year": "",
|
||||
"month": ""
|
||||
},
|
||||
"noTime": {
|
||||
"run": "true"
|
||||
},
|
||||
"fixedInterval": {
|
||||
"run": "false",
|
||||
"interval": 0
|
||||
}
|
||||
},
|
||||
"deadWaitTime": 0,
|
||||
"normalWaitTime": 0,
|
||||
"retryWaitTime": 10000,
|
||||
"sleepWaitTime": 3600000,
|
||||
"comaWaitTime": 86400000,
|
||||
"updatesDatasets": [
|
||||
{
|
||||
"product": "Candles",
|
||||
"dataSet": "Multi-Period-Market"
|
||||
},
|
||||
{
|
||||
"product": "Volumes",
|
||||
"dataSet": "Multi-Period-Market"
|
||||
}
|
||||
],
|
||||
"statusDependencies": [
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Exchange-Raw-Data",
|
||||
"process": "Historic-OHLCVs"
|
||||
},
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Exchange-Raw-Data",
|
||||
"process": "Historic-OHLCVs",
|
||||
"dataSetSection": "Month",
|
||||
"waitUntilNextUpdate": true
|
||||
},
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Candles-Volumes",
|
||||
"process": "Multi-Period-Market"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Multi-Period-Daily",
|
||||
"description": "Produces different files with multi-period candles and volumes on a daily format.",
|
||||
"startMode": {
|
||||
"allMonths": {
|
||||
"run": "false",
|
||||
"minYear": "",
|
||||
"maxYear": ""
|
||||
},
|
||||
"oneMonth": {
|
||||
"run": "false",
|
||||
"year": "",
|
||||
"month": ""
|
||||
},
|
||||
"noTime": {
|
||||
"run": "true"
|
||||
},
|
||||
"fixedInterval": {
|
||||
"run": "false",
|
||||
"interval": 0
|
||||
}
|
||||
},
|
||||
"deadWaitTime": 0,
|
||||
"normalWaitTime": 0,
|
||||
"retryWaitTime": 10000,
|
||||
"sleepWaitTime": 3600000,
|
||||
"comaWaitTime": 86400000,
|
||||
"updatesDatasets": [
|
||||
{
|
||||
"product": "Candles",
|
||||
"dataSet": "Multi-Period-Daily"
|
||||
},
|
||||
{
|
||||
"product": "Volumes",
|
||||
"dataSet": "Multi-Period-Daily"
|
||||
}
|
||||
],
|
||||
"statusDependencies": [
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Exchange-Raw-Data",
|
||||
"process": "Historic-OHLCVs"
|
||||
},
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Exchange-Raw-Data",
|
||||
"process": "Historic-OHLCVs",
|
||||
"dataSetSection": "Month",
|
||||
"waitUntilNextUpdate": true
|
||||
},
|
||||
{
|
||||
"dataMine": "Masters",
|
||||
"bot": "Candles-Volumes",
|
||||
"process": "Multi-Period-Daily"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"products": [
|
||||
{
|
||||
"codeName": "Candles",
|
||||
"displayName": "Candles",
|
||||
"description": "Two sets of files per period that contains candles.",
|
||||
"storageAccount": "Candles-Volumes",
|
||||
"shareWith": "Public",
|
||||
"dataSets": [
|
||||
{
|
||||
"codeName": "Multi-Period-Market",
|
||||
"type": "Market Files",
|
||||
"validPeriods": [ "24-hs", "12-hs", "08-hs", "06-hs", "04-hs", "03-hs", "02-hs", "01-hs" ],
|
||||
"filePath": "Masters/Candles-Volumes.1.0/AACloud.1.1/@Exchange/dataSet.V1/Output/Candles/Multi-Period-Market/@Period",
|
||||
"fileName": "@AssetA_@AssetB.json"
|
||||
},
|
||||
{
|
||||
"codeName": "Multi-Period-Daily",
|
||||
"type": "Daily Files",
|
||||
"validPeriods": [ "45-min", "40-min", "30-min", "20-min", "15-min", "10-min", "05-min", "04-min", "03-min", "02-min", "01-min" ],
|
||||
"filePath": "Masters/Candles-Volumes.1.0/AACloud.1.1/@Exchange/dataSet.V1/Output/Candles/Multi-Period-Daily/@Period/@Year/@Month/@Day",
|
||||
"fileName": "@AssetA_@AssetB.json",
|
||||
"dataRange": {
|
||||
"filePath": "Masters/Candles-Volumes.1.0/AACloud.1.1/@Exchange/dataSet.V1/Output/Candles/Multi-Period-Daily",
|
||||
"fileName": "Data.Range.@AssetA_@AssetB.json"
|
||||
}
|
||||
}
|
||||
],
|
||||
"exchangeList": [
|
||||
{
|
||||
"name": "Poloniex"
|
||||
},
|
||||
{
|
||||
"name": "Coss"
|
||||
}
|
||||
],
|
||||
"plotter": {
|
||||
"dataMine": "Masters",
|
||||
"codeName": "PlottersCandlesVolumes",
|
||||
"moduleName": "Candles"
|
||||
}
|
||||
},
|
||||
{
|
||||
"codeName": "Volumes",
|
||||
"displayName": "Buy and Sell Volumes",
|
||||
"description": "Two sets of files per period that contains buy and sell volumes.",
|
||||
"storageAccount": "Candles-Volumes",
|
||||
"shareWith": "Public",
|
||||
"dataSets": [
|
||||
{
|
||||
"codeName": "Multi-Period-Market",
|
||||
"type": "Market Files",
|
||||
"validPeriods": [ "24-hs", "12-hs", "08-hs", "06-hs", "04-hs", "03-hs", "02-hs", "01-hs" ],
|
||||
"filePath": "Masters/Candles-Volumes.1.0/AACloud.1.1/@Exchange/dataSet.V1/Output/Volumes/Multi-Period-Market/@Period",
|
||||
"fileName": "@AssetA_@AssetB.json"
|
||||
},
|
||||
{
|
||||
"codeName": "Multi-Period-Daily",
|
||||
"type": "Daily Files",
|
||||
"validPeriods": [ "45-min", "40-min", "30-min", "20-min", "15-min", "10-min", "05-min", "04-min", "03-min", "02-min", "01-min" ],
|
||||
"filePath": "Masters/Candles-Volumes.1.0/AACloud.1.1/@Exchange/dataSet.V1/Output/Volumes/Multi-Period-Daily/@Period/@Year/@Month/@Day",
|
||||
"fileName": "@AssetA_@AssetB.json",
|
||||
"dataRange": {
|
||||
"filePath": "Masters/Candles-Volumes.1.0/AACloud.1.1/@Exchange/dataSet.V1/Output/Volumes/Multi-Period-Daily",
|
||||
"fileName": "Data.Range.@AssetA_@AssetB.json"
|
||||
}
|
||||
}
|
||||
],
|
||||
"exchangeList": [
|
||||
{
|
||||
"name": "Poloniex"
|
||||
},
|
||||
{
|
||||
"name": "Coss"
|
||||
}
|
||||
],
|
||||
"plotter": {
|
||||
"dataMine": "Masters",
|
||||
"codeName": "PlottersCandlesVolumes",
|
||||
"moduleName": "Volumes"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user