diff --git a/.vs/AABruce/v15/.suo b/.vs/AABruce/v15/.suo
index cc6cd63c2..109517424 100644
Binary files a/.vs/AABruce/v15/.suo and b/.vs/AABruce/v15/.suo differ
diff --git a/AABruce.njsproj b/AABruce.njsproj
index 710f0f820..c63c955f1 100644
--- a/AABruce.njsproj
+++ b/AABruce.njsproj
@@ -6,8 +6,8 @@
{045bbaef-3c35-4017-8458-e46aef438790}
ShowAllFiles
- One-Min-Daily-Candles\run.js
- C:\Users\Luis\source\repos\AABruce\One-Min-Daily-Candles\
+ Multi-Period-Daily-Candles-Volumes\run.js
+ C:\Users\Luis\source\repos\AABruce\Multi-Period-Daily-Candles-Volumes\
.
{3AF33F2E-1136-4D97-BBB7-1795711AC8B8};{349c5851-65df-11da-9384-00065b846f21};{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}
14.0
@@ -23,12 +23,15 @@
-
+
+
+
+
-
-
+
+
@@ -40,9 +43,10 @@
+
-
+
diff --git a/One-Min-Daily-Candles/One Min Daily Candles Volumes.js b/Multi-Period-Daily-Candles-Volumes/Interval Executor.js
similarity index 97%
rename from One-Min-Daily-Candles/One Min Daily Candles Volumes.js
rename to Multi-Period-Daily-Candles-Volumes/Interval Executor.js
index 7872e9903..c21c761ff 100644
--- a/One-Min-Daily-Candles/One Min Daily Candles Volumes.js
+++ b/Multi-Period-Daily-Candles-Volumes/Interval Executor.js
@@ -1,11 +1,11 @@
-exports.newPloniexHoleFixing = function newPloniexHoleFixing(BOT) {
+exports.newIntervalExecutor = function newIntervalExecutor(BOT) {
let bot = BOT;
const ROOT_DIR = '../';
const INTERVAL_LAPSE = 10 * 1000;
- const MODULE_NAME = "One Min Daily Candles Volumes";
+ const MODULE_NAME = "Interval Executor";
const LOG_INFO = true;
const INTERVAL_MODULE = require('./Interval');
diff --git a/One-Min-Daily-Candles/Interval.js b/Multi-Period-Daily-Candles-Volumes/Interval.js
similarity index 100%
rename from One-Min-Daily-Candles/Interval.js
rename to Multi-Period-Daily-Candles-Volumes/Interval.js
diff --git a/Multi-Period-Daily-Candles-Volumes/run.js b/Multi-Period-Daily-Candles-Volumes/run.js
new file mode 100644
index 000000000..43917e38e
--- /dev/null
+++ b/Multi-Period-Daily-Candles-Volumes/run.js
@@ -0,0 +1,97 @@
+
+let bot = {
+ name: "Bruce",
+ process: "Multi-Period-Daily-Candles-Volumes",
+ type: "Indicator",
+ version: "1.0.0",
+ devTeam: "AA Masters"
+};
+
+const ROOT_DIR = '../';
+
+const MODULE_NAME = "run";
+
+const START_ALL_MONTHS = false;
+const START_ONE_MONTH = true;
+
+const INTERVAL_EXECUTOR_MODULE = require('./Interval Executor');
+
+const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
+const logger = DEBUG_MODULE.newDebugLog();
+logger.fileName = MODULE_NAME;
+logger.bot = bot;
+
+process.on('uncaughtException', function (err) {
+ logger.write('uncaughtException - ' + err.message);
+});
+
+
+process.on('unhandledRejection', (reason, p) => {
+ logger.write("Unhandled Rejection at: Promise " + JSON.stringify(p) + " reason: " + reason);
+});
+
+
+process.on('exit', function (code) {
+ logger.write('About to exit with code:' + code);
+});
+
+
+try {
+ if (START_ALL_MONTHS === true) {
+
+ for (let year = 2018; year > 2014; year--) {
+
+ for (let month = 12; month > 0; month--) {
+
+ let timeDelay = Math.random() * 10 * 1000; // We introduce a short delay so as to not overload the machine.
+
+ setTimeout(startProcess, timeDelay);
+
+ function startProcess() {
+
+ const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot);
+
+ newIntervalExecutor.initialize(year, month, onInitializeReady);
+
+ function onInitializeReady() {
+
+ newIntervalExecutor.start();
+
+ }
+
+ }
+
+ }
+
+ }
+
+ }
+
+
+ if (START_ONE_MONTH === true) {
+
+ startProcess();
+
+ function startProcess() {
+
+ const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot);
+
+ let month = 2;
+ let year = 2018;
+
+ newIntervalExecutor.initialize(year, month, onInitializeReady);
+
+ function onInitializeReady() {
+
+ newIntervalExecutor.start();
+
+ }
+ }
+ }
+
+
+} catch (err) {
+ console.log(err.message);
+ logger.write(err.message);
+}
+
diff --git a/One-Min-Daily-Candles-Volumes/Interval Executor.js b/One-Min-Daily-Candles-Volumes/Interval Executor.js
new file mode 100644
index 000000000..c21c761ff
--- /dev/null
+++ b/One-Min-Daily-Candles-Volumes/Interval Executor.js
@@ -0,0 +1,169 @@
+exports.newIntervalExecutor = function newIntervalExecutor(BOT) {
+
+ let bot = BOT;
+ const ROOT_DIR = '../';
+
+ const INTERVAL_LAPSE = 10 * 1000;
+
+ const MODULE_NAME = "Interval Executor";
+ const LOG_INFO = true;
+
+ const INTERVAL_MODULE = require('./Interval');
+
+ const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
+ const logger = DEBUG_MODULE.newDebugLog();
+ logger.fileName = MODULE_NAME;
+ logger.bot = bot;
+
+ const INTERVAL_MAX_LOOPS = -1; // -1 for infinite loop.
+
+ poloniexHoleFixing = {
+ initialize: initialize,
+ start: start
+ };
+
+ const SHUTDOWN_EVENT = require(ROOT_DIR + 'Azure Web Job Shutdown');
+ let shutdownEvent = SHUTDOWN_EVENT.newShutdownEvent(bot);
+
+ const UTILITIES = require(ROOT_DIR + 'Utilities');
+ let utilities = UTILITIES.newUtilities(bot);
+
+ let year;
+ let month;
+
+ return poloniexHoleFixing;
+
+ function initialize(yearAssigend, monthAssigned, callBackFunction) {
+
+ try {
+
+ year = yearAssigend;
+ month = monthAssigned;
+
+ month = utilities.pad(month, 2); // Adding a left zero when needed.
+
+ logger.fileName = MODULE_NAME + "-" + year + "-" + month;
+
+ const logText = "[INFO] initialize - Entering function 'initialize' " + " @ " + year + "-" + month;
+ console.log(logText);
+ logger.write(logText);
+
+ callBackFunction();
+
+ } catch (err) {
+
+ const logText = "[ERROR] initialize - ' ERROR : " + err.message;
+ console.log(logText);
+ logger.write(logText);
+
+ }
+ }
+
+
+ function start() {
+
+ try {
+
+ if (LOG_INFO === true) {
+ logger.write("[INFO] Entering function 'start', with year = " + year + " and month = " + month);
+ }
+
+ let intervalId;
+
+ if (INTERVAL_MAX_LOOPS === 0) {
+
+ intervalId = setInterval(startNewInterval, INTERVAL_LAPSE);
+
+ }
+
+ let intervalLoopCounter = 0;
+
+ startNewInterval(); // First, inmadiate run.
+
+ function startNewInterval() {
+
+ try {
+
+ if (LOG_INFO === true) {
+
+ if (INTERVAL_MAX_LOOPS !== 0) {
+
+ logger.write("[INFO] Entering function 'startNewInterval', next execution inmediately after the current Interval finishes.");
+
+ } else {
+
+ logger.write("[INFO] Entering function 'startNewInterval', next execution in " + INTERVAL_LAPSE / 1000 / 60 + " minutes.");
+
+ }
+
+ }
+
+
+ if (shutdownEvent.isShuttingDown() === false) {
+
+ let fileProcessingInterval = INTERVAL_MODULE.newInterval(bot);
+
+ fileProcessingInterval.initialize(year, month, onReadyToStart);
+
+ function onReadyToStart() {
+
+ fileProcessingInterval.start(loopControl);
+
+ }
+
+ }
+ else {
+
+ clearInterval(intervalId);
+
+ const logText = "[INFO] requestNewData - Terminating Set Interval - About to exit gracefully the process. ";
+ console.log(logText);
+ logger.write(logText);
+
+ }
+
+ }
+ catch (err) {
+ const logText = "[ERROR] 'startNewInterval' - ERROR : " + err.message;
+ logger.write(logText);
+ }
+ }
+
+ function loopControl(moreIntervalExecutionsNeeded) {
+
+ if (moreIntervalExecutionsNeeded === true) {
+
+ intervalLoopCounter++;
+
+ if (intervalLoopCounter === INTERVAL_MAX_LOOPS) {
+
+ const logText = "[WARN] 'INTERVAL_MAX_LOOPS' reached. Stopping execution. ";
+ logger.write(logText);
+
+ } else {
+
+ const logText = "[WARN] 'loopControl'. Staring new Interval. ";
+ logger.write(logText);
+
+ setTimeout(startNewInterval, INTERVAL_LAPSE);// Looped Execution.
+
+ }
+
+ } else {
+
+ logger.write("[WARN] Received the signal from last Interval execution to not call it anymore. Stopping execution. ");
+
+ }
+
+ }
+
+ }
+ catch (err) {
+ const logText = "[ERROR] 'Start' - ERROR : " + err.message;
+ logger.write(logText);
+ }
+ }
+};
+
+
+
diff --git a/One-Min-Daily-Candles-Volumes/Interval.js b/One-Min-Daily-Candles-Volumes/Interval.js
new file mode 100644
index 000000000..a390644f0
--- /dev/null
+++ b/One-Min-Daily-Candles-Volumes/Interval.js
@@ -0,0 +1,1082 @@
+exports.newInterval = function newInterval(BOT) {
+
+ let bot = BOT;
+ const ROOT_DIR = '../';
+ const GMT_SECONDS = ':00.000 GMT+0000';
+ const GMT_MILI_SECONDS = '.000 GMT+0000';
+ const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000;
+
+ const MODULE_NAME = "Interval";
+ const LOG_INFO = true;
+
+ const EXCHANGE_NAME = "Poloniex";
+ const EXCHANGE_ID = 1;
+
+ const TRADES_FOLDER_NAME = "Trades";
+
+ const CANDLES_FOLDER_NAME = "Candles";
+ const CANDLES_ONE_MIN = "One-Min";
+
+ const VOLUMES_FOLDER_NAME = "Volumes";
+ const VOLUMES_ONE_MIN = "One-Min";
+
+ const GO_RANDOM = false;
+ const FORCE_MARKET = 2; // This allows to debug the execution of an specific market. Not intended for production.
+
+ const MARKETS_MODULE = require(ROOT_DIR + 'Markets');
+
+ const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
+ const logger = DEBUG_MODULE.newDebugLog();
+ logger.fileName = MODULE_NAME;
+ logger.bot = bot;
+
+ interval = {
+ initialize: initialize,
+ start: start
+ };
+
+ let markets;
+
+ const AZURE_FILE_STORAGE = require(ROOT_DIR + 'Azure File Storage');
+ let charlyAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
+ let bruceAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
+
+ const UTILITIES = require(ROOT_DIR + 'Utilities');
+ let utilities = UTILITIES.newUtilities(bot);
+
+ let year;
+ let month;
+
+ return interval;
+
+ function initialize(yearAssigend, monthAssigned, callBackFunction) {
+
+ try {
+
+ year = yearAssigend;
+ month = monthAssigned;
+
+ month = utilities.pad(month, 2); // Adding a left zero when needed.
+
+ logger.fileName = MODULE_NAME + "-" + year + "-" + month;
+
+ const logText = "[INFO] initialize - Entering function 'initialize' " + " @ " + year + "-" + month;
+ console.log(logText);
+ logger.write(logText);
+
+ charlyAzureFileStorage.initialize("Charly");
+ bruceAzureFileStorage.initialize("Bruce");
+
+ markets = MARKETS_MODULE.newMarkets(bot);
+ markets.initialize(callBackFunction);
+
+
+ } catch (err) {
+
+ const logText = "[ERROR] initialize - ' ERROR : " + err.message;
+ console.log(logText);
+ logger.write(logText);
+
+ }
+ }
+
+/*
+
+This process is going to do the following:
+
+Read the trades from Charly's Output and pack them into daily files with candles of one minute.
+
+*/
+
+ function start(callBackFunction) {
+
+ try {
+
+ if (LOG_INFO === true) {
+ logger.write("[INFO] Entering function 'start', with year = " + year + " and month = " + month);
+ }
+
+ let processDate = new Date(year + "-" + month + "-1 00:00:00.000 GMT+0000");
+
+ let lastMinuteOfMonth = new Date(year + "-" + month + "-1 00:00:00.000 GMT+0000");
+
+ lastMinuteOfMonth.setUTCMonth(lastMinuteOfMonth.getUTCMonth() + 1); // First we go 1 month into the future.
+ lastMinuteOfMonth.setUTCSeconds(lastMinuteOfMonth.getUTCSeconds() - 30); // Then we go back 30 seconds, or to the last minute of the original month.
+
+ let thisDatetime = new Date();
+
+ if ((year === thisDatetime.getUTCFullYear() && parseInt(month) > thisDatetime.getUTCMonth() + 1) || year > thisDatetime.getUTCFullYear()) {
+
+ logger.write("[INFO] We are too far in the future. Interval will not execute. Sorry.");
+ return;
+
+ }
+
+ let atHeadOfMarket; // This tell us if we are at the month which includes the head of the market according to current datetime.
+
+ if ((year === thisDatetime.getUTCFullYear() && parseInt(month) === thisDatetime.getUTCMonth() + 1)) {
+
+ atHeadOfMarket = true;
+
+ } else {
+
+ atHeadOfMarket = false;
+
+ }
+
+ let nextIntervalExecution = false; // This tell weather the Interval module will be executed again or not. By default it will not unless some hole have been found in the current execution.
+
+ let marketQueue; // This is the queue of all markets to be procesesd at each interval.
+ let market = { // This is the current market being processed after removing it from the queue.
+ id: 0,
+ assetA: "",
+ assetB: ""
+ };
+
+ let lastCandleFile; // Datetime of the last file certified by the Hole Fixing process as without permanent holes.
+ let firstTradeFile; // Datetime of the first trade file in the whole market history.
+ let lastFileWithoutHoles; // Datetime of the last verified file without holes.
+ let lastCandleClose; // Value of the last candle close.
+ let lastTradeFile; // Datetime pointing to the last Trade File sucessfuly processed and included in the last file.
+
+ marketsLoop();
+
+ /*
+
+ At every run, the process needs to loop through all the markets at this exchange.
+ The following functions marketsLoop(), openMarket(), closeMarket() and closeAndOpenMarket() controls the serialization of this processing.
+
+ */
+
+ function marketsLoop() {
+
+ try {
+
+ if (LOG_INFO === true) {
+ logger.write("[INFO] Entering function 'marketsLoop'");
+ }
+
+ markets.getMarketsByExchange(EXCHANGE_ID, onMarketsReady);
+
+ function onMarketsReady(marketsArray) {
+
+ marketQueue = JSON.parse(marketsArray);
+
+ openMarket(); // First execution and entering into the real loop.
+
+ }
+ }
+ catch (err) {
+ const logText = "[ERROR] 'marketsLoop' - ERROR : " + err.message;
+ logger.write(logText);
+ }
+ }
+
+ function openMarket() {
+
+ // To open a Market means picking a new market from the queue.
+
+ try {
+
+ if (LOG_INFO === true) {
+ logger.write("[INFO] Entering function 'openMarket'");
+ }
+
+
+ if (marketQueue.length === 0) {
+
+ if (LOG_INFO === true) {
+ logger.write("[INFO] 'openMarket' - marketQueue.length === 0");
+ }
+
+ const logText = "[WARN] We processed all the markets.";
+ logger.write(logText);
+
+ callBackFunction(nextIntervalExecution);
+
+ return;
+ }
+
+ if (GO_RANDOM === true) {
+ const index = parseInt(Math.random() * (marketQueue.length - 1));
+
+ market.id = marketQueue[index][0];
+ market.assetA = marketQueue[index][1];
+ market.assetB = marketQueue[index][2];
+ market.status = marketQueue[index][3];
+
+ marketQueue.splice(index, 1);
+ } else {
+ let marketRecord = marketQueue.shift();
+
+ market.id = marketRecord[0];
+ market.assetA = marketRecord[1];
+ market.assetB = marketRecord[2];
+ market.status = marketRecord[3];
+
+ if (FORCE_MARKET > 0) {
+ if (FORCE_MARKET !== market.id) {
+ closeAndOpenMarket();
+ return;
+ }
+ }
+ }
+
+ if (LOG_INFO === true) {
+ logger.write("[INFO] 'openMarket' - marketQueue.length = " + marketQueue.length);
+ logger.write("[INFO] 'openMarket' - market sucessfully opened : " + market.assetA + "_" + market.assetB);
+ }
+
+ if (market.status === markets.ENABLED) {
+
+ getStatusReport();
+
+ } else {
+
+ logger.write("[INFO] 'openMarket' - market " + market.assetA + "_" + market.assetB + " skipped because its status is not valid. Status = " + market.status);
+ closeAndOpenMarket();
+ return;
+
+ }
+
+
+ }
+ catch (err) {
+ const logText = "[ERROR] 'openMarket' - ERROR : " + err.message;
+ logger.write(logText);
+ closeMarket();
+ }
+ }
+
+ function closeMarket() {
+
+ if (LOG_INFO === true) {
+ logger.write("[INFO] Entering function 'closeMarket'");
+ }
+
+ }
+
+ function closeAndOpenMarket() {
+
+ if (LOG_INFO === true) {
+ logger.write("[INFO] Entering function 'closeAndOpenMarket'");
+ }
+
+ openMarket();
+ }
+
+ /*
+
+ The following code executes for each market.
+
+ */
+
+ function getStatusReport() {
+
+ try {
+
+ let reportFilePath;
+ let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json"
+
+ getHistoricTrades();
+
+ function getHistoricTrades() {
+
+ /*
+
+ We need to know where is the begining of the market, since that will help us know how to estimate the value of the last close.
+ If we are at the begining of the market, the last close should be zero.
+
+ */
+
+ reportFilePath = EXCHANGE_NAME + "/Processes/" + "Historic-Trades";
+
+ charlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
+
+ function onStatusReportReceived(text) {
+
+ let statusReport;
+
+ try {
+
+ statusReport = JSON.parse(text);
+
+ firstTradeFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS);
+
+ if ((year === firstTradeFile.getUTCFullYear() && parseInt(month) < firstTradeFile.getUTCMonth() + 1) || year < firstTradeFile.getUTCFullYear()) {
+
+ const logText = "[INFO] 'getStatusReport' - the requested month / year are before the begining of this market " + market.assetA + '_' + market.assetB + " . Skipping it. ";
+ logger.write(logText);
+
+ closeAndOpenMarket();
+
+ } else {
+
+ getHoleFixing();
+
+ }
+
+ } catch (err) {
+
+ const logText = "[INFO] 'getStatusReport' - Failed to read main Historic Trades Status Report for market " + market.assetA + '_' + market.assetB + " . Skipping it. ";
+ logger.write(logText);
+
+ closeAndOpenMarket();
+ }
+ }
+ }
+
+ function getHoleFixing() {
+
+ /*
+
+ The limit in the future as of which candles to include is determined by the Hole Fixing process. We wont include
+ trades not certified to be without hole.
+
+ */
+
+ reportFilePath = EXCHANGE_NAME + "/Processes/" + "Hole-Fixing" + "/" + year + "/" + month;
+
+ charlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
+
+ function onStatusReportReceived(text) {
+
+ let statusReport;
+
+ try {
+ statusReport = JSON.parse(text);
+ }
+ catch (err) {
+ text = undefined; // If the content of the file is corrupt, this equals as if the file did not exist.
+ }
+
+ if (text === undefined) {
+
+ const logText = "[INFO] 'getStatusReport' - The current year / month was not yet hole-fixed for market " + market.assetA + '_' + market.assetB + " . Skipping it. ";
+ logger.write(logText);
+
+ closeAndOpenMarket();
+
+ } else {
+
+ if (statusReport.monthChecked === true) {
+
+ lastFileWithoutHoles = new Date(); // We need this with a valid value.
+ getOneMinDailyCandlesVolumes();
+
+ } else {
+
+ /*
+
+ If the hole report is incomplete, we are only interested if we are at the head of the market.
+ Otherwise, we are not going to calculate the candles of a month which was not fully checked for holes.
+
+ */
+
+ if (atHeadOfMarket === true) {
+
+ lastFileWithoutHoles = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS);
+ getOneMinDailyCandlesVolumes();
+
+ } else {
+
+ const logText = "[INFO] 'getStatusReport' - The current year / month was not completely hole-fixed for market " + market.assetA + '_' + market.assetB + " . Skipping it. ";
+ logger.write(logText);
+
+ closeAndOpenMarket();
+
+ }
+ }
+ }
+ }
+ }
+
+ function getOneMinDailyCandlesVolumes() {
+
+ /* If the process run and was interrupted, there should be a status report that allows us to resume execution. */
+
+ reportFilePath = EXCHANGE_NAME + "/Processes/" + "One-Min-Daily-Candles-Volumes" + "/" + year + "/" + month;
+
+ bruceAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
+
+ function onStatusReportReceived(text) {
+
+ let statusReport;
+
+ try {
+
+ statusReport = JSON.parse(text);
+
+ if (statusReport.monthCompleted === true) {
+
+ const logText = "[INFO] 'getStatusReport' - The current year / month is already complete for market " + market.assetA + '_' + market.assetB + " . Skipping it. ";
+ logger.write(logText);
+
+ closeAndOpenMarket();
+
+ } else {
+
+ lastCandleFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + "00:00" + GMT_SECONDS);
+ lastCandleClose = statusReport.candleClose;
+
+ if (statusReport.fileComplete === true) {
+
+ buildCandles();
+
+ } else {
+
+ lastTradeFile = new Date(statusReport.lastTradeFile.year + "-" + statusReport.lastTradeFile.month + "-" + statusReport.lastTradeFile.days + " " + statusReport.lastTradeFile.hours + ":" + statusReport.lastTradeFile.minutes + GMT_SECONDS);
+ findPreviousContent();
+
+ }
+ }
+
+ } catch (err) {
+
+ /*
+
+ It might happen that the file content is corrupt or it does not exist. In either case we will point our lastCandleFile
+ to the last day of the previous month.
+
+ */
+
+ lastCandleFile = new Date(processDate.valueOf() - ONE_DAY_IN_MILISECONDS);
+ findLastCandleCloseValue();
+
+ }
+ }
+ }
+
+ }
+ catch (err) {
+ const logText = "[ERROR] 'getStatusReport' - ERROR : " + err.message;
+ logger.write(logText);
+ closeMarket();
+ }
+ }
+
+ function findPreviousContent() {
+
+ try {
+
+ let previousCandles;
+ let previousVolumes;
+
+ getCandles();
+
+ function getCandles() {
+
+ let fileName = '' + market.assetA + '_' + market.assetB + '.json';
+
+ let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2);
+
+ let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
+
+ bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
+
+ function onFileReceived(text) {
+
+ let candlesFile;
+
+ try {
+
+ candlesFile = JSON.parse(text);
+
+ previousCandles = candlesFile;
+
+ getVolumes();
+
+ } catch (err) {
+
+ const logText = "[ERR] 'findPreviousContent' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
+ logger.write(logText);
+
+ closeAndOpenMarket();
+ }
+ }
+ }
+
+ function getVolumes() {
+
+ let fileName = '' + market.assetA + '_' + market.assetB + '.json';
+
+ let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2);
+
+ let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
+
+ bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
+
+ function onFileReceived(text) {
+
+ let volumesFile;
+
+ try {
+
+ volumesFile = JSON.parse(text);
+
+ previousVolumes = volumesFile;
+
+ lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); // We know that after the next call a new day will be added.
+
+ buildCandles(previousCandles, previousVolumes);
+
+ } catch (err) {
+
+ const logText = "[ERR] 'findPreviousContent' - Empty or corrupt volume file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
+ logger.write(logText);
+
+ closeAndOpenMarket();
+ }
+ }
+ }
+ }
+ catch (err) {
+ const logText = "[ERROR] 'findPreviousContent' - ERROR : " + err.message;
+ logger.write(logText);
+ closeMarket();
+ }
+
+ }
+
+ function findLastCandleCloseValue() {
+
+ try {
+
+ /*
+
+ We will search and find for the last trade before the begining of the current candle and that will give us the last close value.
+ Before going backwards, we need to be sure we are not at the begining of the market.
+
+ */
+
+ if ((year === firstTradeFile.getUTCFullYear() && parseInt(month) === firstTradeFile.getUTCMonth() + 1)) {
+
+ /*
+
+ We are at the begining of the market, so we will set everyting to build the first candle.
+
+ */
+
+ const logText = "[INFO] 'findLastCandleCloseValue' - Begining of the market detected for market " + market.assetA + '_' + market.assetB + " . lastCandleClose = " + lastCandleClose;
+ logger.write(logText);
+ console.log(logText);
+
+ lastCandleFile = new Date(firstTradeFile.getUTCFullYear() + "-" + (firstTradeFile.getUTCMonth() + 1) + "-" + firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
+ lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS);
+
+ lastCandleClose = 0;
+
+ buildCandles();
+
+ } else {
+
+ /*
+
+ We are not at the begining of the market, so we need scan backwards the trade files until we find a non empty one and get the last trade.
+
+ */
+
+ let date = new Date(processDate.valueOf());
+
+ loopStart();
+
+ function loopStart() {
+
+ date = new Date(date.valueOf() - 60 * 1000);
+
+ let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2);
+ let fileName = market.assetA + '_' + market.assetB + ".json"
+ let filePath = EXCHANGE_NAME + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath;
+
+ charlyAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
+
+ function onFileReceived(text) {
+
+ let tradesFile;
+
+ try {
+
+ tradesFile = JSON.parse(text);
+
+ if (tradesFile.length > 0) {
+
+ lastCandleClose = tradesFile[tradesFile.length - 1][2]; // Position 2 is the rate at which the trade was executed.
+
+ const logText = "[INFO] 'findLastCandleCloseValue' - Trades found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . lastCandleClose = " + lastCandleClose;
+ logger.write(logText);
+ console.log(logText);
+
+ buildCandles();
+
+ } else {
+
+ const logText = "[INFO] 'findLastCandleCloseValue' - No trades found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " .";
+ logger.write(logText);
+ console.log(logText);
+
+ loopStart();
+
+ }
+
+ } catch (err) {
+
+ const logText = "[ERR] 'findLastCandleCloseValue' - Empty or corrupt trade file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
+ logger.write(logText);
+
+ closeAndOpenMarket();
+ }
+ }
+ }
+ }
+ }
+ catch (err) {
+ const logText = "[ERROR] 'findLastCandleCloseValue' - ERROR : " + err.message;
+ logger.write(logText);
+ closeMarket();
+ }
+ }
+
+ function buildCandles(previousCandles, previousVolumes) {
+
+ /*
+
+ Here we are going to scan the trades files packing them in candles files every one day.
+ We need for this the last close value, bacause all candles that are empty of trades at the begining, they need to
+ have a valid open and close value. This was previously calculated before arriving to this function.
+
+ */
+
+ let canAddPrevious = true;
+
+ try {
+
+
+ nextCandleFile();
+
+ function nextCandleFile() {
+
+ lastCandleFile = new Date(lastCandleFile.valueOf() + ONE_DAY_IN_MILISECONDS);
+
+ let date = new Date(lastCandleFile.valueOf() - 60 * 1000);
+
+ if (date.valueOf() < firstTradeFile.valueOf()) { // At the special case where we are at the begining of the market, this might be true.
+
+ date = new Date(firstTradeFile.valueOf() - 60 * 1000);
+
+ }
+
+ if (lastTradeFile !== undefined) {
+
+ date = new Date(lastTradeFile.valueOf());
+
+ }
+
+ let candles = [];
+ let volumes = [];
+
+ if (previousCandles !== undefined && canAddPrevious === true) {
+
+ for (let i = 0; i < previousCandles.length; i++) {
+
+ let candle = {
+ open: previousCandles[i][2],
+ close: previousCandles[i][3],
+ min: previousCandles[i][0],
+ max: previousCandles[i][1],
+ begin: previousCandles[i][4],
+ end: previousCandles[i][5]
+ };
+
+ candles.push(candle);
+ }
+
+ }
+
+ if (previousVolumes !== undefined && canAddPrevious === true) {
+
+ for (let i = 0; i < previousVolumes.length; i++) {
+
+ let volume = {
+ begin: previousVolumes[i][2],
+ end: previousVolumes[i][3],
+ buy: previousVolumes[i][0],
+ sell: previousVolumes[i][1]
+ };
+
+ volumes.push(volume);
+ }
+
+ }
+
+ canAddPrevious = false; // We add them only onece.
+
+ nextDate();
+
+ function nextDate() {
+
+ date = new Date(date.valueOf() + 60 * 1000);
+
+ /* Check if we are outside the current Day / File */
+
+ if (date.getUTCDate() !== lastCandleFile.getUTCDate()) {
+
+ writeFiles(lastCandleFile, candles, volumes, true, onFilesWritten);
+
+ return;
+
+ function onFilesWritten() {
+
+ nextCandleFile();
+
+ }
+
+ }
+
+ /* Check if we are outside the currrent Month */
+
+ if (date.getUTCMonth() + 1 !== parseInt(month)) {
+
+ lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS);
+
+ writeStatusReport(lastCandleFile, lastTradeFile, lastCandleClose, true, true, onStatusReportWritten);
+
+ return;
+
+ function onStatusReportWritten() {
+
+ const logText = "[ERR] 'buildCandles' - Finishing processing the whole month for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
+ logger.write(logText);
+
+ closeAndOpenMarket();
+
+ }
+ }
+
+ /* Check if we have past the most recent hole fixed file */
+
+ if (date.valueOf() > lastFileWithoutHoles.valueOf()) {
+
+ writeFiles(lastCandleFile, candles, volumes, false, onFilesWritten);
+
+ return;
+
+ function onFilesWritten() {
+
+ nextIntervalExecution = true;
+
+ const logText = "[ERR] 'buildCandles' - Head of the market reached for market " + market.assetA + '_' + market.assetB + " . ";
+ logger.write(logText);
+
+ closeAndOpenMarket();
+
+ }
+ }
+
+ readTrades();
+ }
+
+ function readTrades() {
+
+ lastTradeFile = new Date(date.valueOf());
+
+ let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2);
+ let fileName = market.assetA + '_' + market.assetB + ".json"
+ let filePath = EXCHANGE_NAME + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath;
+
+ charlyAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
+
+ function onFileReceived(text) {
+
+ let tradesFile;
+
+ try {
+
+ let candle = {
+ open: lastCandleClose,
+ close: lastCandleClose,
+ min: lastCandleClose,
+ max: lastCandleClose,
+ begin: date.valueOf(),
+ end: date.valueOf() + 60 * 1000 - 1
+ };
+
+ let volume = {
+ begin: date.valueOf(),
+ end: date.valueOf() + 60 * 1000 - 1,
+ buy: 0,
+ sell: 0
+ };
+
+ tradesFile = JSON.parse(text);
+
+ let tradesCount = utilities.pad(tradesFile.length, 5);
+
+ const logText = "[INFO] 'buildCandles' - " + tradesCount +" trades found at " + filePath + " for market " + market.assetA + '_' + market.assetB + ". ";
+ logger.write(logText);
+ console.log(logText);
+
+ if (tradesFile.length > 0) {
+
+ /* Candle open and close Calculations */
+
+ candle.open = tradesFile[0][2];
+ candle.close = tradesFile[tradesFile.length - 1][2];
+
+ lastCandleClose = candle.close;
+
+ }
+
+ for (let i = 0; i < tradesFile.length; i++) {
+
+ const trade = {
+ id: tradesFile[i][0],
+ type: tradesFile[i][1],
+ rate: tradesFile[i][2],
+ amountA: tradesFile[i][3],
+ amountB: tradesFile[i][4],
+ seconds: tradesFile[i][5]
+ };
+
+ /* Candle min and max Calculations */
+
+ if (trade.rate < candle.min) {
+
+ candle.min = trade.rate;
+
+ }
+
+ if (trade.rate > candle.max) {
+
+ candle.max = trade.rate;
+
+ }
+
+ /* Volume Calculations */
+
+ if (trade.type === "sell") {
+ volume.sell = volume.sell + trade.amountA;
+ } else {
+ volume.buy = volume.buy + trade.amountA;
+ }
+
+ }
+
+ candles.push(candle);
+
+ volumes.push(volume);
+
+ nextDate();
+
+ } catch (err) {
+
+ const logText = "[ERR] 'buildCandles' - Empty or corrupt trade file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
+ logger.write(logText);
+
+ closeAndOpenMarket();
+ }
+ }
+ }
+ }
+ }
+
+ catch (err) {
+ const logText = "[ERROR] 'buildCandles' - ERROR : " + err.message;
+ logger.write(logText);
+ closeMarket();
+ }
+
+ }
+
+ function writeFiles(date, candles, volumes, isFileComplete, callBack) {
+
+ /*
+
+ Here we will write the contents of the Candles and Volumens files. If the File is declared as complete, we will also write the status report.
+
+ */
+
+ try {
+
+ writeCandles();
+
+ function writeCandles() {
+
+ let separator = "";
+ let fileRecordCounter = 0;
+
+ let fileContent = "";
+
+ for (i = 0; i < candles.length; i++) {
+
+ let candle = candles[i];
+
+ fileContent = fileContent + separator + '[' + candles[i].min + "," + candles[i].max + "," + candles[i].open + "," + candles[i].close + "," + candles[i].begin + "," + candles[i].end + "]";
+
+ if (separator === "") { separator = ","; }
+
+ fileRecordCounter++;
+
+ }
+
+ fileContent = "[" + fileContent + "]";
+
+ let fileName = '' + market.assetA + '_' + market.assetB + '.json';
+
+ let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2);
+
+ let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
+
+ utilities.createFolderIfNeeded(filePath, bruceAzureFileStorage, onFolderCreated);
+
+ function onFolderCreated() {
+
+ bruceAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated);
+
+ function onFileCreated() {
+
+ const logText = "[WARN] Finished with File @ " + market.assetA + "_" + market.assetB + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName + "";
+ console.log(logText);
+ logger.write(logText);
+
+ writeVolumes();
+ }
+ }
+
+ }
+
+ function writeVolumes() {
+
+ let separator = "";
+ let fileRecordCounter = 0;
+
+ let fileContent = "";
+
+ for (i = 0; i < volumes.length; i++) {
+
+ let candle = volumes[i];
+
+ fileContent = fileContent + separator + '[' + volumes[i].buy + "," + volumes[i].sell + "," + volumes[i].begin + "," + volumes[i].end + "]";
+
+ if (separator === "") { separator = ","; }
+
+ fileRecordCounter++;
+
+ }
+
+ fileContent = "[" + fileContent + "]";
+
+ let fileName = '' + market.assetA + '_' + market.assetB + '.json';
+
+ let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2);
+
+ let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath;
+
+ utilities.createFolderIfNeeded(filePath, bruceAzureFileStorage, onFolderCreated);
+
+ function onFolderCreated() {
+
+ bruceAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated);
+
+ function onFileCreated() {
+
+ const logText = "[WARN] Finished with File @ " + market.assetA + "_" + market.assetB + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName + "";
+ console.log(logText);
+ logger.write(logText);
+
+ writeReport();
+ }
+ }
+ }
+
+ function writeReport() {
+
+ writeStatusReport(date, lastTradeFile, lastCandleClose, isFileComplete, false, onStatusReportWritten);
+
+ function onStatusReportWritten() {
+
+ callBack();
+
+ }
+ }
+ }
+
+ catch (err) {
+ const logText = "[ERROR] 'writeFiles' - ERROR : " + err.message;
+ logger.write(logText);
+ closeMarket();
+ }
+ }
+
+ function writeStatusReport(lastFileDate, lastTradeFile, candleClose, isFileComplete, isMonthComplete, callBack) {
+
+
+ if (LOG_INFO === true) {
+ logger.write("[INFO] Entering function 'writeStatusReport'");
+ }
+
+ try {
+
+ let reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process + "/" + year + "/" + month;
+
+ utilities.createFolderIfNeeded(reportFilePath, bruceAzureFileStorage, onFolderCreated);
+
+ function onFolderCreated() {
+
+ try {
+
+ let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json";
+
+ let report = {
+ lastFile: {
+ year: lastFileDate.getUTCFullYear(),
+ month: (lastFileDate.getUTCMonth() + 1),
+ days: lastFileDate.getUTCDate()
+ },
+ lastTradeFile: {
+ year: lastTradeFile.getUTCFullYear(),
+ month: (lastTradeFile.getUTCMonth() + 1),
+ days: lastTradeFile.getUTCDate(),
+ hours: lastTradeFile.getUTCHours(),
+ minutes: lastTradeFile.getUTCMinutes()
+ },
+ candleClose: candleClose,
+ monthCompleted: isMonthComplete,
+ fileComplete: isFileComplete
+ };
+
+ let fileContent = JSON.stringify(report);
+
+ bruceAzureFileStorage.createTextFile(reportFilePath, fileName, fileContent + '\n', onFileCreated);
+
+ function onFileCreated() {
+
+ if (LOG_INFO === true) {
+ logger.write("[INFO] 'writeStatusReport' - Content written: " + fileContent);
+ }
+
+ callBack();
+ }
+ }
+ catch (err) {
+ const logText = "[ERROR] 'writeStatusReport - onFolderCreated' - ERROR : " + err.message;
+ logger.write(logText);
+ closeMarket();
+ }
+ }
+
+ }
+ catch (err) {
+ const logText = "[ERROR] 'writeStatusReport' - ERROR : " + err.message;
+ logger.write(logText);
+ closeMarket();
+ }
+
+ }
+
+ }
+ catch (err) {
+ const logText = "[ERROR] 'Start' - ERROR : " + err.message;
+ logger.write(logText);
+ }
+ }
+};
diff --git a/One-Min-Daily-Candles/run.js b/One-Min-Daily-Candles-Volumes/run.js
similarity index 63%
rename from One-Min-Daily-Candles/run.js
rename to One-Min-Daily-Candles-Volumes/run.js
index 1b32a2f5e..b56ac1c24 100644
--- a/One-Min-Daily-Candles/run.js
+++ b/One-Min-Daily-Candles-Volumes/run.js
@@ -4,17 +4,17 @@ let bot = {
process: "One-Min-Daily-Candles-Volumes",
type: "Indicator",
version: "1.0.0",
- devTeam: "AA Masters"
+ devTeam: "AA Masters"
};
const ROOT_DIR = '../';
const MODULE_NAME = "run";
-const START_ONE_MIN_DAILY_CANDLES_VOLUMES = false;
-const START_ONE_MIN_DAILY_CANDLES_VOLUMES_AT_ONE_MONTH = true;
+const START_ALL_MONTHS = false;
+const START_ONE_MONTH = true;
-const ONE_MIN_DAILY_CANDLES_MODULE = require('./One Min Daily Candles Volumes');
+const INTERVAL_EXECUTOR_MODULE = require('./Interval Executor');
const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
const logger = DEBUG_MODULE.newDebugLog();
@@ -37,7 +37,7 @@ process.on('exit', function (code) {
try {
- if (START_ONE_MIN_DAILY_CANDLES_VOLUMES === true) {
+ if (START_ALL_MONTHS === true) {
for (let year = 2018; year > 2014; year--) {
@@ -49,13 +49,13 @@ try {
function startProcess() {
- const newPloniexHoleFixing = ONE_MIN_DAILY_CANDLES_MODULE.newPloniexHoleFixing(bot);
+ const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot);
- newPloniexHoleFixing.initialize(year, month, onInitializeReady);
+ newIntervalExecutor.initialize(year, month, onInitializeReady);
function onInitializeReady() {
- newPloniexHoleFixing.start();
+ newIntervalExecutor.start();
}
@@ -68,22 +68,22 @@ try {
}
- if (START_ONE_MIN_DAILY_CANDLES_VOLUMES_AT_ONE_MONTH === true) {
+ if (START_ONE_MONTH === true) {
startProcess();
function startProcess() {
- const newPloniexHoleFixing = ONE_MIN_DAILY_CANDLES_MODULE.newPloniexHoleFixing(bot);
+ const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot);
let month = 2;
let year = 2018;
- newPloniexHoleFixing.initialize(year, month, onInitializeReady);
+ newIntervalExecutor.initialize(year, month, onInitializeReady);
function onInitializeReady() {
- newPloniexHoleFixing.start();
+ newIntervalExecutor.start();
}
}