diff --git a/.vs/AABruce/v15/.suo b/.vs/AABruce/v15/.suo
index 109517424..2be5496f9 100644
Binary files a/.vs/AABruce/v15/.suo and b/.vs/AABruce/v15/.suo differ
diff --git a/AABruce.njsproj b/AABruce.njsproj
index c63c955f1..29412cde0 100644
--- a/AABruce.njsproj
+++ b/AABruce.njsproj
@@ -6,8 +6,8 @@
{045bbaef-3c35-4017-8458-e46aef438790}
ShowAllFiles
- Multi-Period-Daily-Candles-Volumes\run.js
- C:\Users\Luis\source\repos\AABruce\Multi-Period-Daily-Candles-Volumes\
+ One-Min-Daily-Candles-Volumes\run.js
+ C:\Users\Luis\source\repos\One-Min-Daily-Candles-Volumes\
.
{3AF33F2E-1136-4D97-BBB7-1795711AC8B8};{349c5851-65df-11da-9384-00065b846f21};{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}
14.0
@@ -23,9 +23,6 @@
-
-
-
@@ -43,7 +40,6 @@
-
diff --git a/Multi-Period-Daily-Candles-Volumes/Interval Executor.js b/Multi-Period-Daily-Candles-Volumes/Interval Executor.js
deleted file mode 100644
index c21c761ff..000000000
--- a/Multi-Period-Daily-Candles-Volumes/Interval Executor.js
+++ /dev/null
@@ -1,169 +0,0 @@
-exports.newIntervalExecutor = function newIntervalExecutor(BOT) {
-
- let bot = BOT;
- const ROOT_DIR = '../';
-
- const INTERVAL_LAPSE = 10 * 1000;
-
- const MODULE_NAME = "Interval Executor";
- const LOG_INFO = true;
-
- const INTERVAL_MODULE = require('./Interval');
-
- const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
- const logger = DEBUG_MODULE.newDebugLog();
- logger.fileName = MODULE_NAME;
- logger.bot = bot;
-
- const INTERVAL_MAX_LOOPS = -1; // -1 for infinite loop.
-
- poloniexHoleFixing = {
- initialize: initialize,
- start: start
- };
-
- const SHUTDOWN_EVENT = require(ROOT_DIR + 'Azure Web Job Shutdown');
- let shutdownEvent = SHUTDOWN_EVENT.newShutdownEvent(bot);
-
- const UTILITIES = require(ROOT_DIR + 'Utilities');
- let utilities = UTILITIES.newUtilities(bot);
-
- let year;
- let month;
-
- return poloniexHoleFixing;
-
- function initialize(yearAssigend, monthAssigned, callBackFunction) {
-
- try {
-
- year = yearAssigend;
- month = monthAssigned;
-
- month = utilities.pad(month, 2); // Adding a left zero when needed.
-
- logger.fileName = MODULE_NAME + "-" + year + "-" + month;
-
- const logText = "[INFO] initialize - Entering function 'initialize' " + " @ " + year + "-" + month;
- console.log(logText);
- logger.write(logText);
-
- callBackFunction();
-
- } catch (err) {
-
- const logText = "[ERROR] initialize - ' ERROR : " + err.message;
- console.log(logText);
- logger.write(logText);
-
- }
- }
-
-
- function start() {
-
- try {
-
- if (LOG_INFO === true) {
- logger.write("[INFO] Entering function 'start', with year = " + year + " and month = " + month);
- }
-
- let intervalId;
-
- if (INTERVAL_MAX_LOOPS === 0) {
-
- intervalId = setInterval(startNewInterval, INTERVAL_LAPSE);
-
- }
-
- let intervalLoopCounter = 0;
-
- startNewInterval(); // First, inmadiate run.
-
- function startNewInterval() {
-
- try {
-
- if (LOG_INFO === true) {
-
- if (INTERVAL_MAX_LOOPS !== 0) {
-
- logger.write("[INFO] Entering function 'startNewInterval', next execution inmediately after the current Interval finishes.");
-
- } else {
-
- logger.write("[INFO] Entering function 'startNewInterval', next execution in " + INTERVAL_LAPSE / 1000 / 60 + " minutes.");
-
- }
-
- }
-
-
- if (shutdownEvent.isShuttingDown() === false) {
-
- let fileProcessingInterval = INTERVAL_MODULE.newInterval(bot);
-
- fileProcessingInterval.initialize(year, month, onReadyToStart);
-
- function onReadyToStart() {
-
- fileProcessingInterval.start(loopControl);
-
- }
-
- }
- else {
-
- clearInterval(intervalId);
-
- const logText = "[INFO] requestNewData - Terminating Set Interval - About to exit gracefully the process. ";
- console.log(logText);
- logger.write(logText);
-
- }
-
- }
- catch (err) {
- const logText = "[ERROR] 'startNewInterval' - ERROR : " + err.message;
- logger.write(logText);
- }
- }
-
- function loopControl(moreIntervalExecutionsNeeded) {
-
- if (moreIntervalExecutionsNeeded === true) {
-
- intervalLoopCounter++;
-
- if (intervalLoopCounter === INTERVAL_MAX_LOOPS) {
-
- const logText = "[WARN] 'INTERVAL_MAX_LOOPS' reached. Stopping execution. ";
- logger.write(logText);
-
- } else {
-
- const logText = "[WARN] 'loopControl'. Staring new Interval. ";
- logger.write(logText);
-
- setTimeout(startNewInterval, INTERVAL_LAPSE);// Looped Execution.
-
- }
-
- } else {
-
- logger.write("[WARN] Received the signal from last Interval execution to not call it anymore. Stopping execution. ");
-
- }
-
- }
-
- }
- catch (err) {
- const logText = "[ERROR] 'Start' - ERROR : " + err.message;
- logger.write(logText);
- }
- }
-};
-
-
-
diff --git a/Multi-Period-Daily-Candles-Volumes/Interval.js b/Multi-Period-Daily-Candles-Volumes/Interval.js
deleted file mode 100644
index a390644f0..000000000
--- a/Multi-Period-Daily-Candles-Volumes/Interval.js
+++ /dev/null
@@ -1,1082 +0,0 @@
-exports.newInterval = function newInterval(BOT) {
-
- let bot = BOT;
- const ROOT_DIR = '../';
- const GMT_SECONDS = ':00.000 GMT+0000';
- const GMT_MILI_SECONDS = '.000 GMT+0000';
- const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000;
-
- const MODULE_NAME = "Interval";
- const LOG_INFO = true;
-
- const EXCHANGE_NAME = "Poloniex";
- const EXCHANGE_ID = 1;
-
- const TRADES_FOLDER_NAME = "Trades";
-
- const CANDLES_FOLDER_NAME = "Candles";
- const CANDLES_ONE_MIN = "One-Min";
-
- const VOLUMES_FOLDER_NAME = "Volumes";
- const VOLUMES_ONE_MIN = "One-Min";
-
- const GO_RANDOM = false;
- const FORCE_MARKET = 2; // This allows to debug the execution of an specific market. Not intended for production.
-
- const MARKETS_MODULE = require(ROOT_DIR + 'Markets');
-
- const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
- const logger = DEBUG_MODULE.newDebugLog();
- logger.fileName = MODULE_NAME;
- logger.bot = bot;
-
- interval = {
- initialize: initialize,
- start: start
- };
-
- let markets;
-
- const AZURE_FILE_STORAGE = require(ROOT_DIR + 'Azure File Storage');
- let charlyAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
- let bruceAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
-
- const UTILITIES = require(ROOT_DIR + 'Utilities');
- let utilities = UTILITIES.newUtilities(bot);
-
- let year;
- let month;
-
- return interval;
-
- function initialize(yearAssigend, monthAssigned, callBackFunction) {
-
- try {
-
- year = yearAssigend;
- month = monthAssigned;
-
- month = utilities.pad(month, 2); // Adding a left zero when needed.
-
- logger.fileName = MODULE_NAME + "-" + year + "-" + month;
-
- const logText = "[INFO] initialize - Entering function 'initialize' " + " @ " + year + "-" + month;
- console.log(logText);
- logger.write(logText);
-
- charlyAzureFileStorage.initialize("Charly");
- bruceAzureFileStorage.initialize("Bruce");
-
- markets = MARKETS_MODULE.newMarkets(bot);
- markets.initialize(callBackFunction);
-
-
- } catch (err) {
-
- const logText = "[ERROR] initialize - ' ERROR : " + err.message;
- console.log(logText);
- logger.write(logText);
-
- }
- }
-
-/*
-
-This process is going to do the following:
-
-Read the trades from Charly's Output and pack them into daily files with candles of one minute.
-
-*/
-
- function start(callBackFunction) {
-
- try {
-
- if (LOG_INFO === true) {
- logger.write("[INFO] Entering function 'start', with year = " + year + " and month = " + month);
- }
-
- let processDate = new Date(year + "-" + month + "-1 00:00:00.000 GMT+0000");
-
- let lastMinuteOfMonth = new Date(year + "-" + month + "-1 00:00:00.000 GMT+0000");
-
- lastMinuteOfMonth.setUTCMonth(lastMinuteOfMonth.getUTCMonth() + 1); // First we go 1 month into the future.
- lastMinuteOfMonth.setUTCSeconds(lastMinuteOfMonth.getUTCSeconds() - 30); // Then we go back 30 seconds, or to the last minute of the original month.
-
- let thisDatetime = new Date();
-
- if ((year === thisDatetime.getUTCFullYear() && parseInt(month) > thisDatetime.getUTCMonth() + 1) || year > thisDatetime.getUTCFullYear()) {
-
- logger.write("[INFO] We are too far in the future. Interval will not execute. Sorry.");
- return;
-
- }
-
- let atHeadOfMarket; // This tell us if we are at the month which includes the head of the market according to current datetime.
-
- if ((year === thisDatetime.getUTCFullYear() && parseInt(month) === thisDatetime.getUTCMonth() + 1)) {
-
- atHeadOfMarket = true;
-
- } else {
-
- atHeadOfMarket = false;
-
- }
-
- let nextIntervalExecution = false; // This tell weather the Interval module will be executed again or not. By default it will not unless some hole have been found in the current execution.
-
- let marketQueue; // This is the queue of all markets to be procesesd at each interval.
- let market = { // This is the current market being processed after removing it from the queue.
- id: 0,
- assetA: "",
- assetB: ""
- };
-
- let lastCandleFile; // Datetime of the last file certified by the Hole Fixing process as without permanent holes.
- let firstTradeFile; // Datetime of the first trade file in the whole market history.
- let lastFileWithoutHoles; // Datetime of the last verified file without holes.
- let lastCandleClose; // Value of the last candle close.
- let lastTradeFile; // Datetime pointing to the last Trade File sucessfuly processed and included in the last file.
-
- marketsLoop();
-
- /*
-
- At every run, the process needs to loop through all the markets at this exchange.
- The following functions marketsLoop(), openMarket(), closeMarket() and closeAndOpenMarket() controls the serialization of this processing.
-
- */
-
- function marketsLoop() {
-
- try {
-
- if (LOG_INFO === true) {
- logger.write("[INFO] Entering function 'marketsLoop'");
- }
-
- markets.getMarketsByExchange(EXCHANGE_ID, onMarketsReady);
-
- function onMarketsReady(marketsArray) {
-
- marketQueue = JSON.parse(marketsArray);
-
- openMarket(); // First execution and entering into the real loop.
-
- }
- }
- catch (err) {
- const logText = "[ERROR] 'marketsLoop' - ERROR : " + err.message;
- logger.write(logText);
- }
- }
-
- function openMarket() {
-
- // To open a Market means picking a new market from the queue.
-
- try {
-
- if (LOG_INFO === true) {
- logger.write("[INFO] Entering function 'openMarket'");
- }
-
-
- if (marketQueue.length === 0) {
-
- if (LOG_INFO === true) {
- logger.write("[INFO] 'openMarket' - marketQueue.length === 0");
- }
-
- const logText = "[WARN] We processed all the markets.";
- logger.write(logText);
-
- callBackFunction(nextIntervalExecution);
-
- return;
- }
-
- if (GO_RANDOM === true) {
- const index = parseInt(Math.random() * (marketQueue.length - 1));
-
- market.id = marketQueue[index][0];
- market.assetA = marketQueue[index][1];
- market.assetB = marketQueue[index][2];
- market.status = marketQueue[index][3];
-
- marketQueue.splice(index, 1);
- } else {
- let marketRecord = marketQueue.shift();
-
- market.id = marketRecord[0];
- market.assetA = marketRecord[1];
- market.assetB = marketRecord[2];
- market.status = marketRecord[3];
-
- if (FORCE_MARKET > 0) {
- if (FORCE_MARKET !== market.id) {
- closeAndOpenMarket();
- return;
- }
- }
- }
-
- if (LOG_INFO === true) {
- logger.write("[INFO] 'openMarket' - marketQueue.length = " + marketQueue.length);
- logger.write("[INFO] 'openMarket' - market sucessfully opened : " + market.assetA + "_" + market.assetB);
- }
-
- if (market.status === markets.ENABLED) {
-
- getStatusReport();
-
- } else {
-
- logger.write("[INFO] 'openMarket' - market " + market.assetA + "_" + market.assetB + " skipped because its status is not valid. Status = " + market.status);
- closeAndOpenMarket();
- return;
-
- }
-
-
- }
- catch (err) {
- const logText = "[ERROR] 'openMarket' - ERROR : " + err.message;
- logger.write(logText);
- closeMarket();
- }
- }
-
- function closeMarket() {
-
- if (LOG_INFO === true) {
- logger.write("[INFO] Entering function 'closeMarket'");
- }
-
- }
-
- function closeAndOpenMarket() {
-
- if (LOG_INFO === true) {
- logger.write("[INFO] Entering function 'closeAndOpenMarket'");
- }
-
- openMarket();
- }
-
- /*
-
- The following code executes for each market.
-
- */
-
- function getStatusReport() {
-
- try {
-
- let reportFilePath;
- let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json"
-
- getHistoricTrades();
-
- function getHistoricTrades() {
-
- /*
-
- We need to know where is the begining of the market, since that will help us know how to estimate the value of the last close.
- If we are at the begining of the market, the last close should be zero.
-
- */
-
- reportFilePath = EXCHANGE_NAME + "/Processes/" + "Historic-Trades";
-
- charlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
-
- function onStatusReportReceived(text) {
-
- let statusReport;
-
- try {
-
- statusReport = JSON.parse(text);
-
- firstTradeFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS);
-
- if ((year === firstTradeFile.getUTCFullYear() && parseInt(month) < firstTradeFile.getUTCMonth() + 1) || year < firstTradeFile.getUTCFullYear()) {
-
- const logText = "[INFO] 'getStatusReport' - the requested month / year are before the begining of this market " + market.assetA + '_' + market.assetB + " . Skipping it. ";
- logger.write(logText);
-
- closeAndOpenMarket();
-
- } else {
-
- getHoleFixing();
-
- }
-
- } catch (err) {
-
- const logText = "[INFO] 'getStatusReport' - Failed to read main Historic Trades Status Report for market " + market.assetA + '_' + market.assetB + " . Skipping it. ";
- logger.write(logText);
-
- closeAndOpenMarket();
- }
- }
- }
-
- function getHoleFixing() {
-
- /*
-
- The limit in the future as of which candles to include is determined by the Hole Fixing process. We wont include
- trades not certified to be without hole.
-
- */
-
- reportFilePath = EXCHANGE_NAME + "/Processes/" + "Hole-Fixing" + "/" + year + "/" + month;
-
- charlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
-
- function onStatusReportReceived(text) {
-
- let statusReport;
-
- try {
- statusReport = JSON.parse(text);
- }
- catch (err) {
- text = undefined; // If the content of the file is corrupt, this equals as if the file did not exist.
- }
-
- if (text === undefined) {
-
- const logText = "[INFO] 'getStatusReport' - The current year / month was not yet hole-fixed for market " + market.assetA + '_' + market.assetB + " . Skipping it. ";
- logger.write(logText);
-
- closeAndOpenMarket();
-
- } else {
-
- if (statusReport.monthChecked === true) {
-
- lastFileWithoutHoles = new Date(); // We need this with a valid value.
- getOneMinDailyCandlesVolumes();
-
- } else {
-
- /*
-
- If the hole report is incomplete, we are only interested if we are at the head of the market.
- Otherwise, we are not going to calculate the candles of a month which was not fully checked for holes.
-
- */
-
- if (atHeadOfMarket === true) {
-
- lastFileWithoutHoles = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS);
- getOneMinDailyCandlesVolumes();
-
- } else {
-
- const logText = "[INFO] 'getStatusReport' - The current year / month was not completely hole-fixed for market " + market.assetA + '_' + market.assetB + " . Skipping it. ";
- logger.write(logText);
-
- closeAndOpenMarket();
-
- }
- }
- }
- }
- }
-
- function getOneMinDailyCandlesVolumes() {
-
- /* If the process run and was interrupted, there should be a status report that allows us to resume execution. */
-
- reportFilePath = EXCHANGE_NAME + "/Processes/" + "One-Min-Daily-Candles-Volumes" + "/" + year + "/" + month;
-
- bruceAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
-
- function onStatusReportReceived(text) {
-
- let statusReport;
-
- try {
-
- statusReport = JSON.parse(text);
-
- if (statusReport.monthCompleted === true) {
-
- const logText = "[INFO] 'getStatusReport' - The current year / month is already complete for market " + market.assetA + '_' + market.assetB + " . Skipping it. ";
- logger.write(logText);
-
- closeAndOpenMarket();
-
- } else {
-
- lastCandleFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + "00:00" + GMT_SECONDS);
- lastCandleClose = statusReport.candleClose;
-
- if (statusReport.fileComplete === true) {
-
- buildCandles();
-
- } else {
-
- lastTradeFile = new Date(statusReport.lastTradeFile.year + "-" + statusReport.lastTradeFile.month + "-" + statusReport.lastTradeFile.days + " " + statusReport.lastTradeFile.hours + ":" + statusReport.lastTradeFile.minutes + GMT_SECONDS);
- findPreviousContent();
-
- }
- }
-
- } catch (err) {
-
- /*
-
- It might happen that the file content is corrupt or it does not exist. In either case we will point our lastCandleFile
- to the last day of the previous month.
-
- */
-
- lastCandleFile = new Date(processDate.valueOf() - ONE_DAY_IN_MILISECONDS);
- findLastCandleCloseValue();
-
- }
- }
- }
-
- }
- catch (err) {
- const logText = "[ERROR] 'getStatusReport' - ERROR : " + err.message;
- logger.write(logText);
- closeMarket();
- }
- }
-
- function findPreviousContent() {
-
- try {
-
- let previousCandles;
- let previousVolumes;
-
- getCandles();
-
- function getCandles() {
-
- let fileName = '' + market.assetA + '_' + market.assetB + '.json';
-
- let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2);
-
- let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
-
- bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
-
- function onFileReceived(text) {
-
- let candlesFile;
-
- try {
-
- candlesFile = JSON.parse(text);
-
- previousCandles = candlesFile;
-
- getVolumes();
-
- } catch (err) {
-
- const logText = "[ERR] 'findPreviousContent' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
- logger.write(logText);
-
- closeAndOpenMarket();
- }
- }
- }
-
- function getVolumes() {
-
- let fileName = '' + market.assetA + '_' + market.assetB + '.json';
-
- let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2);
-
- let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
-
- bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
-
- function onFileReceived(text) {
-
- let volumesFile;
-
- try {
-
- volumesFile = JSON.parse(text);
-
- previousVolumes = volumesFile;
-
- lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); // We know that after the next call a new day will be added.
-
- buildCandles(previousCandles, previousVolumes);
-
- } catch (err) {
-
- const logText = "[ERR] 'findPreviousContent' - Empty or corrupt volume file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
- logger.write(logText);
-
- closeAndOpenMarket();
- }
- }
- }
- }
- catch (err) {
- const logText = "[ERROR] 'findPreviousContent' - ERROR : " + err.message;
- logger.write(logText);
- closeMarket();
- }
-
- }
-
- function findLastCandleCloseValue() {
-
- try {
-
- /*
-
- We will search and find for the last trade before the begining of the current candle and that will give us the last close value.
- Before going backwards, we need to be sure we are not at the begining of the market.
-
- */
-
- if ((year === firstTradeFile.getUTCFullYear() && parseInt(month) === firstTradeFile.getUTCMonth() + 1)) {
-
- /*
-
- We are at the begining of the market, so we will set everyting to build the first candle.
-
- */
-
- const logText = "[INFO] 'findLastCandleCloseValue' - Begining of the market detected for market " + market.assetA + '_' + market.assetB + " . lastCandleClose = " + lastCandleClose;
- logger.write(logText);
- console.log(logText);
-
- lastCandleFile = new Date(firstTradeFile.getUTCFullYear() + "-" + (firstTradeFile.getUTCMonth() + 1) + "-" + firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
- lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS);
-
- lastCandleClose = 0;
-
- buildCandles();
-
- } else {
-
- /*
-
- We are not at the begining of the market, so we need scan backwards the trade files until we find a non empty one and get the last trade.
-
- */
-
- let date = new Date(processDate.valueOf());
-
- loopStart();
-
- function loopStart() {
-
- date = new Date(date.valueOf() - 60 * 1000);
-
- let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2);
- let fileName = market.assetA + '_' + market.assetB + ".json"
- let filePath = EXCHANGE_NAME + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath;
-
- charlyAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
-
- function onFileReceived(text) {
-
- let tradesFile;
-
- try {
-
- tradesFile = JSON.parse(text);
-
- if (tradesFile.length > 0) {
-
- lastCandleClose = tradesFile[tradesFile.length - 1][2]; // Position 2 is the rate at which the trade was executed.
-
- const logText = "[INFO] 'findLastCandleCloseValue' - Trades found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . lastCandleClose = " + lastCandleClose;
- logger.write(logText);
- console.log(logText);
-
- buildCandles();
-
- } else {
-
- const logText = "[INFO] 'findLastCandleCloseValue' - No trades found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " .";
- logger.write(logText);
- console.log(logText);
-
- loopStart();
-
- }
-
- } catch (err) {
-
- const logText = "[ERR] 'findLastCandleCloseValue' - Empty or corrupt trade file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
- logger.write(logText);
-
- closeAndOpenMarket();
- }
- }
- }
- }
- }
- catch (err) {
- const logText = "[ERROR] 'findLastCandleCloseValue' - ERROR : " + err.message;
- logger.write(logText);
- closeMarket();
- }
- }
-
- function buildCandles(previousCandles, previousVolumes) {
-
- /*
-
- Here we are going to scan the trades files packing them in candles files every one day.
- We need for this the last close value, bacause all candles that are empty of trades at the begining, they need to
- have a valid open and close value. This was previously calculated before arriving to this function.
-
- */
-
- let canAddPrevious = true;
-
- try {
-
-
- nextCandleFile();
-
- function nextCandleFile() {
-
- lastCandleFile = new Date(lastCandleFile.valueOf() + ONE_DAY_IN_MILISECONDS);
-
- let date = new Date(lastCandleFile.valueOf() - 60 * 1000);
-
- if (date.valueOf() < firstTradeFile.valueOf()) { // At the special case where we are at the begining of the market, this might be true.
-
- date = new Date(firstTradeFile.valueOf() - 60 * 1000);
-
- }
-
- if (lastTradeFile !== undefined) {
-
- date = new Date(lastTradeFile.valueOf());
-
- }
-
- let candles = [];
- let volumes = [];
-
- if (previousCandles !== undefined && canAddPrevious === true) {
-
- for (let i = 0; i < previousCandles.length; i++) {
-
- let candle = {
- open: previousCandles[i][2],
- close: previousCandles[i][3],
- min: previousCandles[i][0],
- max: previousCandles[i][1],
- begin: previousCandles[i][4],
- end: previousCandles[i][5]
- };
-
- candles.push(candle);
- }
-
- }
-
- if (previousVolumes !== undefined && canAddPrevious === true) {
-
- for (let i = 0; i < previousVolumes.length; i++) {
-
- let volume = {
- begin: previousVolumes[i][2],
- end: previousVolumes[i][3],
- buy: previousVolumes[i][0],
- sell: previousVolumes[i][1]
- };
-
- volumes.push(volume);
- }
-
- }
-
- canAddPrevious = false; // We add them only onece.
-
- nextDate();
-
- function nextDate() {
-
- date = new Date(date.valueOf() + 60 * 1000);
-
- /* Check if we are outside the current Day / File */
-
- if (date.getUTCDate() !== lastCandleFile.getUTCDate()) {
-
- writeFiles(lastCandleFile, candles, volumes, true, onFilesWritten);
-
- return;
-
- function onFilesWritten() {
-
- nextCandleFile();
-
- }
-
- }
-
- /* Check if we are outside the currrent Month */
-
- if (date.getUTCMonth() + 1 !== parseInt(month)) {
-
- lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS);
-
- writeStatusReport(lastCandleFile, lastTradeFile, lastCandleClose, true, true, onStatusReportWritten);
-
- return;
-
- function onStatusReportWritten() {
-
- const logText = "[ERR] 'buildCandles' - Finishing processing the whole month for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
- logger.write(logText);
-
- closeAndOpenMarket();
-
- }
- }
-
- /* Check if we have past the most recent hole fixed file */
-
- if (date.valueOf() > lastFileWithoutHoles.valueOf()) {
-
- writeFiles(lastCandleFile, candles, volumes, false, onFilesWritten);
-
- return;
-
- function onFilesWritten() {
-
- nextIntervalExecution = true;
-
- const logText = "[ERR] 'buildCandles' - Head of the market reached for market " + market.assetA + '_' + market.assetB + " . ";
- logger.write(logText);
-
- closeAndOpenMarket();
-
- }
- }
-
- readTrades();
- }
-
- function readTrades() {
-
- lastTradeFile = new Date(date.valueOf());
-
- let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2);
- let fileName = market.assetA + '_' + market.assetB + ".json"
- let filePath = EXCHANGE_NAME + "/Output/" + TRADES_FOLDER_NAME + '/' + dateForPath;
-
- charlyAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
-
- function onFileReceived(text) {
-
- let tradesFile;
-
- try {
-
- let candle = {
- open: lastCandleClose,
- close: lastCandleClose,
- min: lastCandleClose,
- max: lastCandleClose,
- begin: date.valueOf(),
- end: date.valueOf() + 60 * 1000 - 1
- };
-
- let volume = {
- begin: date.valueOf(),
- end: date.valueOf() + 60 * 1000 - 1,
- buy: 0,
- sell: 0
- };
-
- tradesFile = JSON.parse(text);
-
- let tradesCount = utilities.pad(tradesFile.length, 5);
-
- const logText = "[INFO] 'buildCandles' - " + tradesCount +" trades found at " + filePath + " for market " + market.assetA + '_' + market.assetB + ". ";
- logger.write(logText);
- console.log(logText);
-
- if (tradesFile.length > 0) {
-
- /* Candle open and close Calculations */
-
- candle.open = tradesFile[0][2];
- candle.close = tradesFile[tradesFile.length - 1][2];
-
- lastCandleClose = candle.close;
-
- }
-
- for (let i = 0; i < tradesFile.length; i++) {
-
- const trade = {
- id: tradesFile[i][0],
- type: tradesFile[i][1],
- rate: tradesFile[i][2],
- amountA: tradesFile[i][3],
- amountB: tradesFile[i][4],
- seconds: tradesFile[i][5]
- };
-
- /* Candle min and max Calculations */
-
- if (trade.rate < candle.min) {
-
- candle.min = trade.rate;
-
- }
-
- if (trade.rate > candle.max) {
-
- candle.max = trade.rate;
-
- }
-
- /* Volume Calculations */
-
- if (trade.type === "sell") {
- volume.sell = volume.sell + trade.amountA;
- } else {
- volume.buy = volume.buy + trade.amountA;
- }
-
- }
-
- candles.push(candle);
-
- volumes.push(volume);
-
- nextDate();
-
- } catch (err) {
-
- const logText = "[ERR] 'buildCandles' - Empty or corrupt trade file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
- logger.write(logText);
-
- closeAndOpenMarket();
- }
- }
- }
- }
- }
-
- catch (err) {
- const logText = "[ERROR] 'buildCandles' - ERROR : " + err.message;
- logger.write(logText);
- closeMarket();
- }
-
- }
-
- function writeFiles(date, candles, volumes, isFileComplete, callBack) {
-
- /*
-
- Here we will write the contents of the Candles and Volumens files. If the File is declared as complete, we will also write the status report.
-
- */
-
- try {
-
- writeCandles();
-
- function writeCandles() {
-
- let separator = "";
- let fileRecordCounter = 0;
-
- let fileContent = "";
-
- for (i = 0; i < candles.length; i++) {
-
- let candle = candles[i];
-
- fileContent = fileContent + separator + '[' + candles[i].min + "," + candles[i].max + "," + candles[i].open + "," + candles[i].close + "," + candles[i].begin + "," + candles[i].end + "]";
-
- if (separator === "") { separator = ","; }
-
- fileRecordCounter++;
-
- }
-
- fileContent = "[" + fileContent + "]";
-
- let fileName = '' + market.assetA + '_' + market.assetB + '.json';
-
- let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2);
-
- let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
-
- utilities.createFolderIfNeeded(filePath, bruceAzureFileStorage, onFolderCreated);
-
- function onFolderCreated() {
-
- bruceAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated);
-
- function onFileCreated() {
-
- const logText = "[WARN] Finished with File @ " + market.assetA + "_" + market.assetB + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName + "";
- console.log(logText);
- logger.write(logText);
-
- writeVolumes();
- }
- }
-
- }
-
- function writeVolumes() {
-
- let separator = "";
- let fileRecordCounter = 0;
-
- let fileContent = "";
-
- for (i = 0; i < volumes.length; i++) {
-
- let candle = volumes[i];
-
- fileContent = fileContent + separator + '[' + volumes[i].buy + "," + volumes[i].sell + "," + volumes[i].begin + "," + volumes[i].end + "]";
-
- if (separator === "") { separator = ","; }
-
- fileRecordCounter++;
-
- }
-
- fileContent = "[" + fileContent + "]";
-
- let fileName = '' + market.assetA + '_' + market.assetB + '.json';
-
- let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2);
-
- let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath;
-
- utilities.createFolderIfNeeded(filePath, bruceAzureFileStorage, onFolderCreated);
-
- function onFolderCreated() {
-
- bruceAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated);
-
- function onFileCreated() {
-
- const logText = "[WARN] Finished with File @ " + market.assetA + "_" + market.assetB + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName + "";
- console.log(logText);
- logger.write(logText);
-
- writeReport();
- }
- }
- }
-
- function writeReport() {
-
- writeStatusReport(date, lastTradeFile, lastCandleClose, isFileComplete, false, onStatusReportWritten);
-
- function onStatusReportWritten() {
-
- callBack();
-
- }
- }
- }
-
- catch (err) {
- const logText = "[ERROR] 'writeFiles' - ERROR : " + err.message;
- logger.write(logText);
- closeMarket();
- }
- }
-
- function writeStatusReport(lastFileDate, lastTradeFile, candleClose, isFileComplete, isMonthComplete, callBack) {
-
-
- if (LOG_INFO === true) {
- logger.write("[INFO] Entering function 'writeStatusReport'");
- }
-
- try {
-
- let reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process + "/" + year + "/" + month;
-
- utilities.createFolderIfNeeded(reportFilePath, bruceAzureFileStorage, onFolderCreated);
-
- function onFolderCreated() {
-
- try {
-
- let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json";
-
- let report = {
- lastFile: {
- year: lastFileDate.getUTCFullYear(),
- month: (lastFileDate.getUTCMonth() + 1),
- days: lastFileDate.getUTCDate()
- },
- lastTradeFile: {
- year: lastTradeFile.getUTCFullYear(),
- month: (lastTradeFile.getUTCMonth() + 1),
- days: lastTradeFile.getUTCDate(),
- hours: lastTradeFile.getUTCHours(),
- minutes: lastTradeFile.getUTCMinutes()
- },
- candleClose: candleClose,
- monthCompleted: isMonthComplete,
- fileComplete: isFileComplete
- };
-
- let fileContent = JSON.stringify(report);
-
- bruceAzureFileStorage.createTextFile(reportFilePath, fileName, fileContent + '\n', onFileCreated);
-
- function onFileCreated() {
-
- if (LOG_INFO === true) {
- logger.write("[INFO] 'writeStatusReport' - Content written: " + fileContent);
- }
-
- callBack();
- }
- }
- catch (err) {
- const logText = "[ERROR] 'writeStatusReport - onFolderCreated' - ERROR : " + err.message;
- logger.write(logText);
- closeMarket();
- }
- }
-
- }
- catch (err) {
- const logText = "[ERROR] 'writeStatusReport' - ERROR : " + err.message;
- logger.write(logText);
- closeMarket();
- }
-
- }
-
- }
- catch (err) {
- const logText = "[ERROR] 'Start' - ERROR : " + err.message;
- logger.write(logText);
- }
- }
-};
diff --git a/Multi-Period-Daily-Candles-Volumes/run.js b/Multi-Period-Daily-Candles-Volumes/run.js
deleted file mode 100644
index 43917e38e..000000000
--- a/Multi-Period-Daily-Candles-Volumes/run.js
+++ /dev/null
@@ -1,97 +0,0 @@
-
-let bot = {
- name: "Bruce",
- process: "Multi-Period-Daily-Candles-Volumes",
- type: "Indicator",
- version: "1.0.0",
- devTeam: "AA Masters"
-};
-
-const ROOT_DIR = '../';
-
-const MODULE_NAME = "run";
-
-const START_ALL_MONTHS = false;
-const START_ONE_MONTH = true;
-
-const INTERVAL_EXECUTOR_MODULE = require('./Interval Executor');
-
-const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
-const logger = DEBUG_MODULE.newDebugLog();
-logger.fileName = MODULE_NAME;
-logger.bot = bot;
-
-process.on('uncaughtException', function (err) {
- logger.write('uncaughtException - ' + err.message);
-});
-
-
-process.on('unhandledRejection', (reason, p) => {
- logger.write("Unhandled Rejection at: Promise " + JSON.stringify(p) + " reason: " + reason);
-});
-
-
-process.on('exit', function (code) {
- logger.write('About to exit with code:' + code);
-});
-
-
-try {
- if (START_ALL_MONTHS === true) {
-
- for (let year = 2018; year > 2014; year--) {
-
- for (let month = 12; month > 0; month--) {
-
- let timeDelay = Math.random() * 10 * 1000; // We introduce a short delay so as to not overload the machine.
-
- setTimeout(startProcess, timeDelay);
-
- function startProcess() {
-
- const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot);
-
- newIntervalExecutor.initialize(year, month, onInitializeReady);
-
- function onInitializeReady() {
-
- newIntervalExecutor.start();
-
- }
-
- }
-
- }
-
- }
-
- }
-
-
- if (START_ONE_MONTH === true) {
-
- startProcess();
-
- function startProcess() {
-
- const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot);
-
- let month = 2;
- let year = 2018;
-
- newIntervalExecutor.initialize(year, month, onInitializeReady);
-
- function onInitializeReady() {
-
- newIntervalExecutor.start();
-
- }
- }
- }
-
-
-} catch (err) {
- console.log(err.message);
- logger.write(err.message);
-}
-