Upgrade to run with AACloudPlatform
This commit is contained in:
Luis Fernando Molina committed
1 parent
2952b1c92b
commit
67a14defdb
15 files changed
+38
-2612
No files matched your search
Binary file not shown.
@@ -1,169 +0,0 @@
|
||||
exports.newIntervalExecutor = function newIntervalExecutor(BOT) {
|
||||
|
||||
let bot = BOT;
|
||||
const ROOT_DIR = '../';
|
||||
|
||||
const INTERVAL_LAPSE = 60 * 60 * 1000;
|
||||
|
||||
const MODULE_NAME = "Interval Executor";
|
||||
const LOG_INFO = true;
|
||||
|
||||
const INTERVAL_MODULE = require('./Interval');
|
||||
|
||||
const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
|
||||
const INTERVAL_MAX_LOOPS = -1; // -1 for infinite loop.
|
||||
|
||||
poloniexHoleFixing = {
|
||||
initialize: initialize,
|
||||
start: start
|
||||
};
|
||||
|
||||
const SHUTDOWN_EVENT = require(ROOT_DIR + 'Azure Web Job Shutdown');
|
||||
let shutdownEvent = SHUTDOWN_EVENT.newShutdownEvent(bot);
|
||||
|
||||
const UTILITIES = require(ROOT_DIR + 'Utilities');
|
||||
let utilities = UTILITIES.newUtilities(bot);
|
||||
|
||||
let year;
|
||||
let month;
|
||||
|
||||
return poloniexHoleFixing;
|
||||
|
||||
function initialize(yearAssigend, monthAssigned, callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
year = yearAssigend;
|
||||
month = monthAssigned;
|
||||
|
||||
month = utilities.pad(month, 2); // Adding a left zero when needed.
|
||||
|
||||
logger.fileName = MODULE_NAME + "-" + year + "-" + month;
|
||||
|
||||
const logText = "[INFO] initialize - Entering function 'initialize' " + " @ " + year + "-" + month;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
callBackFunction();
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] initialize - ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function start() {
|
||||
|
||||
try {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] Entering function 'start', with year = " + year + " and month = " + month);
|
||||
}
|
||||
|
||||
let intervalId;
|
||||
|
||||
if (INTERVAL_MAX_LOOPS === 0) {
|
||||
|
||||
intervalId = setInterval(startNewInterval, INTERVAL_LAPSE);
|
||||
|
||||
}
|
||||
|
||||
let intervalLoopCounter = 0;
|
||||
|
||||
startNewInterval(); // First, inmadiate run.
|
||||
|
||||
function startNewInterval() {
|
||||
|
||||
try {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
|
||||
if (INTERVAL_MAX_LOOPS !== 0) {
|
||||
|
||||
logger.write("[INFO] Entering function 'startNewInterval', next execution inmediately after the current Interval finishes.");
|
||||
|
||||
} else {
|
||||
|
||||
logger.write("[INFO] Entering function 'startNewInterval', next execution in " + INTERVAL_LAPSE / 1000 / 60 + " minutes.");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
if (shutdownEvent.isShuttingDown() === false) {
|
||||
|
||||
let fileProcessingInterval = INTERVAL_MODULE.newInterval(bot);
|
||||
|
||||
fileProcessingInterval.initialize(year, month, onReadyToStart);
|
||||
|
||||
function onReadyToStart() {
|
||||
|
||||
fileProcessingInterval.start(loopControl);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
|
||||
clearInterval(intervalId);
|
||||
|
||||
const logText = "[INFO] requestNewData - Terminating Set Interval - About to exit gracefully the process. ";
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'startNewInterval' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
|
||||
function loopControl(moreIntervalExecutionsNeeded) {
|
||||
|
||||
if (moreIntervalExecutionsNeeded === true) {
|
||||
|
||||
intervalLoopCounter++;
|
||||
|
||||
if (intervalLoopCounter === INTERVAL_MAX_LOOPS) {
|
||||
|
||||
const logText = "[WARN] 'INTERVAL_MAX_LOOPS' reached. Stopping execution. ";
|
||||
logger.write(logText);
|
||||
|
||||
} else {
|
||||
|
||||
const logText = "[WARN] 'loopControl'. Staring new Interval. ";
|
||||
logger.write(logText);
|
||||
|
||||
setTimeout(startNewInterval, INTERVAL_LAPSE);// Looped Execution.
|
||||
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
logger.write("[WARN] Received the signal from last Interval execution to not call it anymore. Stopping execution. ");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'Start' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
@@ -1,895 +0,0 @@
|
||||
exports.newInterval = function newInterval(BOT) {
|
||||
|
||||
let bot = BOT;
|
||||
const ROOT_DIR = '../';
|
||||
const GMT_SECONDS = ':00.000 GMT+0000';
|
||||
const GMT_MILI_SECONDS = '.000 GMT+0000';
|
||||
const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000;
|
||||
|
||||
const MODULE_NAME = "Interval";
|
||||
const LOG_INFO = true;
|
||||
|
||||
const EXCHANGE_NAME = "Poloniex";
|
||||
const EXCHANGE_ID = 1;
|
||||
|
||||
const TRADES_FOLDER_NAME = "Trades";
|
||||
|
||||
const CANDLES_FOLDER_NAME = "Candles";
|
||||
const CANDLES_ONE_MIN = "One-Min";
|
||||
|
||||
const VOLUMES_FOLDER_NAME = "Volumes";
|
||||
const VOLUMES_ONE_MIN = "One-Min";
|
||||
|
||||
const GO_RANDOM = false;
|
||||
const FORCE_MARKET = 2; // This allows to debug the execution of an specific market. Not intended for production.
|
||||
|
||||
const MARKETS_MODULE = require(ROOT_DIR + 'Markets');
|
||||
|
||||
const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
|
||||
interval = {
|
||||
initialize: initialize,
|
||||
start: start
|
||||
};
|
||||
|
||||
let markets;
|
||||
|
||||
const AZURE_FILE_STORAGE = require(ROOT_DIR + 'Azure File Storage');
|
||||
let charlyAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
|
||||
let bruceAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
|
||||
let oliviaAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
|
||||
|
||||
const UTILITIES = require(ROOT_DIR + 'Utilities');
|
||||
let utilities = UTILITIES.newUtilities(bot);
|
||||
|
||||
let year;
|
||||
let month;
|
||||
|
||||
return interval;
|
||||
|
||||
function initialize(yearAssigend, monthAssigned, callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
year = yearAssigend;
|
||||
month = monthAssigned;
|
||||
|
||||
month = utilities.pad(month, 2); // Adding a left zero when needed.
|
||||
|
||||
logger.fileName = MODULE_NAME + "-" + year + "-" + month;
|
||||
|
||||
const logText = "[INFO] initialize - Entering function 'initialize' " + " @ " + year + "-" + month;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
charlyAzureFileStorage.initialize("Charly");
|
||||
bruceAzureFileStorage.initialize("Bruce");
|
||||
oliviaAzureFileStorage.initialize("Olivia");
|
||||
|
||||
markets = MARKETS_MODULE.newMarkets(bot);
|
||||
markets.initialize(callBackFunction);
|
||||
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] initialize - ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
This process is going to do the following:
|
||||
|
||||
Read the candles and volumes from Bruce and produce a single Index File for each market with daily candles and volumes.
|
||||
|
||||
*/
|
||||
|
||||
function start(callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] Entering function 'start', with year = " + year + " and month = " + month);
|
||||
}
|
||||
|
||||
let processDate = new Date(year + "-" + month + "-1 00:00:00.000 GMT+0000");
|
||||
|
||||
let lastMinuteOfMonth = new Date(year + "-" + month + "-1 00:00:00.000 GMT+0000");
|
||||
|
||||
lastMinuteOfMonth.setUTCMonth(lastMinuteOfMonth.getUTCMonth() + 1); // First we go 1 month into the future.
|
||||
lastMinuteOfMonth.setUTCSeconds(lastMinuteOfMonth.getUTCSeconds() - 30); // Then we go back 30 seconds, or to the last minute of the original month.
|
||||
|
||||
let thisDatetime = new Date();
|
||||
|
||||
if ((year === thisDatetime.getUTCFullYear() && parseInt(month) > thisDatetime.getUTCMonth() + 1) || year > thisDatetime.getUTCFullYear()) {
|
||||
|
||||
logger.write("[INFO] We are too far in the future. Interval will not execute. Sorry.");
|
||||
return;
|
||||
|
||||
}
|
||||
|
||||
let atHeadOfMarket; // This tell us if we are at the month which includes the head of the market according to current datetime.
|
||||
|
||||
if ((year === thisDatetime.getUTCFullYear() && parseInt(month) === thisDatetime.getUTCMonth() + 1)) {
|
||||
|
||||
atHeadOfMarket = true;
|
||||
|
||||
} else {
|
||||
|
||||
atHeadOfMarket = false;
|
||||
|
||||
}
|
||||
|
||||
let nextIntervalExecution = false; // This tell weather the Interval module will be executed again or not. By default it will not unless some hole have been found in the current execution.
|
||||
|
||||
let marketQueue; // This is the queue of all markets to be procesesd at each interval.
|
||||
let market = { // This is the current market being processed after removing it from the queue.
|
||||
id: 0,
|
||||
assetA: "",
|
||||
assetB: ""
|
||||
};
|
||||
|
||||
let lastCandleFile; // Datetime of the last file included on the Index File.
|
||||
let firstTradeFile; // Datetime of the first trade file in the whole market history.
|
||||
let maxCandleFile; // Datetime of the last file available to be included in the Index File.
|
||||
|
||||
marketsLoop();
|
||||
|
||||
/*
|
||||
|
||||
At every run, the process needs to loop through all the markets at this exchange.
|
||||
The following functions marketsLoop(), openMarket(), closeMarket() and closeAndOpenMarket() controls the serialization of this processing.
|
||||
|
||||
*/
|
||||
|
||||
function marketsLoop() {
|
||||
|
||||
try {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] Entering function 'marketsLoop'");
|
||||
}
|
||||
|
||||
markets.getMarketsByExchange(EXCHANGE_ID, onMarketsReady);
|
||||
|
||||
function onMarketsReady(marketsArray) {
|
||||
|
||||
marketQueue = JSON.parse(marketsArray);
|
||||
|
||||
openMarket(); // First execution and entering into the real loop.
|
||||
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'marketsLoop' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
|
||||
function openMarket() {
|
||||
|
||||
// To open a Market means picking a new market from the queue.
|
||||
|
||||
try {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] Entering function 'openMarket'");
|
||||
}
|
||||
|
||||
|
||||
if (marketQueue.length === 0) {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] 'openMarket' - marketQueue.length === 0");
|
||||
}
|
||||
|
||||
const logText = "[WARN] We processed all the markets.";
|
||||
logger.write(logText);
|
||||
|
||||
callBackFunction(nextIntervalExecution);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (GO_RANDOM === true) {
|
||||
const index = parseInt(Math.random() * (marketQueue.length - 1));
|
||||
|
||||
market.id = marketQueue[index][0];
|
||||
market.assetA = marketQueue[index][1];
|
||||
market.assetB = marketQueue[index][2];
|
||||
market.status = marketQueue[index][3];
|
||||
|
||||
marketQueue.splice(index, 1);
|
||||
} else {
|
||||
let marketRecord = marketQueue.shift();
|
||||
|
||||
market.id = marketRecord[0];
|
||||
market.assetA = marketRecord[1];
|
||||
market.assetB = marketRecord[2];
|
||||
market.status = marketRecord[3];
|
||||
|
||||
if (FORCE_MARKET > 0) {
|
||||
if (FORCE_MARKET !== market.id) {
|
||||
closeAndOpenMarket();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] 'openMarket' - marketQueue.length = " + marketQueue.length);
|
||||
logger.write("[INFO] 'openMarket' - market sucessfully opened : " + market.assetA + "_" + market.assetB);
|
||||
}
|
||||
|
||||
if (market.status === markets.ENABLED) {
|
||||
|
||||
getStatusReport();
|
||||
|
||||
} else {
|
||||
|
||||
logger.write("[INFO] 'openMarket' - market " + market.assetA + "_" + market.assetB + " skipped because its status is not valid. Status = " + market.status);
|
||||
closeAndOpenMarket();
|
||||
return;
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'openMarket' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
closeMarket();
|
||||
}
|
||||
}
|
||||
|
||||
function closeMarket() {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] Entering function 'closeMarket'");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function closeAndOpenMarket() {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] Entering function 'closeAndOpenMarket'");
|
||||
}
|
||||
|
||||
openMarket();
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
The following code executes for each market.
|
||||
|
||||
*/
|
||||
|
||||
function getStatusReport() {
|
||||
|
||||
try {
|
||||
|
||||
let reportFilePath;
|
||||
let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json"
|
||||
|
||||
getHistoricTrades();
|
||||
|
||||
function getHistoricTrades() {
|
||||
|
||||
/*
|
||||
|
||||
We need to know where is the begining of the market, since that will help us know where the Index Files should start.
|
||||
|
||||
*/
|
||||
|
||||
reportFilePath = EXCHANGE_NAME + "/Processes/" + "Historic-Trades";
|
||||
|
||||
charlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
|
||||
|
||||
function onStatusReportReceived(text) {
|
||||
|
||||
let statusReport;
|
||||
|
||||
try {
|
||||
|
||||
statusReport = JSON.parse(text);
|
||||
|
||||
firstTradeFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS);
|
||||
|
||||
getOneMinDailyCandlesVolumes();
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[INFO] 'getStatusReport' - Failed to read main Historic Trades Status Report for market " + market.assetA + '_' + market.assetB + " . Skipping it. ";
|
||||
logger.write(logText);
|
||||
|
||||
closeAndOpenMarket();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getOneMinDailyCandlesVolumes() {
|
||||
|
||||
/* If the process run and was interrupted, there should be a status report that allows us to resume execution. */
|
||||
|
||||
reportFilePath = EXCHANGE_NAME + "/Processes/" + "One-Min-Daily-Candles-Volumes" + "/" + year + "/" + month;
|
||||
|
||||
bruceAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
|
||||
|
||||
function onStatusReportReceived(text) {
|
||||
|
||||
let statusReport;
|
||||
|
||||
try {
|
||||
|
||||
statusReport = JSON.parse(text);
|
||||
|
||||
maxCandleFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + "00:00" + GMT_SECONDS);
|
||||
|
||||
getThisProcessReport();
|
||||
|
||||
} catch (err) {
|
||||
|
||||
/*
|
||||
|
||||
It might happen that the file content is corrupt or it does not exist. In either case we will point our lastCandleFile
|
||||
to the last day of the previous month.
|
||||
|
||||
*/
|
||||
|
||||
lastCandleFile = new Date(processDate.valueOf() - ONE_DAY_IN_MILISECONDS);
|
||||
findLastCandleCloseValue();
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getThisProcessReport() {
|
||||
|
||||
/* If the process run and was interrupted, there should be a status report that allows us to resume execution. */
|
||||
|
||||
reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process;
|
||||
|
||||
oliviaAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
|
||||
|
||||
function onStatusReportReceived(text) {
|
||||
|
||||
let statusReport;
|
||||
|
||||
try {
|
||||
|
||||
statusReport = JSON.parse(text);
|
||||
|
||||
lastCandleFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + "00:00" + GMT_SECONDS);
|
||||
|
||||
findPreviousContent();
|
||||
|
||||
} catch (err) {
|
||||
|
||||
/*
|
||||
|
||||
It might happen that the file content is corrupt or it does not exist. In either case we will point our lastCandleFile
|
||||
to the begining of the market.
|
||||
|
||||
*/
|
||||
|
||||
lastCandleFile = new Date(firstTradeFile.getUTCFullYear() + "-" + (firstTradeFile.getUTCMonth() + 1) + "-" + firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS);
|
||||
|
||||
lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well.
|
||||
|
||||
buildCandles();
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'getStatusReport' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
closeMarket();
|
||||
}
|
||||
}
|
||||
|
||||
function findPreviousContent() {
|
||||
|
||||
try {
|
||||
|
||||
let previousCandles;
|
||||
let previousVolumes;
|
||||
|
||||
getCandles();
|
||||
|
||||
function getCandles() {
|
||||
|
||||
let fileName = '' + market.assetA + '_' + market.assetB + '.json';
|
||||
|
||||
let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME;
|
||||
|
||||
oliviaAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
|
||||
|
||||
function onFileReceived(text) {
|
||||
|
||||
let candlesFile;
|
||||
|
||||
try {
|
||||
|
||||
candlesFile = JSON.parse(text);
|
||||
|
||||
previousCandles = candlesFile;
|
||||
|
||||
getVolumes();
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERR] 'findPreviousContent' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
|
||||
logger.write(logText);
|
||||
|
||||
closeAndOpenMarket();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getVolumes() {
|
||||
|
||||
let fileName = '' + market.assetA + '_' + market.assetB + '.json';
|
||||
|
||||
let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME;
|
||||
|
||||
oliviaAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
|
||||
|
||||
function onFileReceived(text) {
|
||||
|
||||
let volumesFile;
|
||||
|
||||
try {
|
||||
|
||||
volumesFile = JSON.parse(text);
|
||||
|
||||
previousVolumes = volumesFile;
|
||||
|
||||
buildCandles(previousCandles, previousVolumes);
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERR] 'findPreviousContent' - Empty or corrupt volume file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
|
||||
logger.write(logText);
|
||||
|
||||
closeAndOpenMarket();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'findPreviousContent' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
closeMarket();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function buildCandles(previousCandles, previousVolumes) {
|
||||
|
||||
/*
|
||||
|
||||
Here we are going to scan the candle and volume files to generate daily candles in only one Index File.
|
||||
|
||||
*/
|
||||
|
||||
let outputCandles = [];
|
||||
let outputVolumes = [];
|
||||
|
||||
try {
|
||||
|
||||
if (previousCandles !== undefined) {
|
||||
|
||||
for (let i = 0; i < previousCandles.length; i++) {
|
||||
|
||||
let candle = {
|
||||
open: previousCandles[i][2],
|
||||
close: previousCandles[i][3],
|
||||
min: previousCandles[i][0],
|
||||
max: previousCandles[i][1],
|
||||
begin: previousCandles[i][4],
|
||||
end: previousCandles[i][5]
|
||||
};
|
||||
|
||||
outputCandles.push(candle);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (previousVolumes !== undefined) {
|
||||
|
||||
for (let i = 0; i < previousVolumes.length; i++) {
|
||||
|
||||
let volume = {
|
||||
begin: previousVolumes[i][2],
|
||||
end: previousVolumes[i][3],
|
||||
buy: previousVolumes[i][0],
|
||||
sell: previousVolumes[i][1]
|
||||
};
|
||||
|
||||
outputVolumes.push(volume);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
advanceTime();
|
||||
|
||||
function advanceTime() {
|
||||
|
||||
lastCandleFile = new Date(lastCandleFile.valueOf() + ONE_DAY_IN_MILISECONDS);
|
||||
|
||||
const logText = "[INFO] New processing time @ " + lastCandleFile.getUTCFullYear() + "/" + (lastCandleFile.getUTCMonth() + 1) + "/" + lastCandleFile.getUTCDate() + ".";
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
/* Validation that we are not going past the head of the market. */
|
||||
|
||||
if (lastCandleFile.valueOf() > maxCandleFile.valueOf()) {
|
||||
|
||||
const logText = "[INFO] 'buildCandles' - Head of the market found @ " + lastCandleFile.getUTCFullYear() + "/" + (lastCandleFile.getUTCMonth() + 1) + "/" + lastCandleFile.getUTCDate() + ".";
|
||||
logger.write(logText);
|
||||
|
||||
closeAndOpenMarket();
|
||||
|
||||
return;
|
||||
|
||||
}
|
||||
|
||||
nextCandleFile();
|
||||
|
||||
}
|
||||
|
||||
function nextCandleFile() {
|
||||
|
||||
let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2);
|
||||
let fileName = market.assetA + '_' + market.assetB + ".json"
|
||||
let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath;
|
||||
|
||||
bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
|
||||
|
||||
function onFileReceived(text) {
|
||||
|
||||
let candlesFile;
|
||||
|
||||
try {
|
||||
|
||||
candlesFile = JSON.parse(text);
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERR] 'buildCandles' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
|
||||
logger.write(logText);
|
||||
|
||||
closeAndOpenMarket();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
let outputCandle = {
|
||||
open: 0,
|
||||
close: 0,
|
||||
min: 0,
|
||||
max: 0,
|
||||
begin: 0,
|
||||
end: 0
|
||||
};
|
||||
|
||||
if (candlesFile.length > 0) {
|
||||
|
||||
outputCandle.open = candlesFile[0][2];
|
||||
outputCandle.close = candlesFile[candlesFile.length - 1][3];
|
||||
|
||||
outputCandle.min = candlesFile[0][0];
|
||||
outputCandle.max = candlesFile[0][1];
|
||||
|
||||
outputCandle.begin = candlesFile[0][4];
|
||||
outputCandle.end = candlesFile[candlesFile.length - 1][5];
|
||||
|
||||
} else {
|
||||
|
||||
const logText = "[ERR] 'buildCandles' - No candles found at file " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
|
||||
logger.write(logText);
|
||||
|
||||
closeAndOpenMarket();
|
||||
|
||||
return;
|
||||
|
||||
}
|
||||
|
||||
for (let i = 0; i < candlesFile.length; i++) {
|
||||
|
||||
let candle = {
|
||||
open: candlesFile[i][2],
|
||||
close: candlesFile[i][3],
|
||||
min: candlesFile[i][0],
|
||||
max: candlesFile[i][1],
|
||||
begin: candlesFile[i][4],
|
||||
end: candlesFile[i][5]
|
||||
};
|
||||
|
||||
if (candle.min < outputCandle.min) {
|
||||
|
||||
outputCandle.min = candle.min;
|
||||
|
||||
}
|
||||
|
||||
if (candle.max > outputCandle.max) {
|
||||
|
||||
outputCandle.max = candle.max;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
outputCandles.push(outputCandle);
|
||||
|
||||
nextVolumeFile();
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
function nextVolumeFile() {
|
||||
|
||||
let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2);
|
||||
let fileName = market.assetA + '_' + market.assetB + ".json"
|
||||
let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath;
|
||||
|
||||
bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
|
||||
|
||||
function onFileReceived(text) {
|
||||
|
||||
let volumesFile;
|
||||
|
||||
try {
|
||||
|
||||
volumesFile = JSON.parse(text);
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERR] 'buildCandles' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
|
||||
logger.write(logText);
|
||||
|
||||
closeAndOpenMarket();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
let outputVolume = {
|
||||
buy: 0,
|
||||
sell: 0,
|
||||
begin: 0,
|
||||
end: 0
|
||||
};
|
||||
|
||||
if (volumesFile.length > 0) {
|
||||
|
||||
outputVolume.begin = volumesFile[0][2];
|
||||
outputVolume.end = volumesFile[volumesFile.length - 1][3];
|
||||
|
||||
} else {
|
||||
|
||||
const logText = "[ERR] 'buildCandles' - No volumes found at file " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. ";
|
||||
logger.write(logText);
|
||||
|
||||
closeAndOpenMarket();
|
||||
|
||||
return;
|
||||
|
||||
}
|
||||
|
||||
for (let i = 0; i < volumesFile.length; i++) {
|
||||
|
||||
let volume = {
|
||||
buy: volumesFile[i][0],
|
||||
sell: volumesFile[i][1],
|
||||
begin: volumesFile[i][2],
|
||||
end: volumesFile[i][3]
|
||||
};
|
||||
|
||||
outputVolume.buy = outputVolume.buy + volume.buy;
|
||||
outputVolume.sell = outputVolume.sell + volume.sell;
|
||||
|
||||
}
|
||||
|
||||
outputVolumes.push(outputVolume);
|
||||
|
||||
writeFiles(lastCandleFile, outputCandles, outputVolumes, advanceTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'buildCandles' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
closeMarket();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function writeFiles(date, candles, volumes, callBack) {
|
||||
|
||||
/*
|
||||
|
||||
Here we will write the contents of the Candles and Volumens files.
|
||||
|
||||
*/
|
||||
|
||||
try {
|
||||
|
||||
writeCandles();
|
||||
|
||||
function writeCandles() {
|
||||
|
||||
let separator = "";
|
||||
let fileRecordCounter = 0;
|
||||
|
||||
let fileContent = "";
|
||||
|
||||
for (i = 0; i < candles.length; i++) {
|
||||
|
||||
let candle = candles[i];
|
||||
|
||||
fileContent = fileContent + separator + '[' + candles[i].min + "," + candles[i].max + "," + candles[i].open + "," + candles[i].close + "," + candles[i].begin + "," + candles[i].end + "]";
|
||||
|
||||
if (separator === "") { separator = ","; }
|
||||
|
||||
fileRecordCounter++;
|
||||
|
||||
}
|
||||
|
||||
fileContent = "[" + fileContent + "]";
|
||||
|
||||
let fileName = '' + market.assetA + '_' + market.assetB + '.json';
|
||||
|
||||
let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + "/" + bot.process;
|
||||
|
||||
utilities.createFolderIfNeeded(filePath, oliviaAzureFileStorage, onFolderCreated);
|
||||
|
||||
function onFolderCreated() {
|
||||
|
||||
oliviaAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated);
|
||||
|
||||
function onFileCreated() {
|
||||
|
||||
const logText = "[WARN] Finished with File @ " + market.assetA + "_" + market.assetB + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName + "";
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
writeVolumes();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function writeVolumes() {
|
||||
|
||||
let separator = "";
|
||||
let fileRecordCounter = 0;
|
||||
|
||||
let fileContent = "";
|
||||
|
||||
for (i = 0; i < volumes.length; i++) {
|
||||
|
||||
let candle = volumes[i];
|
||||
|
||||
fileContent = fileContent + separator + '[' + volumes[i].buy + "," + volumes[i].sell + "," + volumes[i].begin + "," + volumes[i].end + "]";
|
||||
|
||||
if (separator === "") { separator = ","; }
|
||||
|
||||
fileRecordCounter++;
|
||||
|
||||
}
|
||||
|
||||
fileContent = "[" + fileContent + "]";
|
||||
|
||||
let fileName = '' + market.assetA + '_' + market.assetB + '.json';
|
||||
|
||||
let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + "/" + bot.process;
|
||||
|
||||
utilities.createFolderIfNeeded(filePath, oliviaAzureFileStorage, onFolderCreated);
|
||||
|
||||
function onFolderCreated() {
|
||||
|
||||
oliviaAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated);
|
||||
|
||||
function onFileCreated() {
|
||||
|
||||
const logText = "[WARN] Finished with File @ " + market.assetA + "_" + market.assetB + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName + "";
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
writeReport();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function writeReport() {
|
||||
|
||||
writeStatusReport(date, onStatusReportWritten);
|
||||
|
||||
function onStatusReportWritten() {
|
||||
|
||||
callBack();
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'writeFiles' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
closeMarket();
|
||||
}
|
||||
}
|
||||
|
||||
function writeStatusReport(lastFileDate, callBack) {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] Entering function 'writeStatusReport'");
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
let reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process;
|
||||
|
||||
utilities.createFolderIfNeeded(reportFilePath, oliviaAzureFileStorage, onFolderCreated);
|
||||
|
||||
function onFolderCreated() {
|
||||
|
||||
try {
|
||||
|
||||
let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json";
|
||||
|
||||
let report = {
|
||||
lastFile: {
|
||||
year: lastFileDate.getUTCFullYear(),
|
||||
month: (lastFileDate.getUTCMonth() + 1),
|
||||
days: lastFileDate.getUTCDate()
|
||||
}
|
||||
};
|
||||
|
||||
let fileContent = JSON.stringify(report);
|
||||
|
||||
oliviaAzureFileStorage.createTextFile(reportFilePath, fileName, fileContent + '\n', onFileCreated);
|
||||
|
||||
function onFileCreated() {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] 'writeStatusReport' - Content written: " + fileContent);
|
||||
}
|
||||
|
||||
callBack();
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'writeStatusReport - onFolderCreated' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
closeMarket();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'writeStatusReport' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
closeMarket();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'Start' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -1,97 +0,0 @@
|
||||
|
||||
let bot = {
|
||||
name: "Olivia",
|
||||
process: "24-hs-Market",
|
||||
type: "Indicator",
|
||||
version: "1.0.0",
|
||||
devTeam: "AA Masters"
|
||||
};
|
||||
|
||||
const ROOT_DIR = '../';
|
||||
|
||||
const MODULE_NAME = "run";
|
||||
|
||||
const START_ALL_MONTHS = false;
|
||||
const START_ONE_MONTH = true;
|
||||
|
||||
const INTERVAL_EXECUTOR_MODULE = require('./Interval Executor');
|
||||
|
||||
const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
|
||||
process.on('uncaughtException', function (err) {
|
||||
logger.write('uncaughtException - ' + err.message);
|
||||
});
|
||||
|
||||
|
||||
process.on('unhandledRejection', (reason, p) => {
|
||||
logger.write("Unhandled Rejection at: Promise " + JSON.stringify(p) + " reason: " + reason);
|
||||
});
|
||||
|
||||
|
||||
process.on('exit', function (code) {
|
||||
logger.write('About to exit with code:' + code);
|
||||
});
|
||||
|
||||
|
||||
try {
|
||||
if (START_ALL_MONTHS === true) {
|
||||
|
||||
for (let year = 2018; year > 2014; year--) {
|
||||
|
||||
for (let month = 12; month > 0; month--) {
|
||||
|
||||
let timeDelay = Math.random() * 10 * 1000; // We introduce a short delay so as to not overload the machine.
|
||||
|
||||
setTimeout(startProcess, timeDelay);
|
||||
|
||||
function startProcess() {
|
||||
|
||||
const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot);
|
||||
|
||||
newIntervalExecutor.initialize(year, month, onInitializeReady);
|
||||
|
||||
function onInitializeReady() {
|
||||
|
||||
newIntervalExecutor.start();
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
if (START_ONE_MONTH === true) {
|
||||
|
||||
startProcess();
|
||||
|
||||
function startProcess() {
|
||||
|
||||
const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot);
|
||||
|
||||
let month = 2;
|
||||
let year = 2018;
|
||||
|
||||
newIntervalExecutor.initialize(year, month, onInitializeReady);
|
||||
|
||||
function onInitializeReady() {
|
||||
|
||||
newIntervalExecutor.start();
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (err) {
|
||||
console.log(err.message);
|
||||
logger.write(err.message);
|
||||
}
|
||||
|
||||
+1
-6
@@ -25,16 +25,11 @@
|
||||
<Compile Include="24-hs-Market\Interval Executor.js" />
|
||||
<Compile Include="24-hs-Market\Interval.js" />
|
||||
<Compile Include="24-hs-Market\run.js" />
|
||||
<Compile Include="Azure File Storage.js" />
|
||||
<Compile Include="Azure Web Job Shutdown.js" />
|
||||
<Compile Include="Debug Log.js" />
|
||||
<Compile Include="Markets.js" />
|
||||
<Compile Include="Poloniex API Client.js" />
|
||||
<Compile Include="Utilities.js" />
|
||||
<Content Include="logs\Dummy File.txt" />
|
||||
<Content Include="obj\Debug\AABruce.njsproj.FileListAbsolute.txt" />
|
||||
<Content Include="obj\Debug\AACarol.njsproj.FileListAbsolute.txt" />
|
||||
<Content Include="obj\Debug\AACharly.njsproj.FileListAbsolute.txt" />
|
||||
<Content Include="this.bot.config.json" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Folder Include="Multi-Period-Market\" />
|
||||
|
||||
@@ -1,247 +0,0 @@
|
||||
|
||||
exports.newAzureFileStorage = function newAzureFileStorage(BOT) {
|
||||
|
||||
let bot = BOT;
|
||||
const ROOT_DIR = '../';
|
||||
|
||||
const MODULE_NAME = "Azure File Storage";
|
||||
|
||||
var fs = require('fs');
|
||||
var util = require('util');
|
||||
var guid = require('node-uuid');
|
||||
var crypto = require('crypto');
|
||||
var storage = require('azure-storage');
|
||||
|
||||
const DEBUG_MODULE = require('./Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
|
||||
const shareName = 'data';
|
||||
let dataOwner;
|
||||
|
||||
let azureFileStorage = {
|
||||
initialize: initialize,
|
||||
createFolder: createFolder,
|
||||
createTextFile: createTextFile,
|
||||
getTextFile: getTextFile,
|
||||
listFilesAndFolders: listFilesAndFolders
|
||||
};
|
||||
|
||||
let fileService;
|
||||
|
||||
return azureFileStorage;
|
||||
|
||||
|
||||
function initialize(dataOwnerBotName) {
|
||||
|
||||
try {
|
||||
|
||||
if (dataOwnerBotName === undefined) {
|
||||
|
||||
dataOwner = bot.name;
|
||||
|
||||
} else {
|
||||
|
||||
dataOwner = dataOwnerBotName;
|
||||
|
||||
}
|
||||
|
||||
fileService = storage.createFileService(readConfig().connectionString);
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'initialize' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
|
||||
function readConfig() {
|
||||
|
||||
try {
|
||||
|
||||
return JSON.parse(fs.readFileSync(ROOT_DIR + dataOwner + '.azure.storage.connstring', 'utf8'));
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'readConfig' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function createFolder(folderPath, callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
fileService.createDirectoryIfNotExists(shareName, folderPath, onFolderCreated);
|
||||
|
||||
function onFolderCreated(err) {
|
||||
|
||||
if (err) {
|
||||
|
||||
const logText = "[ERROR] createFolder - onFolderCreated ' ERROR = " + err.message + " SHARE NAME = " + shareName + " FOLDER PATH = " + folderPath;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
|
||||
callBackFunction();
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'createFolder' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function createTextFile(folderPath, fileName, fileContent, callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
fileService.createFileFromText(shareName, folderPath, fileName, fileContent, onFileCreated);
|
||||
|
||||
function onFileCreated(err, result, response) {
|
||||
|
||||
if (err) {
|
||||
|
||||
const logText = "[WARN] createTextFile - onFileCreated ' ERROR = " + err.message + " SHARE NAME = " + shareName + " FOLDER PATH = " + folderPath + " FILE NAME = " + fileName;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
if (err.message.indexOf("The server is busy") > 0) {
|
||||
|
||||
fileService.createFileFromText(shareName, folderPath, fileName, fileContent, onFileCreatedSecondTry);
|
||||
|
||||
function onFileCreatedSecondTry(err) {
|
||||
|
||||
if (err) {
|
||||
|
||||
const logText = "[ERROR] createTextFile - onFileCreated - onFileCreatedSecondTry : File not created. Giving Up.' ERROR = " + err.message + " SHARE NAME = " + shareName + " FOLDER PATH = " + folderPath + " FILE NAME = " + fileName;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
} else {
|
||||
|
||||
const logText = "[INFO] createTextFile - onFileCreated - onFileCreatedSecondTry ' : File succesfully created on second try. SHARE NAME = " + shareName + " FOLDER PATH = " + folderPath + " FILE NAME = " + fileName;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
callBackFunction();
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'createTextFile' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function getTextFile(folderPath, fileName, callBackFunction, ignoreNotFound) {
|
||||
|
||||
try {
|
||||
|
||||
fileService.getFileToText(shareName, folderPath, fileName, undefined, onFileReceived);
|
||||
|
||||
function onFileReceived(err, text, response) {
|
||||
|
||||
if (err) {
|
||||
|
||||
if (ignoreNotFound === undefined) {
|
||||
|
||||
const logText = "[ERROR] getTextFile - onFileReceived ' ERROR = " + err.message + " SHARE NAME = " + shareName + " FOLDER PATH = " + folderPath + " FILE NAME = " + fileName;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
|
||||
text = undefined;
|
||||
|
||||
}
|
||||
|
||||
callBackFunction(text);
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'getTextFile' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function listFilesAndFolders(folderPath, callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
let items = {
|
||||
files: [],
|
||||
folders: []
|
||||
};
|
||||
|
||||
/*
|
||||
* @param { object } [options] The request options.
|
||||
* @param { int } [options.maxResults] Specifies the maximum number of folders to return per call to Azure ServiceClient.
|
||||
* This does NOT affect list size returned by this function. (maximum: 5000)
|
||||
* @param { LocationMode } [options.locationMode] Specifies the location mode used to decide which location the request should be sent to.
|
||||
* Please see StorageUtilities.LocationMode for the possible values.
|
||||
* @param { int } [options.timeoutIntervalInMs] The server timeout interval, in milliseconds, to use for the request.
|
||||
* @param { int } [options.maximumExecutionTimeInMs] The maximum execution time, in milliseconds, across all potential retries, to use when making this request.
|
||||
* The maximum execution time interval begins at the time that the client begins building the request.The maximum
|
||||
* execution time is checked intermittently while performing requests, and before executing retries.
|
||||
* @param { string } [options.clientRequestId] A string that represents the client request ID with a 1KB character limit.
|
||||
* @param { bool } [options.useNagleAlgorithm] Determines whether the Nagle al
|
||||
*/
|
||||
|
||||
let options = {
|
||||
maxResults: 500
|
||||
};
|
||||
|
||||
fileService.listFilesAndDirectoriesSegmented(shareName, folderPath, null, options, onFilesAndFoldersReceived);
|
||||
|
||||
|
||||
function onFilesAndFoldersReceived(error, result) {
|
||||
|
||||
try {
|
||||
|
||||
if (error) {
|
||||
|
||||
const logText = "[ERROR] 'listFilesAndFolders - onFilesAndFoldersReceived' - ERROR RECEIVED : " + error;
|
||||
logger.write(logText);
|
||||
|
||||
callBackFunction(items);
|
||||
|
||||
} else {
|
||||
|
||||
items.files.push.apply(items.files, result.entries.files);
|
||||
items.folders.push.apply(items.folders, result.entries.directories);
|
||||
|
||||
callBackFunction(items);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'listFilesAndFolders - onFilesAndFoldersReceived' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'listFilesAndFolders' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
@@ -1,41 +0,0 @@
|
||||
|
||||
exports.newShutdownEvent = function newShutdownEvent(BOT) {
|
||||
|
||||
let bot = BOT;
|
||||
const ROOT_DIR = '../';
|
||||
|
||||
const MODULE_NAME = "Azure Web Job Shutdown";
|
||||
|
||||
|
||||
const DEBUG_MODULE = require('./Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
|
||||
let shutdownEvent = {
|
||||
isShuttingDown: isShuttingDown
|
||||
};
|
||||
|
||||
return shutdownEvent;
|
||||
|
||||
|
||||
function isShuttingDown() {
|
||||
|
||||
let filePath = process.env.WEBJOBS_SHUTDOWN_FILE;
|
||||
|
||||
let fs = require('fs');
|
||||
|
||||
if (fs.existsSync(filePath)) {
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
-144
@@ -1,144 +0,0 @@
|
||||
|
||||
|
||||
|
||||
exports.newDebugLog = function newDebugLog() {
|
||||
|
||||
|
||||
const MODULE_NAME = "Debug Log";
|
||||
const fileSystem = require('fs');
|
||||
|
||||
const currentDate = new Date();
|
||||
const dateString = currentDate.getUTCFullYear() + '-' + pad(currentDate.getUTCMonth() + 1, 2) + '-' + pad(currentDate.getUTCDate(), 2) + '-' + pad(currentDate.getUTCHours(), 2) + '-' + pad(currentDate.getUTCMinutes(), 2);
|
||||
const randomId = parseInt(Math.random() * 1000000);
|
||||
|
||||
let fileNumber = 1;
|
||||
let messageId = 0;
|
||||
let firstCall = true;
|
||||
let folderPath;
|
||||
|
||||
debugLog = {
|
||||
bot: undefined,
|
||||
fileName: undefined,
|
||||
write: write
|
||||
};
|
||||
|
||||
|
||||
|
||||
return debugLog;
|
||||
|
||||
|
||||
function createFolders() {
|
||||
|
||||
try {
|
||||
|
||||
folderPath = debugLog.bot.devTeam;
|
||||
|
||||
createFolderSync(folderPath);
|
||||
|
||||
folderPath = debugLog.bot.devTeam + "/" + debugLog.bot.type;
|
||||
|
||||
createFolderSync(folderPath);
|
||||
|
||||
folderPath = debugLog.bot.devTeam + "/" + debugLog.bot.type + "/" + debugLog.bot.name;
|
||||
|
||||
createFolderSync(folderPath);
|
||||
|
||||
folderPath = debugLog.bot.devTeam + "/" + debugLog.bot.type + "/" + debugLog.bot.name + "/" + debugLog.bot.version;
|
||||
|
||||
createFolderSync(folderPath);
|
||||
|
||||
folderPath = debugLog.bot.devTeam + "/" + debugLog.bot.type + "/" + debugLog.bot.name + "/" + debugLog.bot.version + "/" + debugLog.bot.process;
|
||||
|
||||
createFolderSync(folderPath);
|
||||
|
||||
firstCall = false;
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
console.log("Error trying to create the folders needed. Error: " + err);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
function write(Message) {
|
||||
|
||||
if (firstCall === true) {
|
||||
|
||||
createFolders();
|
||||
|
||||
}
|
||||
|
||||
let filePath = getCurrentLogFile(folderPath + "/" + dateString + "---" + randomId + "---", this.fileName);
|
||||
|
||||
let newDate = new Date();
|
||||
newDate = newDate.toISOString();
|
||||
|
||||
messageId++;
|
||||
|
||||
try {
|
||||
|
||||
fileSystem.appendFileSync(filePath, '\r\n' + newDate + " " + messageId + " " + Message);
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
console.log("Error trying to log info into a file. File: " + filePath + " Error:" + err);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function createFolderSync(name) {
|
||||
try {
|
||||
fileSystem.mkdirSync( '../logs/' + name)
|
||||
} catch (err) {
|
||||
if (err.code !== 'EEXIST') throw err
|
||||
}
|
||||
}
|
||||
|
||||
function getCurrentLogFile(relativePath, fileName) {
|
||||
|
||||
let filePath;
|
||||
let stats;
|
||||
let i;
|
||||
|
||||
try {
|
||||
for (i = 1; i < 1000000; i++) {
|
||||
|
||||
filePath = '../logs/' + relativePath + i + "." + fileName + '.log';
|
||||
stats = fileSystem.statSync(filePath);
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
|
||||
if (i > 1) {
|
||||
|
||||
filePath = '../logs/' + relativePath + (i - 1) + "." + fileName + '.log';
|
||||
|
||||
stats = fileSystem.statSync(filePath);
|
||||
const fileSizeInBytes = stats.size;
|
||||
|
||||
if (fileSizeInBytes > 10240000) {
|
||||
|
||||
filePath = '../logs/' + relativePath + i + "." + fileName + '.log';
|
||||
}
|
||||
|
||||
return filePath;
|
||||
|
||||
} else {
|
||||
|
||||
return filePath;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function pad(str, max) {
|
||||
str = str.toString();
|
||||
return str.length < max ? pad("0" + str, max) : str;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
-252
@@ -1,252 +0,0 @@
|
||||
/* This business logic module is responsible for Markets and Assets. */
|
||||
|
||||
|
||||
exports.newMarkets = function newMarkets(BOT) {
|
||||
|
||||
let bot = BOT;
|
||||
const ROOT_DIR = '../';
|
||||
|
||||
const MODULE_NAME = "Markets";
|
||||
|
||||
const FOLDER_PATHS = 'data';
|
||||
const MARKETS_FOLDER_NAME = "other-files";
|
||||
|
||||
const DEBUG_MODULE = require('./Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
|
||||
const EXCHANGE_ID = 1;
|
||||
const fileName = "Markets_" + EXCHANGE_ID + ".json";
|
||||
|
||||
const DATA_OWNER_BOT_NAME = "Carol"
|
||||
|
||||
const markets = {
|
||||
DISABLED: 0,
|
||||
ENABLED: 1,
|
||||
getMarketsByExchange: getMarketsByExchange,
|
||||
getExistingOrNewMarketId: getExistingOrNewMarketId,
|
||||
createNewAsset: createNewAsset,
|
||||
createNewMarket: createNewMarket,
|
||||
disableMarket: disableMarket,
|
||||
initialize: initialize
|
||||
};
|
||||
|
||||
//const dataLayer = DATA_LAYER_MODULE.newDataLayer();
|
||||
|
||||
let marketsArray;
|
||||
|
||||
const AZURE_FILE_STORAGE = require('./Azure File Storage');
|
||||
let azureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
|
||||
|
||||
return markets;
|
||||
|
||||
|
||||
function initialize(callBackFunction) {
|
||||
|
||||
azureFileStorage.initialize(DATA_OWNER_BOT_NAME);
|
||||
|
||||
try {
|
||||
|
||||
azureFileStorage.getTextFile(MARKETS_FOLDER_NAME, fileName, onFileRead);
|
||||
|
||||
function onFileRead(text) {
|
||||
|
||||
try {
|
||||
|
||||
var fileString = text;
|
||||
let cleanString = fileString;
|
||||
|
||||
/* Dont know why but sometimes files comes with a first characater that is not supposed to be there and you can not see it at the file opened with notepad */
|
||||
|
||||
if (fileString[0] !== "[") {
|
||||
|
||||
cleanString = '';
|
||||
|
||||
for (let i = 1; i < fileString.length; i++) {
|
||||
cleanString = cleanString + fileString[i];
|
||||
}
|
||||
}
|
||||
|
||||
marketsArray = JSON.parse(cleanString);
|
||||
callBackFunction();
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] initialize - onFileRead ' ERROR : " + err.message + " File: " + fileName;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] initialize - ' ERROR : " + err.message + " File Not Found: " + fileName;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
function getMarketsByExchange(exchangeId, callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
// TODO: Currently the exchange Id is ignored as the class is still working with only one exchange. We must fix this.
|
||||
|
||||
callBackFunction(JSON.stringify(marketsArray));
|
||||
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] getMarketsByExchange - ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
function getExistingOrNewMarketId(marketName, market, exchangeId, callBackFunction) {
|
||||
|
||||
// TODO: At this function is missing the following: If a market does not exists, it must be greated, and Id generated and returned. We automatically discover new markets when this function is called with a market that didn-t previuously existed.
|
||||
|
||||
try {
|
||||
|
||||
for (let i = 0; i < marketsArray.length; i++) {
|
||||
|
||||
if (marketsArray[i][1] + "_" + marketsArray[i][2] === marketName) {
|
||||
|
||||
callBackFunction(marketName, market, marketsArray[i][0]);
|
||||
return;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] getExistingOrNewMarketId - ' ERROR : " + err.message + " marketId = " + marketId + " idAssetA = " + idAssetA + " idAssetB = " + idAssetB;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
function createNewAsset(assetCode, callBackFunction) {
|
||||
|
||||
// TODO: Upgrade this function in order to update the file containing all assets, not the database that will be deprecated.
|
||||
|
||||
try {
|
||||
|
||||
dataLayer.newAsset(assetCode, onIdReady);
|
||||
|
||||
function onIdReady(resultSet) {
|
||||
|
||||
try {
|
||||
|
||||
callBackFunction(resultSet);
|
||||
resultSet = [];
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] createNewAsset - onIdReady ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] createNewAsset - ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
function createNewMarket(exchangeId, idAssetA, idAssetB, callBackFunction) {
|
||||
|
||||
// TODO: Upgrade this function in order to update the file containing all markets, not the database that will be deprecated.
|
||||
|
||||
try {
|
||||
|
||||
dataLayer.newMarket(exchangeId, idAssetA, idAssetB, onIdReady);
|
||||
|
||||
function onIdReady(resultSet) {
|
||||
|
||||
try {
|
||||
|
||||
callBackFunction(resultSet);
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] createNewMarket - onIdReady ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] createNewMarket - ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function disableMarket(exchangeId, marketId, callBackFunction) {
|
||||
|
||||
// This function puts the requested market in a DISABLED status.
|
||||
|
||||
// TODO: Currently the exchangeId is ignored.
|
||||
|
||||
try {
|
||||
|
||||
for (let i = 0; i < marketsArray.length; i++) {
|
||||
|
||||
if (marketsArray[i][0] === marketId) {
|
||||
|
||||
marketsArray[i][3] = markets.DISABLED;
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
azureFileStorage.createTextFile(MARKETS_FOLDER_NAME, fileName, JSON.stringify(marketsArray), onFileReplaced);
|
||||
|
||||
function onFileReplaced() {
|
||||
|
||||
try {
|
||||
|
||||
callBackFunction();
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] disableMarket - onFileReplaced ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] disableMarket - ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
@@ -1,169 +0,0 @@
|
||||
exports.newIntervalExecutor = function newIntervalExecutor(BOT) {
|
||||
|
||||
let bot = BOT;
|
||||
const ROOT_DIR = '../';
|
||||
|
||||
const INTERVAL_LAPSE = 60 * 60 * 1000; // There is no need for more frecuency since 1 hour files are generated at maximun.
|
||||
|
||||
const MODULE_NAME = "Interval Executor";
|
||||
const LOG_INFO = true;
|
||||
|
||||
const INTERVAL_MODULE = require('./Interval');
|
||||
|
||||
const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
|
||||
const INTERVAL_MAX_LOOPS = -1; // -1 for infinite loop.
|
||||
|
||||
poloniexHoleFixing = {
|
||||
initialize: initialize,
|
||||
start: start
|
||||
};
|
||||
|
||||
const SHUTDOWN_EVENT = require(ROOT_DIR + 'Azure Web Job Shutdown');
|
||||
let shutdownEvent = SHUTDOWN_EVENT.newShutdownEvent(bot);
|
||||
|
||||
const UTILITIES = require(ROOT_DIR + 'Utilities');
|
||||
let utilities = UTILITIES.newUtilities(bot);
|
||||
|
||||
let year;
|
||||
let month;
|
||||
|
||||
return poloniexHoleFixing;
|
||||
|
||||
function initialize(yearAssigend, monthAssigned, callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
year = yearAssigend;
|
||||
month = monthAssigned;
|
||||
|
||||
month = utilities.pad(month, 2); // Adding a left zero when needed.
|
||||
|
||||
logger.fileName = MODULE_NAME + "-" + year + "-" + month;
|
||||
|
||||
const logText = "[INFO] initialize - Entering function 'initialize' " + " @ " + year + "-" + month;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
callBackFunction();
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] initialize - ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function start() {
|
||||
|
||||
try {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] Entering function 'start', with year = " + year + " and month = " + month);
|
||||
}
|
||||
|
||||
let intervalId;
|
||||
|
||||
if (INTERVAL_MAX_LOOPS === 0) {
|
||||
|
||||
intervalId = setInterval(startNewInterval, INTERVAL_LAPSE);
|
||||
|
||||
}
|
||||
|
||||
let intervalLoopCounter = 0;
|
||||
|
||||
startNewInterval(); // First, inmadiate run.
|
||||
|
||||
function startNewInterval() {
|
||||
|
||||
try {
|
||||
|
||||
if (LOG_INFO === true) {
|
||||
|
||||
if (INTERVAL_MAX_LOOPS !== 0) {
|
||||
|
||||
logger.write("[INFO] Entering function 'startNewInterval', next execution inmediately after the current Interval finishes.");
|
||||
|
||||
} else {
|
||||
|
||||
logger.write("[INFO] Entering function 'startNewInterval', next execution in " + INTERVAL_LAPSE / 1000 / 60 + " minutes.");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
if (shutdownEvent.isShuttingDown() === false) {
|
||||
|
||||
let fileProcessingInterval = INTERVAL_MODULE.newInterval(bot);
|
||||
|
||||
fileProcessingInterval.initialize(year, month, onReadyToStart);
|
||||
|
||||
function onReadyToStart() {
|
||||
|
||||
fileProcessingInterval.start(loopControl);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
|
||||
clearInterval(intervalId);
|
||||
|
||||
const logText = "[INFO] requestNewData - Terminating Set Interval - About to exit gracefully the process. ";
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'startNewInterval' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
|
||||
function loopControl(moreIntervalExecutionsNeeded) {
|
||||
|
||||
if (moreIntervalExecutionsNeeded === true) {
|
||||
|
||||
intervalLoopCounter++;
|
||||
|
||||
if (intervalLoopCounter === INTERVAL_MAX_LOOPS) {
|
||||
|
||||
const logText = "[WARN] 'INTERVAL_MAX_LOOPS' reached. Stopping execution. ";
|
||||
logger.write(logText);
|
||||
|
||||
} else {
|
||||
|
||||
const logText = "[WARN] 'loopControl'. Staring new Interval. ";
|
||||
logger.write(logText);
|
||||
|
||||
setTimeout(startNewInterval, INTERVAL_LAPSE);// Looped Execution.
|
||||
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
logger.write("[WARN] Received the signal from last Interval execution to not call it anymore. Stopping execution. ");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'Start' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
exports.newInterval = function newInterval(BOT) {
|
||||
exports.newInterval = function newInterval(BOT, UTILITIES, AZURE_FILE_STORAGE, DEBUG_MODULE, MARKETS_MODULE, POLONIEX_CLIENT_MODULE) {
|
||||
|
||||
let bot = BOT;
|
||||
const ROOT_DIR = '../';
|
||||
|
||||
const GMT_SECONDS = ':00.000 GMT+0000';
|
||||
const GMT_MILI_SECONDS = '.000 GMT+0000';
|
||||
const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000;
|
||||
@@ -23,9 +23,6 @@
|
||||
const GO_RANDOM = false;
|
||||
const FORCE_MARKET = 2; // This allows to debug the execution of an specific market. Not intended for production.
|
||||
|
||||
const MARKETS_MODULE = require(ROOT_DIR + 'Markets');
|
||||
|
||||
const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
@@ -37,12 +34,10 @@
|
||||
|
||||
let markets;
|
||||
|
||||
const AZURE_FILE_STORAGE = require(ROOT_DIR + 'Azure File Storage');
|
||||
let charlyAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
|
||||
let bruceAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
|
||||
let oliviaAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
|
||||
|
||||
const UTILITIES = require(ROOT_DIR + 'Utilities');
|
||||
let utilities = UTILITIES.newUtilities(bot);
|
||||
|
||||
let year;
|
||||
@@ -297,7 +292,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each
|
||||
|
||||
*/
|
||||
|
||||
reportFilePath = EXCHANGE_NAME + "/Processes/" + "Historic-Trades";
|
||||
reportFilePath = EXCHANGE_NAME + "/Processes/" + "Poloniex-Historic-Trades";
|
||||
|
||||
charlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true);
|
||||
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
|
||||
let bot = {
|
||||
name: "Olivia",
|
||||
process: "Multi-Period-Market",
|
||||
type: "Indicator",
|
||||
version: "1.0.0",
|
||||
devTeam: "AA Masters"
|
||||
};
|
||||
|
||||
const ROOT_DIR = '../';
|
||||
|
||||
const MODULE_NAME = "run";
|
||||
|
||||
const START_ALL_MONTHS = false;
|
||||
const START_ONE_MONTH = true;
|
||||
|
||||
const INTERVAL_EXECUTOR_MODULE = require('./Interval Executor');
|
||||
|
||||
const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
|
||||
process.on('uncaughtException', function (err) {
|
||||
logger.write('uncaughtException - ' + err.message);
|
||||
});
|
||||
|
||||
|
||||
process.on('unhandledRejection', (reason, p) => {
|
||||
logger.write("Unhandled Rejection at: Promise " + JSON.stringify(p) + " reason: " + reason);
|
||||
});
|
||||
|
||||
|
||||
process.on('exit', function (code) {
|
||||
logger.write('About to exit with code:' + code);
|
||||
});
|
||||
|
||||
|
||||
try {
|
||||
if (START_ALL_MONTHS === true) {
|
||||
|
||||
for (let year = 2018; year > 2014; year--) {
|
||||
|
||||
for (let month = 12; month > 0; month--) {
|
||||
|
||||
let timeDelay = Math.random() * 10 * 1000; // We introduce a short delay so as to not overload the machine.
|
||||
|
||||
setTimeout(startProcess, timeDelay);
|
||||
|
||||
function startProcess() {
|
||||
|
||||
const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot);
|
||||
|
||||
newIntervalExecutor.initialize(year, month, onInitializeReady);
|
||||
|
||||
function onInitializeReady() {
|
||||
|
||||
newIntervalExecutor.start();
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
if (START_ONE_MONTH === true) {
|
||||
|
||||
startProcess();
|
||||
|
||||
function startProcess() {
|
||||
|
||||
const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot);
|
||||
|
||||
let month = 2;
|
||||
let year = 2018;
|
||||
|
||||
newIntervalExecutor.initialize(year, month, onInitializeReady);
|
||||
|
||||
function onInitializeReady() {
|
||||
|
||||
newIntervalExecutor.start();
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (err) {
|
||||
console.log(err.message);
|
||||
logger.write(err.message);
|
||||
}
|
||||
|
||||
@@ -1,406 +0,0 @@
|
||||
module.exports = (function () {
|
||||
'use strict';
|
||||
|
||||
// Module dependencies
|
||||
var crypto = require('crypto'),
|
||||
request = require('request'),
|
||||
nonce = require('nonce')();
|
||||
|
||||
// Constants
|
||||
var version = '0.0.8',
|
||||
PUBLIC_API_URL = 'https://poloniex.com/public',
|
||||
PRIVATE_API_URL = 'https://poloniex.com/tradingApi',
|
||||
USER_AGENT = 'poloniex.js ' + version;
|
||||
//USER_AGENT = 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0'
|
||||
|
||||
// Helper methods
|
||||
function joinCurrencies(currencyA, currencyB) {
|
||||
// If only one arg, then return the first
|
||||
if (typeof currencyB !== 'string') {
|
||||
return currencyA;
|
||||
}
|
||||
|
||||
return currencyA + '_' + currencyB;
|
||||
}
|
||||
|
||||
// Constructor
|
||||
function Poloniex(key, secret) {
|
||||
// Generate headers signed by this user's key and secret.
|
||||
// The secret is encapsulated and never exposed
|
||||
this._getPrivateHeaders = function (parameters) {
|
||||
var paramString, signature;
|
||||
|
||||
if (!key || !secret) {
|
||||
throw 'Poloniex: Error. API key and secret required';
|
||||
}
|
||||
|
||||
// Convert to `arg1=foo&arg2=bar`
|
||||
paramString = Object.keys(parameters).map(function (param) {
|
||||
return encodeURIComponent(param) + '=' + encodeURIComponent(parameters[param]);
|
||||
}).join('&');
|
||||
|
||||
signature = crypto.createHmac('sha512', secret).update(paramString).digest('hex');
|
||||
|
||||
return {
|
||||
Key: key,
|
||||
Sign: signature
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
// Currently, this fails with `Error: CERT_UNTRUSTED`
|
||||
// Poloniex.STRICT_SSL can be set to `false` to avoid this. Use with caution.
|
||||
// Will be removed in future, once this is resolved.
|
||||
Poloniex.STRICT_SSL = true;
|
||||
|
||||
// Customisable user agent string
|
||||
Poloniex.USER_AGENT = USER_AGENT;
|
||||
|
||||
// Prototype
|
||||
Poloniex.prototype = {
|
||||
constructor: Poloniex,
|
||||
|
||||
// Make an API request
|
||||
_request: function (options, callback) {
|
||||
if (!('headers' in options)) {
|
||||
options.headers = {};
|
||||
}
|
||||
|
||||
options.json = true;
|
||||
options.headers['User-Agent'] = Poloniex.USER_AGENT;
|
||||
options.strictSSL = Poloniex.STRICT_SSL;
|
||||
|
||||
request(options, function (err, response, body) {
|
||||
// Empty response
|
||||
if (!err && (typeof body === 'undefined' || body === null)) {
|
||||
err = 'Empty response';
|
||||
}
|
||||
|
||||
callback(err, body);
|
||||
});
|
||||
|
||||
return this;
|
||||
},
|
||||
|
||||
// Make a public API request
|
||||
_public: function (command, parameters, callback) {
|
||||
var options;
|
||||
|
||||
if (typeof parameters === 'function') {
|
||||
callback = parameters;
|
||||
parameters = {};
|
||||
}
|
||||
|
||||
parameters || (parameters = {});
|
||||
parameters.command = command;
|
||||
options = {
|
||||
method: 'GET',
|
||||
url: PUBLIC_API_URL,
|
||||
qs: parameters
|
||||
};
|
||||
|
||||
options.qs.command = command;
|
||||
return this._request(options, callback);
|
||||
},
|
||||
|
||||
// Make a private API request
|
||||
_private: function (command, parameters, callback) {
|
||||
var options;
|
||||
|
||||
if (typeof parameters === 'function') {
|
||||
callback = parameters;
|
||||
parameters = {};
|
||||
}
|
||||
|
||||
parameters || (parameters = {});
|
||||
parameters.command = command;
|
||||
parameters.nonce = nonce();
|
||||
|
||||
options = {
|
||||
method: 'POST',
|
||||
url: PRIVATE_API_URL,
|
||||
form: parameters,
|
||||
headers: this._getPrivateHeaders(parameters)
|
||||
};
|
||||
|
||||
return this._request(options, callback);
|
||||
},
|
||||
|
||||
/////
|
||||
// PUBLIC METHODS
|
||||
|
||||
returnTicker: function (callback) {
|
||||
return this._public('returnTicker', callback);
|
||||
},
|
||||
|
||||
return24hVolume: function (callback) {
|
||||
return this._public('return24hVolume', callback);
|
||||
},
|
||||
|
||||
returnOrderBook: function (currencyA, currencyB, depth, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB),
|
||||
depth: depth
|
||||
};
|
||||
|
||||
return this._public('returnOrderBook', parameters, callback);
|
||||
},
|
||||
|
||||
returnOrderBooks: function (depth, callback) {
|
||||
var parameters = {
|
||||
currencyPair: 'all',
|
||||
depth: depth
|
||||
};
|
||||
|
||||
return this._public('returnOrderBook', parameters, callback);
|
||||
},
|
||||
|
||||
returnPublicTradeHistory: function (currencyA, currencyB, start, end, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB),
|
||||
start: start,
|
||||
end: end
|
||||
};
|
||||
|
||||
return this._public('returnTradeHistory', parameters, callback);
|
||||
},
|
||||
|
||||
returnChartData: function (currencyA, currencyB, period, start, end, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB),
|
||||
period: period,
|
||||
start: start,
|
||||
end: end
|
||||
};
|
||||
|
||||
return this._public('returnChartData', parameters, callback);
|
||||
},
|
||||
|
||||
returnCurrencies: function (callback) {
|
||||
return this._public('returnCurrencies', callback);
|
||||
},
|
||||
|
||||
returnLoanOrders: function (currency, callback) {
|
||||
return this._public('returnLoanOrders', { currency: currency }, callback);
|
||||
},
|
||||
|
||||
/////
|
||||
// PRIVATE METHODS
|
||||
|
||||
returnBalances: function (callback) {
|
||||
return this._private('returnBalances', {}, callback);
|
||||
},
|
||||
|
||||
returnCompleteBalances: function (callback) {
|
||||
return this._private('returnCompleteBalances', {}, callback);
|
||||
},
|
||||
|
||||
returnDepositAddresses: function (callback) {
|
||||
return this._private('returnDepositAddresses', {}, callback);
|
||||
},
|
||||
|
||||
generateNewAddress: function (currency, callback) {
|
||||
return this._private('returnDepositsWithdrawals', { currency: currency }, callback);
|
||||
},
|
||||
|
||||
returnDepositsWithdrawals: function (start, end, callback) {
|
||||
return this._private('returnDepositsWithdrawals', { start: start, end: end }, callback);
|
||||
},
|
||||
|
||||
returnOpenOrders: function (currencyA, currencyB, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB)
|
||||
};
|
||||
|
||||
return this._private('returnOpenOrders', parameters, callback);
|
||||
},
|
||||
|
||||
returnTradeHistory: function (currencyA, currencyB, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB)
|
||||
};
|
||||
|
||||
return this._private('returnTradeHistory', parameters, callback);
|
||||
},
|
||||
|
||||
returnOrderTrades: function (orderNumber, callback) {
|
||||
var parameters = {
|
||||
orderNumber: orderNumber
|
||||
};
|
||||
|
||||
return this._private('returnOrderTrades', parameters, callback);
|
||||
},
|
||||
|
||||
buy: function (currencyA, currencyB, rate, amount, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB),
|
||||
rate: rate,
|
||||
amount: amount
|
||||
};
|
||||
|
||||
return this._private('buy', parameters, callback);
|
||||
},
|
||||
|
||||
sell: function (currencyA, currencyB, rate, amount, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB),
|
||||
rate: rate,
|
||||
amount: amount
|
||||
};
|
||||
|
||||
return this._private('sell', parameters, callback);
|
||||
},
|
||||
|
||||
cancelOrder: function (currencyA, currencyB, orderNumber, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB),
|
||||
orderNumber: orderNumber
|
||||
};
|
||||
|
||||
return this._private('cancelOrder', parameters, callback);
|
||||
},
|
||||
|
||||
moveOrder: function (orderNumber, rate, amount, callback) {
|
||||
var parameters = {
|
||||
orderNumber: orderNumber,
|
||||
rate: rate,
|
||||
amount: amount ? amount : null
|
||||
};
|
||||
|
||||
return this._private('moveOrder', parameters, callback);
|
||||
},
|
||||
|
||||
withdraw: function (currency, amount, address, callback) {
|
||||
var parameters = {
|
||||
currency: currency,
|
||||
amount: amount,
|
||||
address: address
|
||||
};
|
||||
|
||||
return this._private('withdraw', parameters, callback);
|
||||
},
|
||||
|
||||
returnFeeInfo: function (callback) {
|
||||
return this._private('returnFeeInfo', {}, callback);
|
||||
},
|
||||
|
||||
returnAvailableAccountBalances: function (account, callback) {
|
||||
var options = {};
|
||||
if (account) {
|
||||
options.account = account;
|
||||
}
|
||||
return this._private('returnAvailableAccountBalances', options, callback);
|
||||
},
|
||||
|
||||
returnTradableBalances: function (callback) {
|
||||
return this._private('returnTradableBalances', {}, callback);
|
||||
},
|
||||
|
||||
transferBalance: function (currency, amount, fromAccount, toAccount, callback) {
|
||||
var parameters = {
|
||||
currency: currency,
|
||||
amount: amount,
|
||||
fromAccount: fromAccount,
|
||||
toAccount: toAccount
|
||||
};
|
||||
|
||||
return this._private('transferBalance', parameters, callback);
|
||||
},
|
||||
|
||||
returnMarginAccountSummary: function (callback) {
|
||||
return this._private('returnMarginAccountSummary', {}, callback);
|
||||
},
|
||||
|
||||
marginBuy: function (currencyA, currencyB, rate, amount, lendingRate, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB),
|
||||
rate: rate,
|
||||
amount: amount,
|
||||
lendingRate: lendingRate ? lendingRate : null
|
||||
};
|
||||
|
||||
return this._private('marginBuy', parameters, callback);
|
||||
},
|
||||
|
||||
marginSell: function (currencyA, currencyB, rate, amount, lendingRate, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB),
|
||||
rate: rate,
|
||||
amount: amount,
|
||||
lendingRate: lendingRate ? lendingRate : null
|
||||
};
|
||||
|
||||
return this._private('marginSell', parameters, callback);
|
||||
},
|
||||
|
||||
getMarginPosition: function (currencyA, currencyB, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB)
|
||||
};
|
||||
|
||||
return this._private('getMarginPosition', parameters, callback);
|
||||
},
|
||||
|
||||
closeMarginPosition: function (currencyA, currencyB, callback) {
|
||||
var parameters = {
|
||||
currencyPair: joinCurrencies(currencyA, currencyB)
|
||||
};
|
||||
|
||||
return this._private('closeMarginPosition', parameters, callback);
|
||||
},
|
||||
|
||||
createLoanOffer: function (currency, amount, duration, autoRenew, lendingRate, callback) {
|
||||
var parameters = {
|
||||
currency: currency,
|
||||
amount: amount,
|
||||
duration: duration,
|
||||
autoRenew: autoRenew,
|
||||
lendingRate: lendingRate
|
||||
};
|
||||
|
||||
return this._private('createLoanOffer', parameters, callback);
|
||||
},
|
||||
|
||||
cancelLoanOffer: function (orderNumber, callback) {
|
||||
var parameters = {
|
||||
orderNumber: orderNumber
|
||||
};
|
||||
|
||||
return this._private('cancelLoanOffer', parameters, callback);
|
||||
},
|
||||
|
||||
returnOpenLoanOffers: function (callback) {
|
||||
return this._private('returnOpenLoanOffers', {}, callback);
|
||||
},
|
||||
|
||||
returnActiveLoans: function (callback) {
|
||||
return this._private('returnActiveLoans', {}, callback);
|
||||
},
|
||||
|
||||
returnLendingHistory: function (start, end, limit, callback) {
|
||||
var parameters = {
|
||||
start: start,
|
||||
end: end,
|
||||
limit: limit
|
||||
};
|
||||
|
||||
return this._private('returnLendingHistory', parameters, callback);
|
||||
},
|
||||
|
||||
toggleAutoRenew: function (orderNumber, callback) {
|
||||
return this._private('toggleAutoRenew', { orderNumber: orderNumber }, callback);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
// Backwards Compatibility
|
||||
Poloniex.prototype.getTicker = Poloniex.prototype.returnTicker;
|
||||
Poloniex.prototype.get24hVolume = Poloniex.prototype.return24hVolume;
|
||||
Poloniex.prototype.getOrderBook = Poloniex.prototype.returnOrderBook;
|
||||
Poloniex.prototype.getTradeHistory = Poloniex.prototype.returnChartData;
|
||||
Poloniex.prototype.myBalances = Poloniex.prototype.returnBalances;
|
||||
Poloniex.prototype.myOpenOrders = Poloniex.prototype.returnOpenOrders;
|
||||
Poloniex.prototype.myTradeHistory = Poloniex.prototype.returnTradeHistory;
|
||||
|
||||
return Poloniex;
|
||||
})();
|
||||
@@ -1,81 +0,0 @@
|
||||
|
||||
exports.newUtilities = function newUtilities(BOT) {
|
||||
|
||||
let bot = BOT;
|
||||
const ROOT_DIR = '../';
|
||||
|
||||
const MODULE_NAME = "Utilities";
|
||||
|
||||
|
||||
const DEBUG_MODULE = require('./Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
|
||||
let utilities = {
|
||||
pad: pad,
|
||||
createFolderIfNeeded: createFolderIfNeeded,
|
||||
calculatePresicion: calculatePresicion
|
||||
};
|
||||
|
||||
return utilities;
|
||||
|
||||
|
||||
function pad(str, max) {
|
||||
str = str.toString();
|
||||
return str.length < max ? pad("0" + str, max) : str;
|
||||
}
|
||||
|
||||
function createFolderIfNeeded(path, azureFileStorage, callBackFunction) {
|
||||
|
||||
let splittedPath = path.split("/");
|
||||
|
||||
let partialPath = '';
|
||||
let separator = "";
|
||||
|
||||
|
||||
let i = 0;
|
||||
loop();
|
||||
|
||||
function loop() {
|
||||
|
||||
partialPath = partialPath + separator + splittedPath[i];
|
||||
separator = "/";
|
||||
|
||||
i++;
|
||||
|
||||
azureFileStorage.createFolder(partialPath, checkLoop);
|
||||
|
||||
function checkLoop() {
|
||||
|
||||
if (i === splittedPath.length) {
|
||||
|
||||
callBackFunction();
|
||||
|
||||
} else {
|
||||
|
||||
loop();
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
function calculatePresicion(number) {
|
||||
|
||||
for (let i = -10; i <= 10; i++) {
|
||||
if (number < Math.pow(10, i)) {
|
||||
return Math.pow(10, i - 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"bot": {
|
||||
"name": "Olivia",
|
||||
"type": "Indicator",
|
||||
"version": "1.0.0",
|
||||
"devTeam": "AA Masters"
|
||||
},
|
||||
"processes": [
|
||||
{
|
||||
"name": "Multi-Period-Market",
|
||||
"description": "Produces different files with multi-period candles and volumes for entire markets.",
|
||||
"startMode": {
|
||||
"allMonths": {
|
||||
"run": "true",
|
||||
"minYear": "2014",
|
||||
"maxYear": "2018"
|
||||
},
|
||||
"oneMonth": {
|
||||
"run": "false",
|
||||
"year": "2018",
|
||||
"month": "2"
|
||||
},
|
||||
"noTime": {
|
||||
"run": "false"
|
||||
}
|
||||
},
|
||||
"intervalLapse": 3600000,
|
||||
"intervalMaxLoops": -1
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user