diff --git a/.vs/AAOlivia/v15/.suo b/.vs/AAOlivia/v15/.suo index 895082a62..45232f6e9 100644 Binary files a/.vs/AAOlivia/v15/.suo and b/.vs/AAOlivia/v15/.suo differ diff --git a/24-hs-Market/Interval Executor.js b/24-hs-Market/Interval Executor.js deleted file mode 100644 index 1ad59f305..000000000 --- a/24-hs-Market/Interval Executor.js +++ /dev/null @@ -1,169 +0,0 @@ -exports.newIntervalExecutor = function newIntervalExecutor(BOT) { - - let bot = BOT; - const ROOT_DIR = '../'; - - const INTERVAL_LAPSE = 60 * 60 * 1000; - - const MODULE_NAME = "Interval Executor"; - const LOG_INFO = true; - - const INTERVAL_MODULE = require('./Interval'); - - const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log'); - const logger = DEBUG_MODULE.newDebugLog(); - logger.fileName = MODULE_NAME; - logger.bot = bot; - - const INTERVAL_MAX_LOOPS = -1; // -1 for infinite loop. - - poloniexHoleFixing = { - initialize: initialize, - start: start - }; - - const SHUTDOWN_EVENT = require(ROOT_DIR + 'Azure Web Job Shutdown'); - let shutdownEvent = SHUTDOWN_EVENT.newShutdownEvent(bot); - - const UTILITIES = require(ROOT_DIR + 'Utilities'); - let utilities = UTILITIES.newUtilities(bot); - - let year; - let month; - - return poloniexHoleFixing; - - function initialize(yearAssigend, monthAssigned, callBackFunction) { - - try { - - year = yearAssigend; - month = monthAssigned; - - month = utilities.pad(month, 2); // Adding a left zero when needed. - - logger.fileName = MODULE_NAME + "-" + year + "-" + month; - - const logText = "[INFO] initialize - Entering function 'initialize' " + " @ " + year + "-" + month; - console.log(logText); - logger.write(logText); - - callBackFunction(); - - } catch (err) { - - const logText = "[ERROR] initialize - ' ERROR : " + err.message; - console.log(logText); - logger.write(logText); - - } - } - - - function start() { - - try { - - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'start', with year = " + year + " and month = " + month); - } - - let intervalId; - - if (INTERVAL_MAX_LOOPS === 0) { - - intervalId = setInterval(startNewInterval, INTERVAL_LAPSE); - - } - - let intervalLoopCounter = 0; - - startNewInterval(); // First, inmadiate run. - - function startNewInterval() { - - try { - - if (LOG_INFO === true) { - - if (INTERVAL_MAX_LOOPS !== 0) { - - logger.write("[INFO] Entering function 'startNewInterval', next execution inmediately after the current Interval finishes."); - - } else { - - logger.write("[INFO] Entering function 'startNewInterval', next execution in " + INTERVAL_LAPSE / 1000 / 60 + " minutes."); - - } - - } - - - if (shutdownEvent.isShuttingDown() === false) { - - let fileProcessingInterval = INTERVAL_MODULE.newInterval(bot); - - fileProcessingInterval.initialize(year, month, onReadyToStart); - - function onReadyToStart() { - - fileProcessingInterval.start(loopControl); - - } - - } - else { - - clearInterval(intervalId); - - const logText = "[INFO] requestNewData - Terminating Set Interval - About to exit gracefully the process. "; - console.log(logText); - logger.write(logText); - - } - - } - catch (err) { - const logText = "[ERROR] 'startNewInterval' - ERROR : " + err.message; - logger.write(logText); - } - } - - function loopControl(moreIntervalExecutionsNeeded) { - - if (moreIntervalExecutionsNeeded === true) { - - intervalLoopCounter++; - - if (intervalLoopCounter === INTERVAL_MAX_LOOPS) { - - const logText = "[WARN] 'INTERVAL_MAX_LOOPS' reached. Stopping execution. "; - logger.write(logText); - - } else { - - const logText = "[WARN] 'loopControl'. Staring new Interval. "; - logger.write(logText); - - setTimeout(startNewInterval, INTERVAL_LAPSE);// Looped Execution. - - } - - } else { - - logger.write("[WARN] Received the signal from last Interval execution to not call it anymore. Stopping execution. "); - - } - - } - - } - catch (err) { - const logText = "[ERROR] 'Start' - ERROR : " + err.message; - logger.write(logText); - } - } -}; - - - diff --git a/24-hs-Market/Interval.js b/24-hs-Market/Interval.js deleted file mode 100644 index 15bc63d5a..000000000 --- a/24-hs-Market/Interval.js +++ /dev/null @@ -1,895 +0,0 @@ -exports.newInterval = function newInterval(BOT) { - - let bot = BOT; - const ROOT_DIR = '../'; - const GMT_SECONDS = ':00.000 GMT+0000'; - const GMT_MILI_SECONDS = '.000 GMT+0000'; - const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; - - const MODULE_NAME = "Interval"; - const LOG_INFO = true; - - const EXCHANGE_NAME = "Poloniex"; - const EXCHANGE_ID = 1; - - const TRADES_FOLDER_NAME = "Trades"; - - const CANDLES_FOLDER_NAME = "Candles"; - const CANDLES_ONE_MIN = "One-Min"; - - const VOLUMES_FOLDER_NAME = "Volumes"; - const VOLUMES_ONE_MIN = "One-Min"; - - const GO_RANDOM = false; - const FORCE_MARKET = 2; // This allows to debug the execution of an specific market. Not intended for production. - - const MARKETS_MODULE = require(ROOT_DIR + 'Markets'); - - const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log'); - const logger = DEBUG_MODULE.newDebugLog(); - logger.fileName = MODULE_NAME; - logger.bot = bot; - - interval = { - initialize: initialize, - start: start - }; - - let markets; - - const AZURE_FILE_STORAGE = require(ROOT_DIR + 'Azure File Storage'); - let charlyAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot); - let bruceAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot); - let oliviaAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot); - - const UTILITIES = require(ROOT_DIR + 'Utilities'); - let utilities = UTILITIES.newUtilities(bot); - - let year; - let month; - - return interval; - - function initialize(yearAssigend, monthAssigned, callBackFunction) { - - try { - - year = yearAssigend; - month = monthAssigned; - - month = utilities.pad(month, 2); // Adding a left zero when needed. - - logger.fileName = MODULE_NAME + "-" + year + "-" + month; - - const logText = "[INFO] initialize - Entering function 'initialize' " + " @ " + year + "-" + month; - console.log(logText); - logger.write(logText); - - charlyAzureFileStorage.initialize("Charly"); - bruceAzureFileStorage.initialize("Bruce"); - oliviaAzureFileStorage.initialize("Olivia"); - - markets = MARKETS_MODULE.newMarkets(bot); - markets.initialize(callBackFunction); - - - } catch (err) { - - const logText = "[ERROR] initialize - ' ERROR : " + err.message; - console.log(logText); - logger.write(logText); - - } - } - -/* - -This process is going to do the following: - -Read the candles and volumes from Bruce and produce a single Index File for each market with daily candles and volumes. - -*/ - - function start(callBackFunction) { - - try { - - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'start', with year = " + year + " and month = " + month); - } - - let processDate = new Date(year + "-" + month + "-1 00:00:00.000 GMT+0000"); - - let lastMinuteOfMonth = new Date(year + "-" + month + "-1 00:00:00.000 GMT+0000"); - - lastMinuteOfMonth.setUTCMonth(lastMinuteOfMonth.getUTCMonth() + 1); // First we go 1 month into the future. - lastMinuteOfMonth.setUTCSeconds(lastMinuteOfMonth.getUTCSeconds() - 30); // Then we go back 30 seconds, or to the last minute of the original month. - - let thisDatetime = new Date(); - - if ((year === thisDatetime.getUTCFullYear() && parseInt(month) > thisDatetime.getUTCMonth() + 1) || year > thisDatetime.getUTCFullYear()) { - - logger.write("[INFO] We are too far in the future. Interval will not execute. Sorry."); - return; - - } - - let atHeadOfMarket; // This tell us if we are at the month which includes the head of the market according to current datetime. - - if ((year === thisDatetime.getUTCFullYear() && parseInt(month) === thisDatetime.getUTCMonth() + 1)) { - - atHeadOfMarket = true; - - } else { - - atHeadOfMarket = false; - - } - - let nextIntervalExecution = false; // This tell weather the Interval module will be executed again or not. By default it will not unless some hole have been found in the current execution. - - let marketQueue; // This is the queue of all markets to be procesesd at each interval. - let market = { // This is the current market being processed after removing it from the queue. - id: 0, - assetA: "", - assetB: "" - }; - - let lastCandleFile; // Datetime of the last file included on the Index File. - let firstTradeFile; // Datetime of the first trade file in the whole market history. - let maxCandleFile; // Datetime of the last file available to be included in the Index File. - - marketsLoop(); - - /* - - At every run, the process needs to loop through all the markets at this exchange. - The following functions marketsLoop(), openMarket(), closeMarket() and closeAndOpenMarket() controls the serialization of this processing. - - */ - - function marketsLoop() { - - try { - - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'marketsLoop'"); - } - - markets.getMarketsByExchange(EXCHANGE_ID, onMarketsReady); - - function onMarketsReady(marketsArray) { - - marketQueue = JSON.parse(marketsArray); - - openMarket(); // First execution and entering into the real loop. - - } - } - catch (err) { - const logText = "[ERROR] 'marketsLoop' - ERROR : " + err.message; - logger.write(logText); - } - } - - function openMarket() { - - // To open a Market means picking a new market from the queue. - - try { - - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'openMarket'"); - } - - - if (marketQueue.length === 0) { - - if (LOG_INFO === true) { - logger.write("[INFO] 'openMarket' - marketQueue.length === 0"); - } - - const logText = "[WARN] We processed all the markets."; - logger.write(logText); - - callBackFunction(nextIntervalExecution); - - return; - } - - if (GO_RANDOM === true) { - const index = parseInt(Math.random() * (marketQueue.length - 1)); - - market.id = marketQueue[index][0]; - market.assetA = marketQueue[index][1]; - market.assetB = marketQueue[index][2]; - market.status = marketQueue[index][3]; - - marketQueue.splice(index, 1); - } else { - let marketRecord = marketQueue.shift(); - - market.id = marketRecord[0]; - market.assetA = marketRecord[1]; - market.assetB = marketRecord[2]; - market.status = marketRecord[3]; - - if (FORCE_MARKET > 0) { - if (FORCE_MARKET !== market.id) { - closeAndOpenMarket(); - return; - } - } - } - - if (LOG_INFO === true) { - logger.write("[INFO] 'openMarket' - marketQueue.length = " + marketQueue.length); - logger.write("[INFO] 'openMarket' - market sucessfully opened : " + market.assetA + "_" + market.assetB); - } - - if (market.status === markets.ENABLED) { - - getStatusReport(); - - } else { - - logger.write("[INFO] 'openMarket' - market " + market.assetA + "_" + market.assetB + " skipped because its status is not valid. Status = " + market.status); - closeAndOpenMarket(); - return; - - } - - - } - catch (err) { - const logText = "[ERROR] 'openMarket' - ERROR : " + err.message; - logger.write(logText); - closeMarket(); - } - } - - function closeMarket() { - - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'closeMarket'"); - } - - } - - function closeAndOpenMarket() { - - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'closeAndOpenMarket'"); - } - - openMarket(); - } - - /* - - The following code executes for each market. - - */ - - function getStatusReport() { - - try { - - let reportFilePath; - let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json" - - getHistoricTrades(); - - function getHistoricTrades() { - - /* - - We need to know where is the begining of the market, since that will help us know where the Index Files should start. - - */ - - reportFilePath = EXCHANGE_NAME + "/Processes/" + "Historic-Trades"; - - charlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); - - function onStatusReportReceived(text) { - - let statusReport; - - try { - - statusReport = JSON.parse(text); - - firstTradeFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + statusReport.lastFile.hours + ":" + statusReport.lastFile.minutes + GMT_SECONDS); - - getOneMinDailyCandlesVolumes(); - - } catch (err) { - - const logText = "[INFO] 'getStatusReport' - Failed to read main Historic Trades Status Report for market " + market.assetA + '_' + market.assetB + " . Skipping it. "; - logger.write(logText); - - closeAndOpenMarket(); - } - } - } - - function getOneMinDailyCandlesVolumes() { - - /* If the process run and was interrupted, there should be a status report that allows us to resume execution. */ - - reportFilePath = EXCHANGE_NAME + "/Processes/" + "One-Min-Daily-Candles-Volumes" + "/" + year + "/" + month; - - bruceAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); - - function onStatusReportReceived(text) { - - let statusReport; - - try { - - statusReport = JSON.parse(text); - - maxCandleFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + "00:00" + GMT_SECONDS); - - getThisProcessReport(); - - } catch (err) { - - /* - - It might happen that the file content is corrupt or it does not exist. In either case we will point our lastCandleFile - to the last day of the previous month. - - */ - - lastCandleFile = new Date(processDate.valueOf() - ONE_DAY_IN_MILISECONDS); - findLastCandleCloseValue(); - - } - } - } - - function getThisProcessReport() { - - /* If the process run and was interrupted, there should be a status report that allows us to resume execution. */ - - reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process; - - oliviaAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); - - function onStatusReportReceived(text) { - - let statusReport; - - try { - - statusReport = JSON.parse(text); - - lastCandleFile = new Date(statusReport.lastFile.year + "-" + statusReport.lastFile.month + "-" + statusReport.lastFile.days + " " + "00:00" + GMT_SECONDS); - - findPreviousContent(); - - } catch (err) { - - /* - - It might happen that the file content is corrupt or it does not exist. In either case we will point our lastCandleFile - to the begining of the market. - - */ - - lastCandleFile = new Date(firstTradeFile.getUTCFullYear() + "-" + (firstTradeFile.getUTCMonth() + 1) + "-" + firstTradeFile.getUTCDate() + " " + "00:00" + GMT_SECONDS); - - lastCandleFile = new Date(lastCandleFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well. - - buildCandles(); - - } - } - } - - } - catch (err) { - const logText = "[ERROR] 'getStatusReport' - ERROR : " + err.message; - logger.write(logText); - closeMarket(); - } - } - - function findPreviousContent() { - - try { - - let previousCandles; - let previousVolumes; - - getCandles(); - - function getCandles() { - - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME; - - oliviaAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true); - - function onFileReceived(text) { - - let candlesFile; - - try { - - candlesFile = JSON.parse(text); - - previousCandles = candlesFile; - - getVolumes(); - - } catch (err) { - - const logText = "[ERR] 'findPreviousContent' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; - logger.write(logText); - - closeAndOpenMarket(); - } - } - } - - function getVolumes() { - - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME; - - oliviaAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true); - - function onFileReceived(text) { - - let volumesFile; - - try { - - volumesFile = JSON.parse(text); - - previousVolumes = volumesFile; - - buildCandles(previousCandles, previousVolumes); - - } catch (err) { - - const logText = "[ERR] 'findPreviousContent' - Empty or corrupt volume file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; - logger.write(logText); - - closeAndOpenMarket(); - } - } - } - } - catch (err) { - const logText = "[ERROR] 'findPreviousContent' - ERROR : " + err.message; - logger.write(logText); - closeMarket(); - } - - } - - function buildCandles(previousCandles, previousVolumes) { - - /* - - Here we are going to scan the candle and volume files to generate daily candles in only one Index File. - - */ - - let outputCandles = []; - let outputVolumes = []; - - try { - - if (previousCandles !== undefined) { - - for (let i = 0; i < previousCandles.length; i++) { - - let candle = { - open: previousCandles[i][2], - close: previousCandles[i][3], - min: previousCandles[i][0], - max: previousCandles[i][1], - begin: previousCandles[i][4], - end: previousCandles[i][5] - }; - - outputCandles.push(candle); - } - - } - - if (previousVolumes !== undefined) { - - for (let i = 0; i < previousVolumes.length; i++) { - - let volume = { - begin: previousVolumes[i][2], - end: previousVolumes[i][3], - buy: previousVolumes[i][0], - sell: previousVolumes[i][1] - }; - - outputVolumes.push(volume); - } - - } - - advanceTime(); - - function advanceTime() { - - lastCandleFile = new Date(lastCandleFile.valueOf() + ONE_DAY_IN_MILISECONDS); - - const logText = "[INFO] New processing time @ " + lastCandleFile.getUTCFullYear() + "/" + (lastCandleFile.getUTCMonth() + 1) + "/" + lastCandleFile.getUTCDate() + "."; - console.log(logText); - logger.write(logText); - - /* Validation that we are not going past the head of the market. */ - - if (lastCandleFile.valueOf() > maxCandleFile.valueOf()) { - - const logText = "[INFO] 'buildCandles' - Head of the market found @ " + lastCandleFile.getUTCFullYear() + "/" + (lastCandleFile.getUTCMonth() + 1) + "/" + lastCandleFile.getUTCDate() + "."; - logger.write(logText); - - closeAndOpenMarket(); - - return; - - } - - nextCandleFile(); - - } - - function nextCandleFile() { - - let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2); - let fileName = market.assetA + '_' + market.assetB + ".json" - let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + '/' + CANDLES_ONE_MIN + '/' + dateForPath; - - bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true); - - function onFileReceived(text) { - - let candlesFile; - - try { - - candlesFile = JSON.parse(text); - - } catch (err) { - - const logText = "[ERR] 'buildCandles' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; - logger.write(logText); - - closeAndOpenMarket(); - - return; - } - - let outputCandle = { - open: 0, - close: 0, - min: 0, - max: 0, - begin: 0, - end: 0 - }; - - if (candlesFile.length > 0) { - - outputCandle.open = candlesFile[0][2]; - outputCandle.close = candlesFile[candlesFile.length - 1][3]; - - outputCandle.min = candlesFile[0][0]; - outputCandle.max = candlesFile[0][1]; - - outputCandle.begin = candlesFile[0][4]; - outputCandle.end = candlesFile[candlesFile.length - 1][5]; - - } else { - - const logText = "[ERR] 'buildCandles' - No candles found at file " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; - logger.write(logText); - - closeAndOpenMarket(); - - return; - - } - - for (let i = 0; i < candlesFile.length; i++) { - - let candle = { - open: candlesFile[i][2], - close: candlesFile[i][3], - min: candlesFile[i][0], - max: candlesFile[i][1], - begin: candlesFile[i][4], - end: candlesFile[i][5] - }; - - if (candle.min < outputCandle.min) { - - outputCandle.min = candle.min; - - } - - if (candle.max > outputCandle.max) { - - outputCandle.max = candle.max; - - } - - } - - outputCandles.push(outputCandle); - - nextVolumeFile(); - - } - } - - function nextVolumeFile() { - - let dateForPath = lastCandleFile.getUTCFullYear() + '/' + utilities.pad(lastCandleFile.getUTCMonth() + 1, 2) + '/' + utilities.pad(lastCandleFile.getUTCDate(), 2); - let fileName = market.assetA + '_' + market.assetB + ".json" - let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + '/' + VOLUMES_ONE_MIN + '/' + dateForPath; - - bruceAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true); - - function onFileReceived(text) { - - let volumesFile; - - try { - - volumesFile = JSON.parse(text); - - } catch (err) { - - const logText = "[ERR] 'buildCandles' - Empty or corrupt candle file found at " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; - logger.write(logText); - - closeAndOpenMarket(); - - return; - } - - let outputVolume = { - buy: 0, - sell: 0, - begin: 0, - end: 0 - }; - - if (volumesFile.length > 0) { - - outputVolume.begin = volumesFile[0][2]; - outputVolume.end = volumesFile[volumesFile.length - 1][3]; - - } else { - - const logText = "[ERR] 'buildCandles' - No volumes found at file " + filePath + " for market " + market.assetA + '_' + market.assetB + " . Skipping this Market. "; - logger.write(logText); - - closeAndOpenMarket(); - - return; - - } - - for (let i = 0; i < volumesFile.length; i++) { - - let volume = { - buy: volumesFile[i][0], - sell: volumesFile[i][1], - begin: volumesFile[i][2], - end: volumesFile[i][3] - }; - - outputVolume.buy = outputVolume.buy + volume.buy; - outputVolume.sell = outputVolume.sell + volume.sell; - - } - - outputVolumes.push(outputVolume); - - writeFiles(lastCandleFile, outputCandles, outputVolumes, advanceTime); - } - } - } - - catch (err) { - const logText = "[ERROR] 'buildCandles' - ERROR : " + err.message; - logger.write(logText); - closeMarket(); - } - - } - - function writeFiles(date, candles, volumes, callBack) { - - /* - - Here we will write the contents of the Candles and Volumens files. - - */ - - try { - - writeCandles(); - - function writeCandles() { - - let separator = ""; - let fileRecordCounter = 0; - - let fileContent = ""; - - for (i = 0; i < candles.length; i++) { - - let candle = candles[i]; - - fileContent = fileContent + separator + '[' + candles[i].min + "," + candles[i].max + "," + candles[i].open + "," + candles[i].close + "," + candles[i].begin + "," + candles[i].end + "]"; - - if (separator === "") { separator = ","; } - - fileRecordCounter++; - - } - - fileContent = "[" + fileContent + "]"; - - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePath = EXCHANGE_NAME + "/Output/" + CANDLES_FOLDER_NAME + "/" + bot.process; - - utilities.createFolderIfNeeded(filePath, oliviaAzureFileStorage, onFolderCreated); - - function onFolderCreated() { - - oliviaAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); - - function onFileCreated() { - - const logText = "[WARN] Finished with File @ " + market.assetA + "_" + market.assetB + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName + ""; - console.log(logText); - logger.write(logText); - - writeVolumes(); - } - } - - } - - function writeVolumes() { - - let separator = ""; - let fileRecordCounter = 0; - - let fileContent = ""; - - for (i = 0; i < volumes.length; i++) { - - let candle = volumes[i]; - - fileContent = fileContent + separator + '[' + volumes[i].buy + "," + volumes[i].sell + "," + volumes[i].begin + "," + volumes[i].end + "]"; - - if (separator === "") { separator = ","; } - - fileRecordCounter++; - - } - - fileContent = "[" + fileContent + "]"; - - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePath = EXCHANGE_NAME + "/Output/" + VOLUMES_FOLDER_NAME + "/" + bot.process; - - utilities.createFolderIfNeeded(filePath, oliviaAzureFileStorage, onFolderCreated); - - function onFolderCreated() { - - oliviaAzureFileStorage.createTextFile(filePath, fileName, fileContent + '\n', onFileCreated); - - function onFileCreated() { - - const logText = "[WARN] Finished with File @ " + market.assetA + "_" + market.assetB + ", " + fileRecordCounter + " records inserted into " + filePath + "/" + fileName + ""; - console.log(logText); - logger.write(logText); - - writeReport(); - } - } - } - - function writeReport() { - - writeStatusReport(date, onStatusReportWritten); - - function onStatusReportWritten() { - - callBack(); - - } - } - } - - catch (err) { - const logText = "[ERROR] 'writeFiles' - ERROR : " + err.message; - logger.write(logText); - closeMarket(); - } - } - - function writeStatusReport(lastFileDate, callBack) { - - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'writeStatusReport'"); - } - - try { - - let reportFilePath = EXCHANGE_NAME + "/Processes/" + bot.process; - - utilities.createFolderIfNeeded(reportFilePath, oliviaAzureFileStorage, onFolderCreated); - - function onFolderCreated() { - - try { - - let fileName = "Status.Report." + market.assetA + '_' + market.assetB + ".json"; - - let report = { - lastFile: { - year: lastFileDate.getUTCFullYear(), - month: (lastFileDate.getUTCMonth() + 1), - days: lastFileDate.getUTCDate() - } - }; - - let fileContent = JSON.stringify(report); - - oliviaAzureFileStorage.createTextFile(reportFilePath, fileName, fileContent + '\n', onFileCreated); - - function onFileCreated() { - - if (LOG_INFO === true) { - logger.write("[INFO] 'writeStatusReport' - Content written: " + fileContent); - } - - callBack(); - } - } - catch (err) { - const logText = "[ERROR] 'writeStatusReport - onFolderCreated' - ERROR : " + err.message; - logger.write(logText); - closeMarket(); - } - } - - } - catch (err) { - const logText = "[ERROR] 'writeStatusReport' - ERROR : " + err.message; - logger.write(logText); - closeMarket(); - } - - } - - } - catch (err) { - const logText = "[ERROR] 'Start' - ERROR : " + err.message; - logger.write(logText); - } - } -}; diff --git a/24-hs-Market/run.js b/24-hs-Market/run.js deleted file mode 100644 index cd137207d..000000000 --- a/24-hs-Market/run.js +++ /dev/null @@ -1,97 +0,0 @@ - -let bot = { - name: "Olivia", - process: "24-hs-Market", - type: "Indicator", - version: "1.0.0", - devTeam: "AA Masters" -}; - -const ROOT_DIR = '../'; - -const MODULE_NAME = "run"; - -const START_ALL_MONTHS = false; -const START_ONE_MONTH = true; - -const INTERVAL_EXECUTOR_MODULE = require('./Interval Executor'); - -const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log'); -const logger = DEBUG_MODULE.newDebugLog(); -logger.fileName = MODULE_NAME; -logger.bot = bot; - -process.on('uncaughtException', function (err) { - logger.write('uncaughtException - ' + err.message); -}); - - -process.on('unhandledRejection', (reason, p) => { - logger.write("Unhandled Rejection at: Promise " + JSON.stringify(p) + " reason: " + reason); -}); - - -process.on('exit', function (code) { - logger.write('About to exit with code:' + code); -}); - - -try { - if (START_ALL_MONTHS === true) { - - for (let year = 2018; year > 2014; year--) { - - for (let month = 12; month > 0; month--) { - - let timeDelay = Math.random() * 10 * 1000; // We introduce a short delay so as to not overload the machine. - - setTimeout(startProcess, timeDelay); - - function startProcess() { - - const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot); - - newIntervalExecutor.initialize(year, month, onInitializeReady); - - function onInitializeReady() { - - newIntervalExecutor.start(); - - } - - } - - } - - } - - } - - - if (START_ONE_MONTH === true) { - - startProcess(); - - function startProcess() { - - const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot); - - let month = 2; - let year = 2018; - - newIntervalExecutor.initialize(year, month, onInitializeReady); - - function onInitializeReady() { - - newIntervalExecutor.start(); - - } - } - } - - -} catch (err) { - console.log(err.message); - logger.write(err.message); -} - diff --git a/AAOlivia.njsproj b/AAOlivia.njsproj index 673ff4c62..e497b5ee8 100644 --- a/AAOlivia.njsproj +++ b/AAOlivia.njsproj @@ -25,16 +25,11 @@ - - - - - - + diff --git a/Azure File Storage.js b/Azure File Storage.js deleted file mode 100644 index dcb2f6c89..000000000 --- a/Azure File Storage.js +++ /dev/null @@ -1,247 +0,0 @@ - -exports.newAzureFileStorage = function newAzureFileStorage(BOT) { - - let bot = BOT; - const ROOT_DIR = '../'; - - const MODULE_NAME = "Azure File Storage"; - - var fs = require('fs'); - var util = require('util'); - var guid = require('node-uuid'); - var crypto = require('crypto'); - var storage = require('azure-storage'); - - const DEBUG_MODULE = require('./Debug Log'); - const logger = DEBUG_MODULE.newDebugLog(); - logger.fileName = MODULE_NAME; - logger.bot = bot; - - const shareName = 'data'; - let dataOwner; - - let azureFileStorage = { - initialize: initialize, - createFolder: createFolder, - createTextFile: createTextFile, - getTextFile: getTextFile, - listFilesAndFolders: listFilesAndFolders - }; - - let fileService; - - return azureFileStorage; - - - function initialize(dataOwnerBotName) { - - try { - - if (dataOwnerBotName === undefined) { - - dataOwner = bot.name; - - } else { - - dataOwner = dataOwnerBotName; - - } - - fileService = storage.createFileService(readConfig().connectionString); - - } - catch (err) { - const logText = "[ERROR] 'initialize' - ERROR : " + err.message; - logger.write(logText); - } - } - - function readConfig() { - - try { - - return JSON.parse(fs.readFileSync(ROOT_DIR + dataOwner + '.azure.storage.connstring', 'utf8')); - - } - catch (err) { - const logText = "[ERROR] 'readConfig' - ERROR : " + err.message; - logger.write(logText); - } - - } - - function createFolder(folderPath, callBackFunction) { - - try { - - fileService.createDirectoryIfNotExists(shareName, folderPath, onFolderCreated); - - function onFolderCreated(err) { - - if (err) { - - const logText = "[ERROR] createFolder - onFolderCreated ' ERROR = " + err.message + " SHARE NAME = " + shareName + " FOLDER PATH = " + folderPath; - console.log(logText); - logger.write(logText); - - } - - callBackFunction(); - - } - - } - catch (err) { - const logText = "[ERROR] 'createFolder' - ERROR : " + err.message; - logger.write(logText); - } - - } - - function createTextFile(folderPath, fileName, fileContent, callBackFunction) { - - try { - - fileService.createFileFromText(shareName, folderPath, fileName, fileContent, onFileCreated); - - function onFileCreated(err, result, response) { - - if (err) { - - const logText = "[WARN] createTextFile - onFileCreated ' ERROR = " + err.message + " SHARE NAME = " + shareName + " FOLDER PATH = " + folderPath + " FILE NAME = " + fileName; - console.log(logText); - logger.write(logText); - - if (err.message.indexOf("The server is busy") > 0) { - - fileService.createFileFromText(shareName, folderPath, fileName, fileContent, onFileCreatedSecondTry); - - function onFileCreatedSecondTry(err) { - - if (err) { - - const logText = "[ERROR] createTextFile - onFileCreated - onFileCreatedSecondTry : File not created. Giving Up.' ERROR = " + err.message + " SHARE NAME = " + shareName + " FOLDER PATH = " + folderPath + " FILE NAME = " + fileName; - console.log(logText); - logger.write(logText); - - } else { - - const logText = "[INFO] createTextFile - onFileCreated - onFileCreatedSecondTry ' : File succesfully created on second try. SHARE NAME = " + shareName + " FOLDER PATH = " + folderPath + " FILE NAME = " + fileName; - console.log(logText); - logger.write(logText); - - } - } - } - } - - callBackFunction(); - } - - } - catch (err) { - const logText = "[ERROR] 'createTextFile' - ERROR : " + err.message; - logger.write(logText); - } - - } - - function getTextFile(folderPath, fileName, callBackFunction, ignoreNotFound) { - - try { - - fileService.getFileToText(shareName, folderPath, fileName, undefined, onFileReceived); - - function onFileReceived(err, text, response) { - - if (err) { - - if (ignoreNotFound === undefined) { - - const logText = "[ERROR] getTextFile - onFileReceived ' ERROR = " + err.message + " SHARE NAME = " + shareName + " FOLDER PATH = " + folderPath + " FILE NAME = " + fileName; - console.log(logText); - logger.write(logText); - - } - - text = undefined; - - } - - callBackFunction(text); - } - - } - catch (err) { - const logText = "[ERROR] 'getTextFile' - ERROR : " + err.message; - logger.write(logText); - } - - } - - function listFilesAndFolders(folderPath, callBackFunction) { - - try { - - let items = { - files: [], - folders: [] - }; - - /* - * @param { object } [options] The request options. - * @param { int } [options.maxResults] Specifies the maximum number of folders to return per call to Azure ServiceClient. - * This does NOT affect list size returned by this function. (maximum: 5000) - * @param { LocationMode } [options.locationMode] Specifies the location mode used to decide which location the request should be sent to. - * Please see StorageUtilities.LocationMode for the possible values. - * @param { int } [options.timeoutIntervalInMs] The server timeout interval, in milliseconds, to use for the request. - * @param { int } [options.maximumExecutionTimeInMs] The maximum execution time, in milliseconds, across all potential retries, to use when making this request. - * The maximum execution time interval begins at the time that the client begins building the request.The maximum - * execution time is checked intermittently while performing requests, and before executing retries. - * @param { string } [options.clientRequestId] A string that represents the client request ID with a 1KB character limit. - * @param { bool } [options.useNagleAlgorithm] Determines whether the Nagle al - */ - - let options = { - maxResults: 500 - }; - - fileService.listFilesAndDirectoriesSegmented(shareName, folderPath, null, options, onFilesAndFoldersReceived); - - - function onFilesAndFoldersReceived(error, result) { - - try { - - if (error) { - - const logText = "[ERROR] 'listFilesAndFolders - onFilesAndFoldersReceived' - ERROR RECEIVED : " + error; - logger.write(logText); - - callBackFunction(items); - - } else { - - items.files.push.apply(items.files, result.entries.files); - items.folders.push.apply(items.folders, result.entries.directories); - - callBackFunction(items); - - } - - } - catch (err) { - const logText = "[ERROR] 'listFilesAndFolders - onFilesAndFoldersReceived' - ERROR : " + err.message; - logger.write(logText); - } - - } - - } - catch (err) { - const logText = "[ERROR] 'listFilesAndFolders' - ERROR : " + err.message; - logger.write(logText); - } - } - -}; \ No newline at end of file diff --git a/Azure Web Job Shutdown.js b/Azure Web Job Shutdown.js deleted file mode 100644 index 22ed305de..000000000 --- a/Azure Web Job Shutdown.js +++ /dev/null @@ -1,41 +0,0 @@ - -exports.newShutdownEvent = function newShutdownEvent(BOT) { - - let bot = BOT; - const ROOT_DIR = '../'; - - const MODULE_NAME = "Azure Web Job Shutdown"; - - - const DEBUG_MODULE = require('./Debug Log'); - const logger = DEBUG_MODULE.newDebugLog(); - logger.fileName = MODULE_NAME; - logger.bot = bot; - - let shutdownEvent = { - isShuttingDown: isShuttingDown - }; - - return shutdownEvent; - - - function isShuttingDown() { - - let filePath = process.env.WEBJOBS_SHUTDOWN_FILE; - - let fs = require('fs'); - - if (fs.existsSync(filePath)) { - return true; - } - else - { - return false; - } - - } - - - - -} \ No newline at end of file diff --git a/Debug Log.js b/Debug Log.js deleted file mode 100644 index fc543e5d7..000000000 --- a/Debug Log.js +++ /dev/null @@ -1,144 +0,0 @@ - - - -exports.newDebugLog = function newDebugLog() { - - - const MODULE_NAME = "Debug Log"; - const fileSystem = require('fs'); - - const currentDate = new Date(); - const dateString = currentDate.getUTCFullYear() + '-' + pad(currentDate.getUTCMonth() + 1, 2) + '-' + pad(currentDate.getUTCDate(), 2) + '-' + pad(currentDate.getUTCHours(), 2) + '-' + pad(currentDate.getUTCMinutes(), 2); - const randomId = parseInt(Math.random() * 1000000); - - let fileNumber = 1; - let messageId = 0; - let firstCall = true; - let folderPath; - - debugLog = { - bot: undefined, - fileName: undefined, - write: write - }; - - - - return debugLog; - - - function createFolders() { - - try { - - folderPath = debugLog.bot.devTeam; - - createFolderSync(folderPath); - - folderPath = debugLog.bot.devTeam + "/" + debugLog.bot.type; - - createFolderSync(folderPath); - - folderPath = debugLog.bot.devTeam + "/" + debugLog.bot.type + "/" + debugLog.bot.name; - - createFolderSync(folderPath); - - folderPath = debugLog.bot.devTeam + "/" + debugLog.bot.type + "/" + debugLog.bot.name + "/" + debugLog.bot.version; - - createFolderSync(folderPath); - - folderPath = debugLog.bot.devTeam + "/" + debugLog.bot.type + "/" + debugLog.bot.name + "/" + debugLog.bot.version + "/" + debugLog.bot.process; - - createFolderSync(folderPath); - - firstCall = false; - - } - catch (err) { - console.log("Error trying to create the folders needed. Error: " + err); - } - - - } - - - function write(Message) { - - if (firstCall === true) { - - createFolders(); - - } - - let filePath = getCurrentLogFile(folderPath + "/" + dateString + "---" + randomId + "---", this.fileName); - - let newDate = new Date(); - newDate = newDate.toISOString(); - - messageId++; - - try { - - fileSystem.appendFileSync(filePath, '\r\n' + newDate + " " + messageId + " " + Message); - - } - catch (err) { - console.log("Error trying to log info into a file. File: " + filePath + " Error:" + err); - } - } - - - function createFolderSync(name) { - try { - fileSystem.mkdirSync( '../logs/' + name) - } catch (err) { - if (err.code !== 'EEXIST') throw err - } - } - - function getCurrentLogFile(relativePath, fileName) { - - let filePath; - let stats; - let i; - - try { - for (i = 1; i < 1000000; i++) { - - filePath = '../logs/' + relativePath + i + "." + fileName + '.log'; - stats = fileSystem.statSync(filePath); - } - } - catch (err) { - - if (i > 1) { - - filePath = '../logs/' + relativePath + (i - 1) + "." + fileName + '.log'; - - stats = fileSystem.statSync(filePath); - const fileSizeInBytes = stats.size; - - if (fileSizeInBytes > 10240000) { - - filePath = '../logs/' + relativePath + i + "." + fileName + '.log'; - } - - return filePath; - - } else { - - return filePath; - - } - - } - - } - - function pad(str, max) { - str = str.toString(); - return str.length < max ? pad("0" + str, max) : str; - } - -}; - diff --git a/Markets.js b/Markets.js deleted file mode 100644 index 54ba5512d..000000000 --- a/Markets.js +++ /dev/null @@ -1,252 +0,0 @@ -/* This business logic module is responsible for Markets and Assets. */ - - -exports.newMarkets = function newMarkets(BOT) { - - let bot = BOT; - const ROOT_DIR = '../'; - - const MODULE_NAME = "Markets"; - - const FOLDER_PATHS = 'data'; - const MARKETS_FOLDER_NAME = "other-files"; - - const DEBUG_MODULE = require('./Debug Log'); - const logger = DEBUG_MODULE.newDebugLog(); - logger.fileName = MODULE_NAME; - logger.bot = bot; - - const EXCHANGE_ID = 1; - const fileName = "Markets_" + EXCHANGE_ID + ".json"; - - const DATA_OWNER_BOT_NAME = "Carol" - - const markets = { - DISABLED: 0, - ENABLED: 1, - getMarketsByExchange: getMarketsByExchange, - getExistingOrNewMarketId: getExistingOrNewMarketId, - createNewAsset: createNewAsset, - createNewMarket: createNewMarket, - disableMarket: disableMarket, - initialize: initialize - }; - - //const dataLayer = DATA_LAYER_MODULE.newDataLayer(); - - let marketsArray; - - const AZURE_FILE_STORAGE = require('./Azure File Storage'); - let azureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot); - - return markets; - - - function initialize(callBackFunction) { - - azureFileStorage.initialize(DATA_OWNER_BOT_NAME); - - try { - - azureFileStorage.getTextFile(MARKETS_FOLDER_NAME, fileName, onFileRead); - - function onFileRead(text) { - - try { - - var fileString = text; - let cleanString = fileString; - - /* Dont know why but sometimes files comes with a first characater that is not supposed to be there and you can not see it at the file opened with notepad */ - - if (fileString[0] !== "[") { - - cleanString = ''; - - for (let i = 1; i < fileString.length; i++) { - cleanString = cleanString + fileString[i]; - } - } - - marketsArray = JSON.parse(cleanString); - callBackFunction(); - - } - catch (err) { - const logText = "[ERROR] initialize - onFileRead ' ERROR : " + err.message + " File: " + fileName; - console.log(logText); - logger.write(logText); - } - - } - } - catch (err) { - const logText = "[ERROR] initialize - ' ERROR : " + err.message + " File Not Found: " + fileName; - console.log(logText); - logger.write(logText); - } - - - } - - - - function getMarketsByExchange(exchangeId, callBackFunction) { - - try { - - // TODO: Currently the exchange Id is ignored as the class is still working with only one exchange. We must fix this. - - callBackFunction(JSON.stringify(marketsArray)); - - - } catch (err) { - - const logText = "[ERROR] getMarketsByExchange - ' ERROR : " + err.message; - console.log(logText); - logger.write(logText); - - } - - } - - - function getExistingOrNewMarketId(marketName, market, exchangeId, callBackFunction) { - - // TODO: At this function is missing the following: If a market does not exists, it must be greated, and Id generated and returned. We automatically discover new markets when this function is called with a market that didn-t previuously existed. - - try { - - for (let i = 0; i < marketsArray.length; i++) { - - if (marketsArray[i][1] + "_" + marketsArray[i][2] === marketName) { - - callBackFunction(marketName, market, marketsArray[i][0]); - return; - - } - } - - - } catch (err) { - - const logText = "[ERROR] getExistingOrNewMarketId - ' ERROR : " + err.message + " marketId = " + marketId + " idAssetA = " + idAssetA + " idAssetB = " + idAssetB; - console.log(logText); - logger.write(logText); - - } - - } - - - function createNewAsset(assetCode, callBackFunction) { - - // TODO: Upgrade this function in order to update the file containing all assets, not the database that will be deprecated. - - try { - - dataLayer.newAsset(assetCode, onIdReady); - - function onIdReady(resultSet) { - - try { - - callBackFunction(resultSet); - resultSet = []; - - } catch (err) { - - const logText = "[ERROR] createNewAsset - onIdReady ' ERROR : " + err.message; - console.log(logText); - logger.write(logText); - - } - } - - } catch (err) { - - const logText = "[ERROR] createNewAsset - ' ERROR : " + err.message; - console.log(logText); - logger.write(logText); - - } - - } - - - - function createNewMarket(exchangeId, idAssetA, idAssetB, callBackFunction) { - - // TODO: Upgrade this function in order to update the file containing all markets, not the database that will be deprecated. - - try { - - dataLayer.newMarket(exchangeId, idAssetA, idAssetB, onIdReady); - - function onIdReady(resultSet) { - - try { - - callBackFunction(resultSet); - - } catch (err) { - - const logText = "[ERROR] createNewMarket - onIdReady ' ERROR : " + err.message; - console.log(logText); - logger.write(logText); - } - } - - } catch (err) { - - const logText = "[ERROR] createNewMarket - ' ERROR : " + err.message; - console.log(logText); - logger.write(logText); - } - } - - - function disableMarket(exchangeId, marketId, callBackFunction) { - - // This function puts the requested market in a DISABLED status. - - // TODO: Currently the exchangeId is ignored. - - try { - - for (let i = 0; i < marketsArray.length; i++) { - - if (marketsArray[i][0] === marketId) { - - marketsArray[i][3] = markets.DISABLED; - break; - - } - - } - - azureFileStorage.createTextFile(MARKETS_FOLDER_NAME, fileName, JSON.stringify(marketsArray), onFileReplaced); - - function onFileReplaced() { - - try { - - callBackFunction(); - - } catch (err) { - - const logText = "[ERROR] disableMarket - onFileReplaced ' ERROR : " + err.message; - console.log(logText); - logger.write(logText); - } - } - - } catch (err) { - - const logText = "[ERROR] disableMarket - ' ERROR : " + err.message; - console.log(logText); - logger.write(logText); - } - } - -}; \ No newline at end of file diff --git a/Multi-Period-Market/Interval Executor.js b/Multi-Period-Market/Interval Executor.js deleted file mode 100644 index baf531dcc..000000000 --- a/Multi-Period-Market/Interval Executor.js +++ /dev/null @@ -1,169 +0,0 @@ -exports.newIntervalExecutor = function newIntervalExecutor(BOT) { - - let bot = BOT; - const ROOT_DIR = '../'; - - const INTERVAL_LAPSE = 60 * 60 * 1000; // There is no need for more frecuency since 1 hour files are generated at maximun. - - const MODULE_NAME = "Interval Executor"; - const LOG_INFO = true; - - const INTERVAL_MODULE = require('./Interval'); - - const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log'); - const logger = DEBUG_MODULE.newDebugLog(); - logger.fileName = MODULE_NAME; - logger.bot = bot; - - const INTERVAL_MAX_LOOPS = -1; // -1 for infinite loop. - - poloniexHoleFixing = { - initialize: initialize, - start: start - }; - - const SHUTDOWN_EVENT = require(ROOT_DIR + 'Azure Web Job Shutdown'); - let shutdownEvent = SHUTDOWN_EVENT.newShutdownEvent(bot); - - const UTILITIES = require(ROOT_DIR + 'Utilities'); - let utilities = UTILITIES.newUtilities(bot); - - let year; - let month; - - return poloniexHoleFixing; - - function initialize(yearAssigend, monthAssigned, callBackFunction) { - - try { - - year = yearAssigend; - month = monthAssigned; - - month = utilities.pad(month, 2); // Adding a left zero when needed. - - logger.fileName = MODULE_NAME + "-" + year + "-" + month; - - const logText = "[INFO] initialize - Entering function 'initialize' " + " @ " + year + "-" + month; - console.log(logText); - logger.write(logText); - - callBackFunction(); - - } catch (err) { - - const logText = "[ERROR] initialize - ' ERROR : " + err.message; - console.log(logText); - logger.write(logText); - - } - } - - - function start() { - - try { - - if (LOG_INFO === true) { - logger.write("[INFO] Entering function 'start', with year = " + year + " and month = " + month); - } - - let intervalId; - - if (INTERVAL_MAX_LOOPS === 0) { - - intervalId = setInterval(startNewInterval, INTERVAL_LAPSE); - - } - - let intervalLoopCounter = 0; - - startNewInterval(); // First, inmadiate run. - - function startNewInterval() { - - try { - - if (LOG_INFO === true) { - - if (INTERVAL_MAX_LOOPS !== 0) { - - logger.write("[INFO] Entering function 'startNewInterval', next execution inmediately after the current Interval finishes."); - - } else { - - logger.write("[INFO] Entering function 'startNewInterval', next execution in " + INTERVAL_LAPSE / 1000 / 60 + " minutes."); - - } - - } - - - if (shutdownEvent.isShuttingDown() === false) { - - let fileProcessingInterval = INTERVAL_MODULE.newInterval(bot); - - fileProcessingInterval.initialize(year, month, onReadyToStart); - - function onReadyToStart() { - - fileProcessingInterval.start(loopControl); - - } - - } - else { - - clearInterval(intervalId); - - const logText = "[INFO] requestNewData - Terminating Set Interval - About to exit gracefully the process. "; - console.log(logText); - logger.write(logText); - - } - - } - catch (err) { - const logText = "[ERROR] 'startNewInterval' - ERROR : " + err.message; - logger.write(logText); - } - } - - function loopControl(moreIntervalExecutionsNeeded) { - - if (moreIntervalExecutionsNeeded === true) { - - intervalLoopCounter++; - - if (intervalLoopCounter === INTERVAL_MAX_LOOPS) { - - const logText = "[WARN] 'INTERVAL_MAX_LOOPS' reached. Stopping execution. "; - logger.write(logText); - - } else { - - const logText = "[WARN] 'loopControl'. Staring new Interval. "; - logger.write(logText); - - setTimeout(startNewInterval, INTERVAL_LAPSE);// Looped Execution. - - } - - } else { - - logger.write("[WARN] Received the signal from last Interval execution to not call it anymore. Stopping execution. "); - - } - - } - - } - catch (err) { - const logText = "[ERROR] 'Start' - ERROR : " + err.message; - logger.write(logText); - } - } -}; - - - diff --git a/Multi-Period-Market/Interval.js b/Multi-Period-Market/Interval.js index 208b6e9a2..466eaeae6 100644 --- a/Multi-Period-Market/Interval.js +++ b/Multi-Period-Market/Interval.js @@ -1,7 +1,7 @@ -exports.newInterval = function newInterval(BOT) { +exports.newInterval = function newInterval(BOT, UTILITIES, AZURE_FILE_STORAGE, DEBUG_MODULE, MARKETS_MODULE, POLONIEX_CLIENT_MODULE) { let bot = BOT; - const ROOT_DIR = '../'; + const GMT_SECONDS = ':00.000 GMT+0000'; const GMT_MILI_SECONDS = '.000 GMT+0000'; const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; @@ -23,9 +23,6 @@ const GO_RANDOM = false; const FORCE_MARKET = 2; // This allows to debug the execution of an specific market. Not intended for production. - const MARKETS_MODULE = require(ROOT_DIR + 'Markets'); - - const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log'); const logger = DEBUG_MODULE.newDebugLog(); logger.fileName = MODULE_NAME; logger.bot = bot; @@ -37,12 +34,10 @@ let markets; - const AZURE_FILE_STORAGE = require(ROOT_DIR + 'Azure File Storage'); let charlyAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot); let bruceAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot); let oliviaAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot); - const UTILITIES = require(ROOT_DIR + 'Utilities'); let utilities = UTILITIES.newUtilities(bot); let year; @@ -297,7 +292,7 @@ Read the candles and volumes from Bruce and produce a single Index File for each */ - reportFilePath = EXCHANGE_NAME + "/Processes/" + "Historic-Trades"; + reportFilePath = EXCHANGE_NAME + "/Processes/" + "Poloniex-Historic-Trades"; charlyAzureFileStorage.getTextFile(reportFilePath, fileName, onStatusReportReceived, true); diff --git a/Multi-Period-Market/run.js b/Multi-Period-Market/run.js deleted file mode 100644 index 0161b0f9a..000000000 --- a/Multi-Period-Market/run.js +++ /dev/null @@ -1,97 +0,0 @@ - -let bot = { - name: "Olivia", - process: "Multi-Period-Market", - type: "Indicator", - version: "1.0.0", - devTeam: "AA Masters" -}; - -const ROOT_DIR = '../'; - -const MODULE_NAME = "run"; - -const START_ALL_MONTHS = false; -const START_ONE_MONTH = true; - -const INTERVAL_EXECUTOR_MODULE = require('./Interval Executor'); - -const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log'); -const logger = DEBUG_MODULE.newDebugLog(); -logger.fileName = MODULE_NAME; -logger.bot = bot; - -process.on('uncaughtException', function (err) { - logger.write('uncaughtException - ' + err.message); -}); - - -process.on('unhandledRejection', (reason, p) => { - logger.write("Unhandled Rejection at: Promise " + JSON.stringify(p) + " reason: " + reason); -}); - - -process.on('exit', function (code) { - logger.write('About to exit with code:' + code); -}); - - -try { - if (START_ALL_MONTHS === true) { - - for (let year = 2018; year > 2014; year--) { - - for (let month = 12; month > 0; month--) { - - let timeDelay = Math.random() * 10 * 1000; // We introduce a short delay so as to not overload the machine. - - setTimeout(startProcess, timeDelay); - - function startProcess() { - - const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot); - - newIntervalExecutor.initialize(year, month, onInitializeReady); - - function onInitializeReady() { - - newIntervalExecutor.start(); - - } - - } - - } - - } - - } - - - if (START_ONE_MONTH === true) { - - startProcess(); - - function startProcess() { - - const newIntervalExecutor = INTERVAL_EXECUTOR_MODULE.newIntervalExecutor(bot); - - let month = 2; - let year = 2018; - - newIntervalExecutor.initialize(year, month, onInitializeReady); - - function onInitializeReady() { - - newIntervalExecutor.start(); - - } - } - } - - -} catch (err) { - console.log(err.message); - logger.write(err.message); -} - diff --git a/Poloniex API Client.js b/Poloniex API Client.js deleted file mode 100644 index d7e74a657..000000000 --- a/Poloniex API Client.js +++ /dev/null @@ -1,406 +0,0 @@ -module.exports = (function () { - 'use strict'; - - // Module dependencies - var crypto = require('crypto'), - request = require('request'), - nonce = require('nonce')(); - - // Constants - var version = '0.0.8', - PUBLIC_API_URL = 'https://poloniex.com/public', - PRIVATE_API_URL = 'https://poloniex.com/tradingApi', - USER_AGENT = 'poloniex.js ' + version; - //USER_AGENT = 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0' - - // Helper methods - function joinCurrencies(currencyA, currencyB) { - // If only one arg, then return the first - if (typeof currencyB !== 'string') { - return currencyA; - } - - return currencyA + '_' + currencyB; - } - - // Constructor - function Poloniex(key, secret) { - // Generate headers signed by this user's key and secret. - // The secret is encapsulated and never exposed - this._getPrivateHeaders = function (parameters) { - var paramString, signature; - - if (!key || !secret) { - throw 'Poloniex: Error. API key and secret required'; - } - - // Convert to `arg1=foo&arg2=bar` - paramString = Object.keys(parameters).map(function (param) { - return encodeURIComponent(param) + '=' + encodeURIComponent(parameters[param]); - }).join('&'); - - signature = crypto.createHmac('sha512', secret).update(paramString).digest('hex'); - - return { - Key: key, - Sign: signature - }; - }; - } - - // Currently, this fails with `Error: CERT_UNTRUSTED` - // Poloniex.STRICT_SSL can be set to `false` to avoid this. Use with caution. - // Will be removed in future, once this is resolved. - Poloniex.STRICT_SSL = true; - - // Customisable user agent string - Poloniex.USER_AGENT = USER_AGENT; - - // Prototype - Poloniex.prototype = { - constructor: Poloniex, - - // Make an API request - _request: function (options, callback) { - if (!('headers' in options)) { - options.headers = {}; - } - - options.json = true; - options.headers['User-Agent'] = Poloniex.USER_AGENT; - options.strictSSL = Poloniex.STRICT_SSL; - - request(options, function (err, response, body) { - // Empty response - if (!err && (typeof body === 'undefined' || body === null)) { - err = 'Empty response'; - } - - callback(err, body); - }); - - return this; - }, - - // Make a public API request - _public: function (command, parameters, callback) { - var options; - - if (typeof parameters === 'function') { - callback = parameters; - parameters = {}; - } - - parameters || (parameters = {}); - parameters.command = command; - options = { - method: 'GET', - url: PUBLIC_API_URL, - qs: parameters - }; - - options.qs.command = command; - return this._request(options, callback); - }, - - // Make a private API request - _private: function (command, parameters, callback) { - var options; - - if (typeof parameters === 'function') { - callback = parameters; - parameters = {}; - } - - parameters || (parameters = {}); - parameters.command = command; - parameters.nonce = nonce(); - - options = { - method: 'POST', - url: PRIVATE_API_URL, - form: parameters, - headers: this._getPrivateHeaders(parameters) - }; - - return this._request(options, callback); - }, - - ///// - // PUBLIC METHODS - - returnTicker: function (callback) { - return this._public('returnTicker', callback); - }, - - return24hVolume: function (callback) { - return this._public('return24hVolume', callback); - }, - - returnOrderBook: function (currencyA, currencyB, depth, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB), - depth: depth - }; - - return this._public('returnOrderBook', parameters, callback); - }, - - returnOrderBooks: function (depth, callback) { - var parameters = { - currencyPair: 'all', - depth: depth - }; - - return this._public('returnOrderBook', parameters, callback); - }, - - returnPublicTradeHistory: function (currencyA, currencyB, start, end, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB), - start: start, - end: end - }; - - return this._public('returnTradeHistory', parameters, callback); - }, - - returnChartData: function (currencyA, currencyB, period, start, end, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB), - period: period, - start: start, - end: end - }; - - return this._public('returnChartData', parameters, callback); - }, - - returnCurrencies: function (callback) { - return this._public('returnCurrencies', callback); - }, - - returnLoanOrders: function (currency, callback) { - return this._public('returnLoanOrders', { currency: currency }, callback); - }, - - ///// - // PRIVATE METHODS - - returnBalances: function (callback) { - return this._private('returnBalances', {}, callback); - }, - - returnCompleteBalances: function (callback) { - return this._private('returnCompleteBalances', {}, callback); - }, - - returnDepositAddresses: function (callback) { - return this._private('returnDepositAddresses', {}, callback); - }, - - generateNewAddress: function (currency, callback) { - return this._private('returnDepositsWithdrawals', { currency: currency }, callback); - }, - - returnDepositsWithdrawals: function (start, end, callback) { - return this._private('returnDepositsWithdrawals', { start: start, end: end }, callback); - }, - - returnOpenOrders: function (currencyA, currencyB, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB) - }; - - return this._private('returnOpenOrders', parameters, callback); - }, - - returnTradeHistory: function (currencyA, currencyB, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB) - }; - - return this._private('returnTradeHistory', parameters, callback); - }, - - returnOrderTrades: function (orderNumber, callback) { - var parameters = { - orderNumber: orderNumber - }; - - return this._private('returnOrderTrades', parameters, callback); - }, - - buy: function (currencyA, currencyB, rate, amount, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB), - rate: rate, - amount: amount - }; - - return this._private('buy', parameters, callback); - }, - - sell: function (currencyA, currencyB, rate, amount, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB), - rate: rate, - amount: amount - }; - - return this._private('sell', parameters, callback); - }, - - cancelOrder: function (currencyA, currencyB, orderNumber, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB), - orderNumber: orderNumber - }; - - return this._private('cancelOrder', parameters, callback); - }, - - moveOrder: function (orderNumber, rate, amount, callback) { - var parameters = { - orderNumber: orderNumber, - rate: rate, - amount: amount ? amount : null - }; - - return this._private('moveOrder', parameters, callback); - }, - - withdraw: function (currency, amount, address, callback) { - var parameters = { - currency: currency, - amount: amount, - address: address - }; - - return this._private('withdraw', parameters, callback); - }, - - returnFeeInfo: function (callback) { - return this._private('returnFeeInfo', {}, callback); - }, - - returnAvailableAccountBalances: function (account, callback) { - var options = {}; - if (account) { - options.account = account; - } - return this._private('returnAvailableAccountBalances', options, callback); - }, - - returnTradableBalances: function (callback) { - return this._private('returnTradableBalances', {}, callback); - }, - - transferBalance: function (currency, amount, fromAccount, toAccount, callback) { - var parameters = { - currency: currency, - amount: amount, - fromAccount: fromAccount, - toAccount: toAccount - }; - - return this._private('transferBalance', parameters, callback); - }, - - returnMarginAccountSummary: function (callback) { - return this._private('returnMarginAccountSummary', {}, callback); - }, - - marginBuy: function (currencyA, currencyB, rate, amount, lendingRate, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB), - rate: rate, - amount: amount, - lendingRate: lendingRate ? lendingRate : null - }; - - return this._private('marginBuy', parameters, callback); - }, - - marginSell: function (currencyA, currencyB, rate, amount, lendingRate, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB), - rate: rate, - amount: amount, - lendingRate: lendingRate ? lendingRate : null - }; - - return this._private('marginSell', parameters, callback); - }, - - getMarginPosition: function (currencyA, currencyB, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB) - }; - - return this._private('getMarginPosition', parameters, callback); - }, - - closeMarginPosition: function (currencyA, currencyB, callback) { - var parameters = { - currencyPair: joinCurrencies(currencyA, currencyB) - }; - - return this._private('closeMarginPosition', parameters, callback); - }, - - createLoanOffer: function (currency, amount, duration, autoRenew, lendingRate, callback) { - var parameters = { - currency: currency, - amount: amount, - duration: duration, - autoRenew: autoRenew, - lendingRate: lendingRate - }; - - return this._private('createLoanOffer', parameters, callback); - }, - - cancelLoanOffer: function (orderNumber, callback) { - var parameters = { - orderNumber: orderNumber - }; - - return this._private('cancelLoanOffer', parameters, callback); - }, - - returnOpenLoanOffers: function (callback) { - return this._private('returnOpenLoanOffers', {}, callback); - }, - - returnActiveLoans: function (callback) { - return this._private('returnActiveLoans', {}, callback); - }, - - returnLendingHistory: function (start, end, limit, callback) { - var parameters = { - start: start, - end: end, - limit: limit - }; - - return this._private('returnLendingHistory', parameters, callback); - }, - - toggleAutoRenew: function (orderNumber, callback) { - return this._private('toggleAutoRenew', { orderNumber: orderNumber }, callback); - } - - }; - - // Backwards Compatibility - Poloniex.prototype.getTicker = Poloniex.prototype.returnTicker; - Poloniex.prototype.get24hVolume = Poloniex.prototype.return24hVolume; - Poloniex.prototype.getOrderBook = Poloniex.prototype.returnOrderBook; - Poloniex.prototype.getTradeHistory = Poloniex.prototype.returnChartData; - Poloniex.prototype.myBalances = Poloniex.prototype.returnBalances; - Poloniex.prototype.myOpenOrders = Poloniex.prototype.returnOpenOrders; - Poloniex.prototype.myTradeHistory = Poloniex.prototype.returnTradeHistory; - - return Poloniex; -})(); \ No newline at end of file diff --git a/Utilities.js b/Utilities.js deleted file mode 100644 index b70a5b5f2..000000000 --- a/Utilities.js +++ /dev/null @@ -1,81 +0,0 @@ - -exports.newUtilities = function newUtilities(BOT) { - - let bot = BOT; - const ROOT_DIR = '../'; - - const MODULE_NAME = "Utilities"; - - - const DEBUG_MODULE = require('./Debug Log'); - const logger = DEBUG_MODULE.newDebugLog(); - logger.fileName = MODULE_NAME; - logger.bot = bot; - - let utilities = { - pad: pad, - createFolderIfNeeded: createFolderIfNeeded, - calculatePresicion: calculatePresicion - }; - - return utilities; - - - function pad(str, max) { - str = str.toString(); - return str.length < max ? pad("0" + str, max) : str; - } - - function createFolderIfNeeded(path, azureFileStorage, callBackFunction) { - - let splittedPath = path.split("/"); - - let partialPath = ''; - let separator = ""; - - - let i = 0; - loop(); - - function loop() { - - partialPath = partialPath + separator + splittedPath[i]; - separator = "/"; - - i++; - - azureFileStorage.createFolder(partialPath, checkLoop); - - function checkLoop() { - - if (i === splittedPath.length) { - - callBackFunction(); - - } else { - - loop(); - - } - - - } - - } - - } - - - function calculatePresicion(number) { - - for (let i = -10; i <= 10; i++) { - if (number < Math.pow(10, i)) { - return Math.pow(10, i - 3); - } - } - } - - - - -} \ No newline at end of file diff --git a/this.bot.config.json b/this.bot.config.json new file mode 100644 index 000000000..5cec80417 --- /dev/null +++ b/this.bot.config.json @@ -0,0 +1,34 @@ +{ + "bot": { + "name": "Olivia", + "type": "Indicator", + "version": "1.0.0", + "devTeam": "AA Masters" + }, + "processes": [ + { + "name": "Multi-Period-Market", + "description": "Produces different files with multi-period candles and volumes for entire markets.", + "startMode": { + "allMonths": { + "run": "true", + "minYear": "2014", + "maxYear": "2018" + }, + "oneMonth": { + "run": "false", + "year": "2018", + "month": "2" + }, + "noTime": { + "run": "false" + } + }, + "intervalLapse": 3600000, + "intervalMaxLoops": -1 + } + ] +} + + +