From 066ea6159ad0b61bf01ad175d2628e4800c27dcd Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 20 Aug 2019 10:43:40 +0200 Subject: [PATCH 01/32] Cnagens on Trading Engine modules --- Commons.js | 6 ++- User.Bot.js | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/Commons.js b/Commons.js index dd6a6a7da..0a81b6709 100644 --- a/Commons.js +++ b/Commons.js @@ -514,9 +514,13 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { function loop() { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Entering function."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> i = " + i); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Processing candle # " + i); } let candle = candles[i]; + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Candle Begin @ " + (new Date(candle.begin)).toLocaleString()) } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Candle End @ " + (new Date(candle.end)).toLocaleString()) } + let percentageBandwidth = percentageBandwidthMap.get(candle.begin); let bollingerBand = bollingerBandsMap.get(candle.begin); let bollingerChannel = getElement(bollingerChannelsArray, candle.begin, candle.end); diff --git a/User.Bot.js b/User.Bot.js index 89b84fe38..d646dd02a 100644 --- a/User.Bot.js +++ b/User.Bot.js @@ -11,6 +11,8 @@ const STRATEGIES_FOLDER_NAME = "Simulation-Strategies"; const TRADES_FOLDER_NAME = "Simulation-Trades"; + const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; + const commons = COMMONS.newCommons(bot, logger, UTILITIES); thisObject = { @@ -180,7 +182,7 @@ } - writeRecordsFile(); + writeFinalResults(); } catch (err) { @@ -195,6 +197,121 @@ } } + function writeFinalResults() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFinalResults -> Entering function."); } + + if ( // We only write this file if we are at the head of the market. + Math.trunc(currentDay.valueOf() / ONE_DAY_IN_MILISECONDS) * ONE_DAY_IN_MILISECONDS + !== + Math.trunc((new Date()).valueOf() / ONE_DAY_IN_MILISECONDS) * ONE_DAY_IN_MILISECONDS + ) { + writeRecordsFile(); + return; + } + + let separator = ""; + let fileRecordCounter = 0; + + let fileContent = ""; + + for (let i = 0; i < recordsArray.length; i++) { + + let record = recordsArray[i]; + + /* Will only add to the file the records of the current day */ + + if (record.begin < currentDay.valueOf()) { continue; } + + fileContent = fileContent + separator + '[' + + record.begin + "," + + record.end + "," + + record.type + "," + + record.marketRate + "," + + record.amount + "," + + record.balanceA + "," + + record.balanceB + "," + + record.profit + "," + + record.lastTradeProfitLoss + "," + + record.stopLoss + "," + + record.roundtrips + "," + + record.hits + "," + + record.fails + "," + + record.hitRatio + "," + + record.ROI + "," + + record.periods + "," + + record.days + "," + + record.anualizedRateOfReturn + "," + + record.positionRate + "," + + record.lastTradeROI + "," + + record.strategy + "," + + record.strategyStageNumber + "," + + record.takeProfit + "," + + record.stopLossPhase + "," + + record.takeProfitPhase + "," + + JSON.stringify(record.executionRecord) + "," + + record.positionSize + "," + + record.initialBalanceA + "," + + record.minimumBalanceA + "," + + record.maximumBalanceA + "," + + record.initialBalanceB + "," + + record.minimumBalanceB + "," + + record.maximumBalanceB + "," + + record.baseAsset + "," + + record.positionPeriods + "," + + record.positionDays + "]"; + + if (separator === "") { separator = ","; } + + fileRecordCounter++; + + } + + fileContent = "[" + fileContent + "]"; + + let fileName = '' + market.assetA + '_' + market.assetB + '.json'; + + let filePathRoot = bot.devTeam + "/" + bot.codeName + "." + bot.version.major + "." + bot.version.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; + let filePath = filePathRoot + "/Output/" + SIMULATED_RECORDS_FOLDER_NAME + "/" + "Multi-Period-Daily" + "/" + outputPeriodLabel; + filePath += '/' + fileName + + fileStorage.createTextFile(bot.devTeam, filePath, fileContent + '\n', onFileCreated); + + function onFileCreated(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFinalResults -> onFileCreated -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFinalResults -> onFileCreated -> fileContent = " + fileContent); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> writeFinalResults -> onFileCreated -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> writeFinalResults -> onFileCreated -> filePath = " + filePath); + logger.write(MODULE_NAME, "[ERROR] start -> writeFinalResults -> onFileCreated -> market = " + market.assetA + "_" + market.assetB); + + callBackFunction(err); + return; + + } + + writeRecordsFile(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> writeFinalResults -> onFileCreated -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> writeFinalResults -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + function writeRecordsFile() { try { From bc91573dcc18f8058c51d2787066af4fd7be84b2 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 20 Aug 2019 11:06:22 +0200 Subject: [PATCH 02/32] Code Tyding --- Assistant.js | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/Assistant.js b/Assistant.js index f1f4f3611..ddac3bd04 100644 --- a/Assistant.js +++ b/Assistant.js @@ -1,9 +1,7 @@ exports.newAssistant = function newAssistant(BOT, logger, UTILITIES) { /* - - This module allows trading bots to execute actions on the exchange, and also on its current recorded state. - + This module allows trading bots to execute actions on the exchange. */ const MODULE_NAME = "Assistant"; @@ -418,7 +416,7 @@ actions. Reasons why the positions might not be there are: 1. The user / account owner closed the positions manually. - 2. The exchange for some eventuality closed the positions. In some exchanges positions have an expiration time. + 2. The exchange for some reason closed the positions. In some exchanges positions have an expiration time. 3. The orders were executed. Situations 1 and 2 are similar unexpected and we will stop the bot execution when we detect them. Number 3 is an @@ -1334,31 +1332,20 @@ context.newHistoryRecord.messageRelevance = pRelevance; context.newHistoryRecord.messageTitle = bot.startMode + "." + runIndex +": "+ pTitle; context.newHistoryRecord.messageBody = pBody; - } function rememberThis(pKey, pValue) { - if (context.executionContext.remember === undefined) { - context.executionContext.remember = {}; - } - context.executionContext.remember[pKey] = pValue; - } function remindMeOf(pKey) { - if (context.executionContext.remember === undefined) { - context.executionContext.remember = {}; - } - return context.executionContext.remember[pKey]; - } function sendEmail(pTitle, pBody, pTo) { @@ -1367,15 +1354,11 @@ } function addExtraData(pExtraDataArray) { - context.extraData.push(pExtraDataArray); } function truncDecimals(pFloatValue, pDecimals) { - if (!pDecimals) pDecimals = 8; // Default value - return parseFloat(parseFloat(pFloatValue).toFixed(pDecimals)); - } }; From c0ae66361b8c67d8c037d4288ef817a3dce70bc3 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 20 Aug 2019 11:08:33 +0200 Subject: [PATCH 03/32] Refactoring --- Context.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Context.js b/Context.js index dc365ec50..23f09ab5c 100644 --- a/Context.js +++ b/Context.js @@ -304,24 +304,24 @@ thisObject.executionHistory = []; - if (process.env.INITIAL_BALANCE_ASSET_A === undefined || process.env.INITIAL_BALANCE_ASSET_B === undefined) { throw new Error("Environment Variables INITIAL_INVESTMENT_A and INITIAL_INVESTMENT_B cannot be undefined.") } + if (process.env.INITIAL_BALANCE_ASSET_A === undefined || process.env.INITIAL_BALANCE_ASSET_B === undefined) { throw new Error("Environment Variables INITIAL_BALANCE_A and INITIAL_BALANCE_B cannot be undefined.") } - const INITIAL_INVESTMENT_A = Number(process.env.INITIAL_BALANCE_ASSET_A); - const INITIAL_INVESTMENT_B = Number(process.env.INITIAL_BALANCE_ASSET_B); + const INITIAL_BALANCE_A = Number(process.env.INITIAL_BALANCE_ASSET_A); + const INITIAL_BALANCE_B = Number(process.env.INITIAL_BALANCE_ASSET_B); thisObject.executionContext = { investment: { // This is used to calculate profits. - assetA: INITIAL_INVESTMENT_A, - assetB: INITIAL_INVESTMENT_B + assetA: INITIAL_BALANCE_A, + assetB: INITIAL_BALANCE_B }, balance: { // This is the total balance that includes positions at the order book + funds available to be traded. - assetA: INITIAL_INVESTMENT_A, - assetB: INITIAL_INVESTMENT_B // It starts with the initial investment. + assetA: INITIAL_BALANCE_A, + assetB: INITIAL_BALANCE_B // It starts with the initial investment. }, availableBalance: { // This is the balance the bot has at any moment in time available to be traded (not in positions at the order book). - assetA: INITIAL_INVESTMENT_A, - assetB: INITIAL_INVESTMENT_B // It starts with the initial investment. + assetA: INITIAL_BALANCE_A, + assetB: INITIAL_BALANCE_B // It starts with the initial investment. }, profits: { assetA: 0, From 26088087c08c25299ff763fba73ad5df132c7628 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 20 Aug 2019 11:12:32 +0200 Subject: [PATCH 04/32] Investment for InitialBalance Refactoring --- Assistant.js | 34 +++++++++++++++++----------------- Context.js | 6 +++--- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Assistant.js b/Assistant.js index ddac3bd04..75ac8c03a 100644 --- a/Assistant.js +++ b/Assistant.js @@ -16,7 +16,7 @@ getPositions: getPositions, getBalance: getBalance, getAvailableBalance: getAvailableBalance, - getInvestment: getInvestment, + getInitialBalance: getInitialBalance, getProfits: getProfits, getCombinedProfits: getCombinedProfits, getROI: getROI, @@ -272,15 +272,15 @@ /* Calculate Profits */ - if (context.executionContext.investment.assetA > 0) { + if (context.executionContext.initialBalance.assetA > 0) { - context.executionContext.profits.assetA = (thisObject.truncDecimals(context.executionContext.balance.assetA) - thisObject.truncDecimals(context.executionContext.investment.assetA)) / thisObject.truncDecimals(context.executionContext.investment.assetA); + context.executionContext.profits.assetA = (thisObject.truncDecimals(context.executionContext.balance.assetA) - thisObject.truncDecimals(context.executionContext.initialBalance.assetA)) / thisObject.truncDecimals(context.executionContext.initialBalance.assetA); context.executionContext.profits.assetA = thisObject.truncDecimals(context.executionContext.profits.assetA); } - if (context.executionContext.investment.assetB > 0) { + if (context.executionContext.initialBalance.assetB > 0) { - context.executionContext.profits.assetB = (thisObject.truncDecimals(context.executionContext.balance.assetB) - thisObject.truncDecimals(context.executionContext.investment.assetB)) / thisObject.truncDecimals(context.executionContext.investment.assetB); + context.executionContext.profits.assetB = (thisObject.truncDecimals(context.executionContext.balance.assetB) - thisObject.truncDecimals(context.executionContext.initialBalance.assetB)) / thisObject.truncDecimals(context.executionContext.initialBalance.assetB); context.executionContext.profits.assetB = thisObject.truncDecimals(context.executionContext.profits.assetB); } @@ -289,21 +289,21 @@ /* Calculate Combined Profits */ - if (context.executionContext.investment.assetA > 0) { + if (context.executionContext.initialBalance.assetA > 0) { - let convertedAssetsB = (thisObject.truncDecimals(context.executionContext.balance.assetB) - thisObject.truncDecimals(context.executionContext.investment.assetB)) * marketRate; + let convertedAssetsB = (thisObject.truncDecimals(context.executionContext.balance.assetB) - thisObject.truncDecimals(context.executionContext.initialBalance.assetB)) * marketRate; convertedAssetsB = thisObject.truncDecimals(convertedAssetsB); - context.executionContext.combinedProfits.assetA = (thisObject.truncDecimals(context.executionContext.balance.assetA) + convertedAssetsB - thisObject.truncDecimals(context.executionContext.investment.assetA)) / thisObject.truncDecimals(context.executionContext.investment.assetA) * 100; + context.executionContext.combinedProfits.assetA = (thisObject.truncDecimals(context.executionContext.balance.assetA) + convertedAssetsB - thisObject.truncDecimals(context.executionContext.initialBalance.assetA)) / thisObject.truncDecimals(context.executionContext.initialBalance.assetA) * 100; context.executionContext.combinedProfits.assetA = thisObject.truncDecimals(context.executionContext.combinedProfits.assetA); } - if (context.executionContext.investment.assetB > 0) { + if (context.executionContext.initialBalance.assetB > 0) { - let convertedAssetsA = (thisObject.truncDecimals(context.executionContext.balance.assetA) - thisObject.truncDecimals(context.executionContext.investment.assetA)) / marketRate; + let convertedAssetsA = (thisObject.truncDecimals(context.executionContext.balance.assetA) - thisObject.truncDecimals(context.executionContext.initialBalance.assetA)) / marketRate; convertedAssetsA = thisObject.truncDecimals(convertedAssetsA); - context.executionContext.combinedProfits.assetB = (thisObject.truncDecimals(context.executionContext.balance.assetB) + convertedAssetsA - thisObject.truncDecimals(context.executionContext.investment.assetB)) / thisObject.truncDecimals(context.executionContext.investment.assetB) * 100; + context.executionContext.combinedProfits.assetB = (thisObject.truncDecimals(context.executionContext.balance.assetB) + convertedAssetsA - thisObject.truncDecimals(context.executionContext.initialBalance.assetB)) / thisObject.truncDecimals(context.executionContext.initialBalance.assetB) * 100; context.executionContext.combinedProfits.assetB = thisObject.truncDecimals(context.executionContext.combinedProfits.assetB); } @@ -1274,12 +1274,12 @@ return JSON.parse(JSON.stringify(context.executionContext.availableBalance)); } - function getInvestment() { + function getInitialBalance() { - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getInvestment -> Entering function."); } - if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getInvestment -> JSON.stringify(context.executionContext.investment) = " + JSON.stringify(context.executionContext.investment)); } + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getInitialBalance -> Entering function."); } + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getInitialBalance -> JSON.stringify(context.executionContext.initialBalance) = " + JSON.stringify(context.executionContext.initialBalance)); } - return JSON.parse(JSON.stringify(context.executionContext.investment)); + return JSON.parse(JSON.stringify(context.executionContext.initialBalance)); } function getProfits() { @@ -1309,8 +1309,8 @@ if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getROI -> Entering function."); } let ROI = { - assetA: (context.executionContext.balance.assetA - context.executionContext.investment.assetA) / context.executionContext.investment.assetA * 100, - assetB: (context.executionContext.balance.assetB - context.executionContext.investment.assetB) / context.executionContext.investment.assetB * 100 + assetA: (context.executionContext.balance.assetA - context.executionContext.initialBalance.assetA) / context.executionContext.initialBalance.assetA * 100, + assetB: (context.executionContext.balance.assetB - context.executionContext.initialBalance.assetB) / context.executionContext.initialBalance.assetB * 100 } if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] getROI -> JSON.stringify(ROI) = " + JSON.stringify(ROI)); } diff --git a/Context.js b/Context.js index 23f09ab5c..dfa757cba 100644 --- a/Context.js +++ b/Context.js @@ -311,17 +311,17 @@ thisObject.executionContext = { - investment: { // This is used to calculate profits. + initialBalance: { // This is used to calculate profits. assetA: INITIAL_BALANCE_A, assetB: INITIAL_BALANCE_B }, balance: { // This is the total balance that includes positions at the order book + funds available to be traded. assetA: INITIAL_BALANCE_A, - assetB: INITIAL_BALANCE_B // It starts with the initial investment. + assetB: INITIAL_BALANCE_B // It starts with the initial initialBalance. }, availableBalance: { // This is the balance the bot has at any moment in time available to be traded (not in positions at the order book). assetA: INITIAL_BALANCE_A, - assetB: INITIAL_BALANCE_B // It starts with the initial investment. + assetB: INITIAL_BALANCE_B // It starts with the initial initialBalance. }, profits: { assetA: 0, From d7ac9723126a3c67cbd76e43548ecda97f5bba1f Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 20 Aug 2019 21:54:36 +0200 Subject: [PATCH 05/32] Fixing Many problems with live execution --- Commons.js | 343 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 234 insertions(+), 109 deletions(-) diff --git a/Commons.js b/Commons.js index 0a81b6709..2fad1c1cc 100644 --- a/Commons.js +++ b/Commons.js @@ -163,6 +163,26 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } } + /* Slippage */ + + let slippage = 0 + if (definition.tradingSystem !== undefined) { + if (definition.tradingSystem.parameters !== undefined) { + if (definition.tradingSystem.parameters.baseAsset !== undefined) { + if (definition.tradingSystem.parameters.baseAsset.formula !== undefined) { + if (definition.tradingSystem.parameters.baseAsset.formula.code !== undefined) { + let code = JSON.parse(definition.tradingSystem.parameters.baseAsset.formula.code) + if (code.initialDatetime !== undefined) { + slippage = code.slippage + } + } + } + } + } + } + + let timerToCloseStage = 0 + /* Stop Loss Management */ const MIN_STOP_LOSS_VALUE = 1 // We can not let the stop be zero to avoid division by 0 error or infinity numbers as a result. @@ -186,6 +206,7 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { let type = '""'; let marketRate = 0; let takePositionNow = false + let closePositionNow = false /* In some cases we need to know if we are positioned at the last candle of the calendar day, for that we need these variables. */ @@ -531,7 +552,7 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { let ticker - if (process.env.START_MODE === "live") { + if (process.env.START_MODE === "live" && i === candles.length - 2) { ticker = assistant.getTicker() } else { ticker = { @@ -546,27 +567,27 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { /* If any of the needed indicators is missing, then that period is not calculated */ if (candle.end < initialDate.valueOf()) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Record before the initialDate."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle before the initialDate."); } controlLoop(); return } if (bollingerBand === undefined) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Record because Bollinger Band is undefined."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because Bollinger Band is undefined."); } controlLoop(); return } if (percentageBandwidth === undefined) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Record because %B is undefined."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because %B is undefined."); } controlLoop(); return } // percentageBandwidth might start after the first few candles. if (bollingerChannel === undefined) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Record because Bollingeer Channel is undefined."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because Bollingeer Channel is undefined."); } controlLoop(); return } if (bollingerSubChannel === undefined) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Record because Bollingeer Sub Channel is undefined."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because Bollingeer Sub Channel is undefined."); } controlLoop(); return } @@ -586,6 +607,7 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { if (i === candles.length - 1) { if ((candles.length < candlesPerDay) || (candles.length > candlesPerDay && candles.length < candlesPerDay * 2)) { /*We are at the head of the market, thus we skip the last candle because it has not close yet. */ + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because it is the last one and has not been closed yet."); } controlLoop(); return /* Note here that in the last candle of the first day or the second day it will use an incomplete candle and partially calculated indicators. @@ -595,6 +617,7 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } else { // We are processing Market Files if (i === candles.length - 1) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because it is the last one and has not been closed yet."); } controlLoop(); return } @@ -1560,17 +1583,38 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { if ((baseAsset === 'BTC' && candle.max >= stopLoss) || (baseAsset !== 'BTC' && candle.min <= stopLoss)) { - if (baseAsset === 'BTC') { - strategy.positionSize = balanceAssetB / stopLoss; - strategy.positionRate = stopLoss; + /* Check the Slipagge */ - balanceAssetA = balanceAssetA + balanceAssetB / stopLoss; + let stopLossHitPoint = stopLoss + + if (baseAsset === 'BTC') { + if (stopLossHitPoint < candle.open) { stopLossHitPoint = candle.open} + if (stopLossHitPoint < candle.close) { stopLossHitPoint = candle.close } + } else { + if (stopLossHitPoint > candle.open) { stopLossHitPoint = candle.open } + if (stopLossHitPoint > candle.close) { stopLossHitPoint = candle.close } + } + + let slippedStopLoss = stopLossHitPoint + let slippageAmount = stopLossHitPoint * slippage / 100 + + if (baseAsset === 'BTC') { + slippedStopLoss = slippedStopLoss + slippageAmount + } else { + slippedStopLoss = slippedStopLoss - slippageAmount + } + + if (baseAsset === 'BTC') { + strategy.positionSize = balanceAssetB / slippedStopLoss; + strategy.positionRate = slippedStopLoss; + + balanceAssetA = balanceAssetA + balanceAssetB / slippedStopLoss; balanceAssetB = 0; } else { - strategy.positionSize = balanceAssetA * stopLoss; - strategy.positionRate = stopLoss; + strategy.positionSize = balanceAssetA * slippedStopLoss; + strategy.positionRate = slippedStopLoss; - balanceAssetB = balanceAssetB + balanceAssetA * stopLoss; + balanceAssetB = balanceAssetB + balanceAssetA * slippedStopLoss; balanceAssetA = 0; } @@ -1581,7 +1625,7 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } } - marketRate = stopLoss; + marketRate = slippedStopLoss; type = '"Close@StopLoss"'; strategyStage = 'Close Stage'; stopLossStage = 'No Stage'; @@ -1589,12 +1633,9 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { currentTrade.end = candle.end; currentTrade.status = 1; currentTrade.exitType = 1; - currentTrade.endRate = stopLoss; + currentTrade.endRate = slippedStopLoss; - currentStrategy.number = currentStrategyIndex - currentStrategy.end = candle.end; - currentStrategy.endRate = candle.min; - currentStrategy.status = 1; + closePositionNow = true; if (processingDailyFiles) { if (positionedAtYesterday) { @@ -1605,10 +1646,6 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { yesterday.currentTrade.status = currentTrade.status; yesterday.currentTrade.exitType = currentTrade.exitType; yesterday.currentTrade.endRate = currentTrade.endRate; - yesterday.currentStrategy.number = currentStrategy.number; - yesterday.currentStrategy.end = currentStrategy.end; - yesterday.currentStrategy.endRate = currentStrategy.endRate; - yesterday.currentStrategy.status = currentStrategy.status; } } } @@ -1617,17 +1654,38 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { if ((baseAsset === 'BTC' && candle.min <= takeProfit) || (baseAsset !== 'BTC' && candle.max >= takeProfit)) { - if (baseAsset === 'BTC') { - strategy.positionSize = balanceAssetB / takeProfit; - strategy.positionRate = takeProfit; + /* Check the Slipagge */ - balanceAssetA = balanceAssetA + balanceAssetB / takeProfit; + let takeProfitHitPoint = stopLoss + + if (baseAsset === 'BTC') { + if (takeProfitHitPoint > candle.open) { takeProfitHitPoint = candle.open } + if (takeProfitHitPoint > candle.close) { takeProfitHitPoint = candle.close } + } else { + if (takeProfitHitPoint < candle.open) { takeProfitHitPoint = candle.open } + if (takeProfitHitPoint < candle.close) { takeProfitHitPoint = candle.close } + } + + let slippedTakeProfit = takeProfit + let slippageAmount = stopLoss * slippage / 100 + + if (baseAsset === 'BTC') { + slippedTakeProfit = slippedTakeProfit - slippageAmount + } else { + slippedTakeProfit = slippedTakeProfit + slippageAmount + } + + if (baseAsset === 'BTC') { + strategy.positionSize = balanceAssetB / slippedTakeProfit; + strategy.positionRate = slippedTakeProfit; + + balanceAssetA = balanceAssetA + balanceAssetB / slippedTakeProfit; balanceAssetB = 0; } else { - strategy.positionSize = balanceAssetA * takeProfit; - strategy.positionRate = takeProfit; + strategy.positionSize = balanceAssetA * slippedTakeProfit; + strategy.positionRate = slippedTakeProfit; - balanceAssetB = balanceAssetB + balanceAssetA * takeProfit; + balanceAssetB = balanceAssetB + balanceAssetA * slippedTakeProfit; balanceAssetA = 0; } @@ -1639,7 +1697,7 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } } - marketRate = takeProfit; + marketRate = slippedTakeProfit; type = '"Close@TakeProfit"'; strategyStage = 'Close Stage'; stopLossStage = 'No Stage'; @@ -1648,12 +1706,9 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { currentTrade.end = candle.end; currentTrade.status = 1; currentTrade.exitType = 2; - currentTrade.endRate = takeProfit; + currentTrade.endRate = slippedTakeProfit; - currentStrategy.number = currentStrategyIndex - currentStrategy.end = candle.end; - currentStrategy.endRate = candle.min; - currentStrategy.status = 1; + closePositionNow = true; if (processingDailyFiles) { if (positionedAtYesterday) { @@ -1665,11 +1720,6 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { yesterday.currentTrade.status = currentTrade.status; yesterday.currentTrade.exitType = currentTrade.exitType; yesterday.currentTrade.endRate = currentTrade.endRate; - - yesterday.currentStrategy.number = currentStrategy.number; - yesterday.currentStrategy.end = currentStrategy.end; - yesterday.currentStrategy.endRate = currentStrategy.endRate; - yesterday.currentStrategy.status = currentStrategy.status; } } } @@ -1686,7 +1736,23 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { let strategy = tradingSystem.strategies[currentStrategyIndex]; tradePositionSize = strategy.positionSize; - tradePositionRate = strategy.positionRate; + + if (i === candles.length - 2) { /* Only at the last closed candle I can take the value from the formula, since the use of ticker could alter the value at previous candles.*/ + tradePositionRate = strategy.positionRate; + } else { + + /* Check the Slipagge */ + + tradePositionRate = candle.close; + + let slippageAmount = tradePositionRate * slippage / 100 + + if (baseAsset === 'BTC') { + tradePositionRate = tradePositionRate - slippageAmount + } else { + tradePositionRate = tradePositionRate + slippageAmount + } + } if (processingDailyFiles) { if (positionedAtYesterday) { @@ -1695,46 +1761,52 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } } - /* Check that we are in LIVE MODE */ - if (process.env.START_MODE === "live") { - /* We see if we need to put the actual order at the exchange. */ - if (interExecutionMemory.executionContext !== undefined) { - switch (interExecutionMemory.executionContext.status) { - case "Without a Position": { // We need to put the order because It was not put yet. - if (strategy.openStage !== undefined) { - if (strategy.openStage.openExecution !== undefined) { - putOpeningOrder() - return + if (i > candles.length - 10) { /* Only at the last candles makes sense to check if we are in live mode or not.*/ + /* Check that we are in LIVE MODE */ + if (process.env.START_MODE === "live") { + /* We see if we need to put the actual order at the exchange. */ + if (interExecutionMemory.executionContext !== undefined) { + switch (interExecutionMemory.executionContext.status) { + case "Without a Position": { // We need to put the order because It was not put yet. + if (strategy.openStage !== undefined) { + if (strategy.openStage.openExecution !== undefined) { + putOpeningOrder() + return + } } + break } - break - } - case "Position Closed": { // Waiting for a confirmation that the position was closed. - if (strategy.openStage !== undefined) { - if (strategy.openStage.openExecution !== undefined) { - putOpeningOrder() - return + case "Position Closed": { // Waiting for a confirmation that the position was closed. + if (strategy.openStage !== undefined) { + if (strategy.openStage.openExecution !== undefined) { + putOpeningOrder() + return + } } + break + } + case "Taking Position": { // Waiting for a confirmation that the position was taken. + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionNow -> Exiting code block because status is Taking Position."); } + break + } + case "In a Position": { // This should mean that we already put the order at the exchange. + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionNow -> Exiting code block because status is In a Position."); } + break } - break } - case "Taking Position": { // Waiting for a confirmation that the position was taken. - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionNow -> Exiting code block because status is Taking Position."); } - break - } - case "In a Position": { // This should mean that we already put the order at the exchange. - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionNow -> Exiting code block because status is In a Position."); } - break - } - } - } else { // The context does not exist so it means we are not in a position. - if (strategy.openStage !== undefined) { - if (strategy.openStage.openExecution !== undefined) { - putOpeningOrder() - return + } else { // The context does not exist so it means we are not in a position. + if (strategy.openStage !== undefined) { + if (strategy.openStage.openExecution !== undefined) { + putOpeningOrder() + return + } } } + } else { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionNow -> Not trading live."); } } + } else { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionNow -> Not the last closed candle."); } } takePositionAtSimulation() @@ -1801,25 +1873,35 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { let amountA let amountB let positionDirection + let availableBalance = assistant.getAvailableBalance() if (baseAsset === 'BTC') { positionDirection = "sell" - openPositionRate = tradePositionRate + openPositionRate = tradePositionRate - 100 // 100 is Provisional since we do have management of orders yet amountA = tradePositionSize * openPositionRate amountB = tradePositionSize + + if (amountB > availableBalance.assetB) { // The assistant know what fees were paid. + amountB = availableBalance.assetB + } } else { positionDirection = "buy" - openPositionRate = tradePositionRate + openPositionRate = tradePositionRate + 100 // 100 is Provisional since we do have management of orders yet amountA = tradePositionSize amountB = tradePositionSize / openPositionRate + + if (amountA > availableBalance.assetA) { // The assistant know what fees were paid. + amountA = availableBalance.assetA + } } interExecutionMemory.executionContext = { - status: "Taking Position" + status: "Taking Position", + periods: periods, } if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> Ready to put order."); } @@ -1915,48 +1997,53 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } } - /* Closing a Position */ - if (strategyStage === 'Close Stage') { + if (closePositionNow === true) { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing a Position -> Entering code block."); } + closePositionNow = false + /* Position size and rate */ let strategy = tradingSystem.strategies[currentStrategyIndex]; - /* Check that we are in LIVE MODE */ - if (process.env.START_MODE === "live") { - /* We see if we need to put the actual order at the exchange. */ - if (interExecutionMemory.executionContext !== undefined) { - switch (interExecutionMemory.executionContext.status) { - case "Without a Position": { // No way to close anything at the exchange. - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing a Position -> Exiting code block because status is Without a Position."); } - break - } - case "In a Position": { // This should mean that we already put the order at the exchange. - if (strategy.closeStage !== undefined) { - if (strategy.closeStage.closeExecution !== undefined) { - putClosingOrder() - return - } + if (i > candles.length - 10) { /* Only at the last candles makes sense to check if we are in live mode or not.*/ + /* Check that we are in LIVE MODE */ + if (process.env.START_MODE === "live") { + /* We see if we need to put the actual order at the exchange. */ + if (interExecutionMemory.executionContext !== undefined) { + switch (interExecutionMemory.executionContext.status) { + case "Without a Position": { // No way to close anything at the exchange. + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing a Position -> Exiting code block because status is Without a Position."); } + break } - break - } - case "Closing Position": { // Waiting for a confirmation that the position was taken. - if (strategy.closeStage !== undefined) { - if (strategy.closeStage.closeExecution !== undefined) { - putClosingOrder() - return + case "In a Position": { // This should mean that we already put the order at the exchange. + if (strategy.closeStage !== undefined) { + if (strategy.closeStage.closeExecution !== undefined) { + putClosingOrder() + return + } } + break + } + case "Closing Position": { // Waiting for a confirmation that the position was taken. + if (strategy.closeStage !== undefined) { + if (strategy.closeStage.closeExecution !== undefined) { + putClosingOrder() + return + } + } + break } - break - } - case "Position Closed": { // - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing a Position -> Exiting code block because status is Position Closed."); } - break + case "Position Closed": { // + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing a Position -> Exiting code block because status is Position Closed."); } + break + } } } } + } else { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing a Position -> Not within the last 10 candles."); } } closePositionAtSimulation() @@ -1993,6 +2080,7 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { let amountA let amountB let positionDirection + let availableBalance = assistant.getAvailableBalance() if (baseAsset === 'BTC') { positionDirection = "buy" @@ -2001,6 +2089,10 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { amountA = interExecutionMemory.executionContext.amountA amountB = interExecutionMemory.executionContext.amountA / closePositionRate + + if (amountA > availableBalance.assetA) { // The assistant know what fees were paid. + amountA = availableBalance.assetA + } } else { positionDirection = "sell" @@ -2008,10 +2100,15 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { amountA = interExecutionMemory.executionContext.amountB * closePositionRate amountB = interExecutionMemory.executionContext.amountB + + if (amountB > availableBalance.assetB) { // The assistant know what fees were paid. + amountB = availableBalance.assetB + } } interExecutionMemory.executionContext = { - status: "Closing Position" + status: "Closing Position", + periods: periods, } if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putClosingOrder -> About to close position at the exchange."); } @@ -2152,8 +2249,7 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } } - currentStrategyIndex = -1; - strategyStage = 'No Stage'; + timerToCloseStage = candle.begin stopLossStage = 'No Stage'; takeProfitStage = 'No Stage'; stopLossPhase = 0; @@ -2176,6 +2272,35 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } } + /* Closing the Closing Stage */ + if (strategyStage === 'Close Stage') { + if (candle.begin - 5 * 60 * 1000 > timerToCloseStage) { + + currentStrategy.number = currentStrategyIndex + currentStrategy.end = candle.end; + currentStrategy.endRate = candle.min; + currentStrategy.status = 1; + + if (processingDailyFiles) { + if (positionedAtYesterday) { + + yesterday.currentStrategy.number = currentStrategy.number; + yesterday.currentStrategy.end = currentStrategy.end; + yesterday.currentStrategy.endRate = currentStrategy.endRate; + yesterday.currentStrategy.status = currentStrategy.status; + } + } + + currentStrategyIndex = -1; + strategyStage = 'No Stage'; + timerToCloseStage = 0 + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing the Closing Stage -> Exiting Close Stage."); } + } else { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing the Closing Stage -> Waiting for timer."); } + } + } + /* Not a buy or sell condition */ marketRate = candle.close; From 9ef05adf0c47d16119648592a3748fd3c6fcb086 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 23 Aug 2019 18:12:31 +0200 Subject: [PATCH 06/32] Trading Engine Changes --- Commons.js | 223 +++++++++++++++++++++----------- TradingEngineProcessMainLoop.js | 8 +- 2 files changed, 153 insertions(+), 78 deletions(-) diff --git a/Commons.js b/Commons.js index 2fad1c1cc..54bbff3a8 100644 --- a/Commons.js +++ b/Commons.js @@ -164,16 +164,61 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } /* Slippage */ - - let slippage = 0 + let slippage = { // Default Values + positionRate: 0, + stopLoss: 0, + takeProfit: 0 + } if (definition.tradingSystem !== undefined) { if (definition.tradingSystem.parameters !== undefined) { if (definition.tradingSystem.parameters.baseAsset !== undefined) { if (definition.tradingSystem.parameters.baseAsset.formula !== undefined) { if (definition.tradingSystem.parameters.baseAsset.formula.code !== undefined) { - let code = JSON.parse(definition.tradingSystem.parameters.baseAsset.formula.code) - if (code.initialDatetime !== undefined) { - slippage = code.slippage + try { + let code = JSON.parse(definition.tradingSystem.parameters.baseAsset.formula.code) + if (code.slippage !== undefined) { + if (code.slippage.positionRate !== undefined) { + slippage.positionRate = code.slippage.positionRate + } + if (code.slippage.stopLoss !== undefined) { + slippage.stopLoss = code.slippage.stopLoss + } + if (code.slippage.takeProfit !== undefined) { + slippage.takeProfit = code.slippage.takeProfit + } + } + } catch (err) { + tradingSystem.parameters.baseAsset.formula.error = "Slippage Error: " + err.message + } + } + } + } + } + } + + /* Fee Structure */ + let feeStructure = { // Default Values + maker: 0, + taker: 0 + } + + if (definition.tradingSystem !== undefined) { + if (definition.tradingSystem.parameters !== undefined) { + if (definition.tradingSystem.parameters.baseAsset !== undefined) { + if (definition.tradingSystem.parameters.baseAsset.formula !== undefined) { + if (definition.tradingSystem.parameters.baseAsset.formula.code !== undefined) { + try { + let code = JSON.parse(definition.tradingSystem.parameters.baseAsset.formula.code) + if (code.feeStructure !== undefined) { + if (code.feeStructure.maker !== undefined) { + feeStructure.maker = code.feeStructure.maker + } + if (code.feeStructure.taker !== undefined) { + feeStructure.taker = code.feeStructure.taker + } + } + } catch (err) { + tradingSystem.parameters.baseAsset.formula.error = "Fee Structure Error: " + err.message } } } @@ -550,16 +595,10 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { /* Assistant Info */ - let ticker - - if (process.env.START_MODE === "live" && i === candles.length - 2) { - ticker = assistant.getTicker() - } else { - ticker = { - bid: candle.close, - ask: candle.close, - last: candle.close - } + let ticker = { + bid: candle.close, + ask: candle.close, + last: candle.close } //let LRC = LRCMap.get(candle.begin); @@ -1280,8 +1319,6 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { takeProfitStage = 'Open Stage'; stopLossPhase = 1; takeProfitPhase = 1; - currentTrade.begin = candle.begin; - currentTrade.beginRate = strategy.positionRate; if (processingDailyFiles) { if (positionedAtYesterday) { @@ -1290,8 +1327,6 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { yesterday.takeProfitStage = takeProfitStage; yesterday.stopLossPhase = stopLossPhase; yesterday.takeProfitPhase = takeProfitPhase; - yesterday.currentTrade.begin = currentTrade.begin; - yesterday.currentTrade.beginRate = currentTrade.beginRate; } } @@ -1583,20 +1618,23 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { if ((baseAsset === 'BTC' && candle.max >= stopLoss) || (baseAsset !== 'BTC' && candle.min <= stopLoss)) { - /* Check the Slipagge */ + /* + Hit Point Validation - let stopLossHitPoint = stopLoss + This prevents misscalculations when a formula places the stop loss in this case way beyond the market price. + If we take the stop loss value at those situation would be a huge distortion of facts. + */ if (baseAsset === 'BTC') { - if (stopLossHitPoint < candle.open) { stopLossHitPoint = candle.open} - if (stopLossHitPoint < candle.close) { stopLossHitPoint = candle.close } + if (stopLoss < candle.min) { stopLoss = candle.min } } else { - if (stopLossHitPoint > candle.open) { stopLossHitPoint = candle.open } - if (stopLossHitPoint > candle.close) { stopLossHitPoint = candle.close } + if (stopLoss > candle.max) { stopLoss = candle.max } } - let slippedStopLoss = stopLossHitPoint - let slippageAmount = stopLossHitPoint * slippage / 100 + let slippedStopLoss = stopLoss + + /* Apply the Slippage */ + let slippageAmount = slippedStopLoss * slippage.stopLoss / 100 if (baseAsset === 'BTC') { slippedStopLoss = slippedStopLoss + slippageAmount @@ -1604,17 +1642,25 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { slippedStopLoss = slippedStopLoss - slippageAmount } - if (baseAsset === 'BTC') { - strategy.positionSize = balanceAssetB / slippedStopLoss; - strategy.positionRate = slippedStopLoss; + let finalStopLoss = slippedStopLoss; - balanceAssetA = balanceAssetA + balanceAssetB / slippedStopLoss; + let feePaid = 0 + + if (baseAsset === 'BTC') { + strategy.positionSize = balanceAssetB / finalStopLoss; + strategy.positionRate = finalStopLoss; + + feePaid = balanceAssetB / finalStopLoss * feeStructure.taker / 100 + + balanceAssetA = balanceAssetA + balanceAssetB / finalStopLoss - feePaid; balanceAssetB = 0; } else { - strategy.positionSize = balanceAssetA * slippedStopLoss; - strategy.positionRate = slippedStopLoss; + strategy.positionSize = balanceAssetA * finalStopLoss; + strategy.positionRate = finalStopLoss; - balanceAssetB = balanceAssetB + balanceAssetA * slippedStopLoss; + feePaid = balanceAssetA * finalStopLoss * feeStructure.taker / 100 + + balanceAssetB = balanceAssetB + balanceAssetA * finalStopLoss - feePaid; balanceAssetA = 0; } @@ -1625,7 +1671,7 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } } - marketRate = slippedStopLoss; + marketRate = finalStopLoss; type = '"Close@StopLoss"'; strategyStage = 'Close Stage'; stopLossStage = 'No Stage'; @@ -1633,8 +1679,8 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { currentTrade.end = candle.end; currentTrade.status = 1; currentTrade.exitType = 1; - currentTrade.endRate = slippedStopLoss; - + currentTrade.endRate = finalStopLoss; + closePositionNow = true; if (processingDailyFiles) { @@ -1654,38 +1700,48 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { if ((baseAsset === 'BTC' && candle.min <= takeProfit) || (baseAsset !== 'BTC' && candle.max >= takeProfit)) { - /* Check the Slipagge */ + /* + Hit Point Validation: - let takeProfitHitPoint = stopLoss + This prevents misscalculations when a formula places the take profit in this case way beyond the market price. + If we take the stop loss value at those situation would be a huge distortion of facts. + */ if (baseAsset === 'BTC') { - if (takeProfitHitPoint > candle.open) { takeProfitHitPoint = candle.open } - if (takeProfitHitPoint > candle.close) { takeProfitHitPoint = candle.close } + if (takeProfit > candle.max) { takeProfit = candle.max } } else { - if (takeProfitHitPoint < candle.open) { takeProfitHitPoint = candle.open } - if (takeProfitHitPoint < candle.close) { takeProfitHitPoint = candle.close } + if (takeProfit < candle.min) { takeProfit = candle.min } } let slippedTakeProfit = takeProfit - let slippageAmount = stopLoss * slippage / 100 + /* Apply the Slippage */ + let slippageAmount = slippedTakeProfit * slippage.takeProfit / 100 if (baseAsset === 'BTC') { - slippedTakeProfit = slippedTakeProfit - slippageAmount - } else { slippedTakeProfit = slippedTakeProfit + slippageAmount + } else { + slippedTakeProfit = slippedTakeProfit - slippageAmount } - if (baseAsset === 'BTC') { - strategy.positionSize = balanceAssetB / slippedTakeProfit; - strategy.positionRate = slippedTakeProfit; + let finalTakeProfit = slippedTakeProfit; - balanceAssetA = balanceAssetA + balanceAssetB / slippedTakeProfit; + let feePaid = 0 + + if (baseAsset === 'BTC') { + strategy.positionSize = balanceAssetB / finalTakeProfit; + strategy.positionRate = finalTakeProfit; + + feePaid = balanceAssetB / finalTakeProfit * feeStructure.taker / 100 + + balanceAssetA = balanceAssetA + balanceAssetB / finalTakeProfit - feePaid; balanceAssetB = 0; } else { - strategy.positionSize = balanceAssetA * slippedTakeProfit; - strategy.positionRate = slippedTakeProfit; + strategy.positionSize = balanceAssetA * finalTakeProfit; + strategy.positionRate = finalTakeProfit; - balanceAssetB = balanceAssetB + balanceAssetA * slippedTakeProfit; + feePaid = balanceAssetA * finalTakeProfit * feeStructure.taker / 100 + + balanceAssetB = balanceAssetB + balanceAssetA * finalTakeProfit - feePaid; balanceAssetA = 0; } @@ -1697,7 +1753,7 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } } - marketRate = slippedTakeProfit; + marketRate = finalTakeProfit; type = '"Close@TakeProfit"'; strategyStage = 'Close Stage'; stopLossStage = 'No Stage'; @@ -1706,8 +1762,8 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { currentTrade.end = candle.end; currentTrade.status = 1; currentTrade.exitType = 2; - currentTrade.endRate = slippedTakeProfit; - + currentTrade.endRate = finalTakeProfit; + closePositionNow = true; if (processingDailyFiles) { @@ -1736,22 +1792,15 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { let strategy = tradingSystem.strategies[currentStrategyIndex]; tradePositionSize = strategy.positionSize; + tradePositionRate = strategy.positionRate; - if (i === candles.length - 2) { /* Only at the last closed candle I can take the value from the formula, since the use of ticker could alter the value at previous candles.*/ - tradePositionRate = strategy.positionRate; + /* We take what was calculated at the formula and apply the slippage. */ + let slippageAmount = tradePositionRate * slippage.positionRate / 100 + + if (baseAsset === 'BTC') { + tradePositionRate = tradePositionRate - slippageAmount } else { - - /* Check the Slipagge */ - - tradePositionRate = candle.close; - - let slippageAmount = tradePositionRate * slippage / 100 - - if (baseAsset === 'BTC') { - tradePositionRate = tradePositionRate - slippageAmount - } else { - tradePositionRate = tradePositionRate + slippageAmount - } + tradePositionRate = tradePositionRate + slippageAmount } if (processingDailyFiles) { @@ -1761,6 +1810,18 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } } + /* Update the trade record information. */ + currentTrade.begin = candle.begin; + currentTrade.beginRate = tradePositionRate; + + if (processingDailyFiles) { + if (positionedAtYesterday) { + yesterday.currentTrade.begin = currentTrade.begin; + yesterday.currentTrade.beginRate = currentTrade.beginRate; + } + } + + /* Check if we need to execute. */ if (i > candles.length - 10) { /* Only at the last candles makes sense to check if we are in live mode or not.*/ /* Check that we are in LIVE MODE */ if (process.env.START_MODE === "live") { @@ -1972,11 +2033,19 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { lastTradeProfitLoss = 0; lastTradeROI = 0; + let feePaid = 0 + if (baseAsset === 'BTC') { - balanceAssetB = balanceAssetB + tradePositionSize * tradePositionRate; + + feePaid = tradePositionSize * tradePositionRate * feeStructure.taker / 100 + + balanceAssetB = balanceAssetB + tradePositionSize * tradePositionRate - feePaid; balanceAssetA = balanceAssetA - tradePositionSize; } else { - balanceAssetA = balanceAssetA + tradePositionSize / tradePositionRate; + + feePaid = tradePositionSize / tradePositionRate * feeStructure.taker / 100 + + balanceAssetA = balanceAssetA + tradePositionSize / tradePositionRate - feePaid; balanceAssetB = balanceAssetB - tradePositionSize; } @@ -2257,8 +2326,6 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { if (processingDailyFiles) { if (positionedAtYesterday) { - yesterday.currentStrategyIndex = currentStrategyIndex; - yesterday.strategyStage = strategyStage; yesterday.stopLossStage = stopLossStage; yesterday.takeProfitStage = takeProfitStage; yesterday.stopLossPhase = stopLossPhase; @@ -2293,6 +2360,14 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { currentStrategyIndex = -1; strategyStage = 'No Stage'; + + if (processingDailyFiles) { + if (positionedAtYesterday) { + yesterday.currentStrategyIndex = currentStrategyIndex; + yesterday.strategyStage = strategyStage; + } + } + timerToCloseStage = 0 if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing the Closing Stage -> Exiting Close Stage."); } diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index bcfdd1727..b6dbc7661 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -54,8 +54,8 @@ } USER_BOT_MODULE = {} - //USER_BOT_MODULE.newUserBot = require(ROOT_DIR + 'User.Bot').newUserBot // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. - USER_BOT_MODULE.newUserBot = eval(text); // Use this for production + USER_BOT_MODULE.newUserBot = require(ROOT_DIR + 'User.Bot').newUserBot // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. + //USER_BOT_MODULE.newUserBot = eval(text); // Use this for production filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; filePath += "/Commons.js" @@ -70,8 +70,8 @@ return; } COMMONS_MODULE = {} - //COMMONS_MODULE.newCommons = require(ROOT_DIR + 'Commons').newCommons // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. - COMMONS_MODULE.newCommons = eval(text); // Use this for production + COMMONS_MODULE.newCommons = require(ROOT_DIR + 'Commons').newCommons // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. + //COMMONS_MODULE.newCommons = eval(text); // Use this for production callBackFunction(global.DEFAULT_OK_RESPONSE); } From 2f84c8e52b9d7cddd1fa2b7d7dd269a1e92d28fb Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 24 Aug 2019 09:47:01 +0200 Subject: [PATCH 07/32] Bug Fix on Trading Engine Code --- Commons.js | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Commons.js b/Commons.js index 54bbff3a8..bc88fe4cb 100644 --- a/Commons.js +++ b/Commons.js @@ -2156,23 +2156,17 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { closePositionRate = ticker.last + 100; // This is provisional and totally arbitrary, until we have a formula on the designer that defines this stuff. - amountA = interExecutionMemory.executionContext.amountA - amountB = interExecutionMemory.executionContext.amountA / closePositionRate + amountA = availableBalance.assetA + amountB = availableBalance.assetA / closePositionRate - if (amountA > availableBalance.assetA) { // The assistant know what fees were paid. - amountA = availableBalance.assetA - } } else { positionDirection = "sell" closePositionRate = ticker.last - 100; // This is provisional and totally arbitrary, until we have a formula on the designer that defines this stuff. - amountA = interExecutionMemory.executionContext.amountB * closePositionRate - amountB = interExecutionMemory.executionContext.amountB + amountA = availableBalance.assetB * closePositionRate + amountB = availableBalance.assetB - if (amountB > availableBalance.assetB) { // The assistant know what fees were paid. - amountB = availableBalance.assetB - } } interExecutionMemory.executionContext = { From 5960a9fc5a59a562b3a43104f442c75b681ea644 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 24 Aug 2019 14:54:07 +0200 Subject: [PATCH 08/32] Parameters Splitted in 4 --- Commons.js | 64 ++++++++++++++++++++++-------------------------- Run.js | 70 ++++++++++++++++++++++++++++------------------------- User.Bot.js | 3 +-- 3 files changed, 67 insertions(+), 70 deletions(-) diff --git a/Commons.js b/Commons.js index bc88fe4cb..2e6c26b92 100644 --- a/Commons.js +++ b/Commons.js @@ -171,25 +171,23 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } if (definition.tradingSystem !== undefined) { if (definition.tradingSystem.parameters !== undefined) { - if (definition.tradingSystem.parameters.baseAsset !== undefined) { - if (definition.tradingSystem.parameters.baseAsset.formula !== undefined) { - if (definition.tradingSystem.parameters.baseAsset.formula.code !== undefined) { - try { - let code = JSON.parse(definition.tradingSystem.parameters.baseAsset.formula.code) - if (code.slippage !== undefined) { - if (code.slippage.positionRate !== undefined) { - slippage.positionRate = code.slippage.positionRate - } - if (code.slippage.stopLoss !== undefined) { - slippage.stopLoss = code.slippage.stopLoss - } - if (code.slippage.takeProfit !== undefined) { - slippage.takeProfit = code.slippage.takeProfit - } - } - } catch (err) { - tradingSystem.parameters.baseAsset.formula.error = "Slippage Error: " + err.message + if (definition.tradingSystem.parameters.slippage !== undefined) { + if (definition.tradingSystem.parameters.slippage.code !== undefined) { + try { + let code = JSON.parse(definition.tradingSystem.parameters.slippage.code) + + if (code.positionRate !== undefined) { + slippage.positionRate = code.positionRate } + if (code.stopLoss !== undefined) { + slippage.stopLoss = code.stopLoss + } + if (code.takeProfit !== undefined) { + slippage.takeProfit = code.takeProfit + } + + } catch (err) { + tradingSystem.parameters.slippage.error = err.message } } } @@ -204,22 +202,20 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { if (definition.tradingSystem !== undefined) { if (definition.tradingSystem.parameters !== undefined) { - if (definition.tradingSystem.parameters.baseAsset !== undefined) { - if (definition.tradingSystem.parameters.baseAsset.formula !== undefined) { - if (definition.tradingSystem.parameters.baseAsset.formula.code !== undefined) { - try { - let code = JSON.parse(definition.tradingSystem.parameters.baseAsset.formula.code) - if (code.feeStructure !== undefined) { - if (code.feeStructure.maker !== undefined) { - feeStructure.maker = code.feeStructure.maker - } - if (code.feeStructure.taker !== undefined) { - feeStructure.taker = code.feeStructure.taker - } - } - } catch (err) { - tradingSystem.parameters.baseAsset.formula.error = "Fee Structure Error: " + err.message + if (definition.tradingSystem.parameters.feeStructure !== undefined) { + if (definition.tradingSystem.parameters.feeStructure.code !== undefined) { + try { + let code = JSON.parse(definition.tradingSystem.parameters.feeStructure.code) + + if (code.maker !== undefined) { + feeStructure.maker = code.maker } + if (code.taker !== undefined) { + feeStructure.taker = code.taker + } + + } catch (err) { + tradingSystem.parameters.feeStructure.error = err.message } } } @@ -2251,12 +2247,10 @@ exports.newCommons = function newCommons(bot, logger, UTILITIES) { } currentTrade.lastTradeROI = lastTradeROI; - currentTrade.stopRate = stopLoss; if (processingDailyFiles) { if (positionedAtYesterday) { yesterday.currentTrade.lastTradeROI = currentTrade.lastTradeROI; - yesterday.currentTrade.stopRate = currentTrade.stopRate; } } diff --git a/Run.js b/Run.js index 54bc7ec74..befb62319 100644 --- a/Run.js +++ b/Run.js @@ -27,9 +27,6 @@ process.on('exit', function (code) { console.log('[INFO] Run -> process.on.exit -> About to exit -> code = ' + code) }) -const definitionPath = path.resolve(process.env.INTER_PROCESS_FILES_PATH + '/definition.json'); -const definition = require(definitionPath); - let sequenceList = require('./sequence'); let isRunSequence = false; let sequenceStep = 0; @@ -53,7 +50,8 @@ if (isRunSequence) { readExecutionConfiguration() } -function sequenceExecution(currentStep) { +async function sequenceExecution(currentStep) { + let execution = sequenceList[currentStep]; process.env.STOP_GRACEFULLY = true; @@ -77,6 +75,9 @@ function sequenceExecution(currentStep) { execution.exchangeName ? global.EXCHANGE_NAME = execution.exchangeName : undefined; + + const definition = await strategy.getStrategy(); + if (definition) { if (definition.personalData) { if (definition.personalData.exchangeAccounts) { @@ -174,16 +175,18 @@ async function readExecutionConfiguration(execution) { } if (definition.tradingSystem !== undefined) { if (definition.tradingSystem.parameters !== undefined) { - if (definition.tradingSystem.parameters.baseAsset !== undefined) { - if (definition.tradingSystem.parameters.baseAsset.formula !== undefined) { - if (definition.tradingSystem.parameters.baseAsset.formula.code !== undefined) { - let code = JSON.parse(definition.tradingSystem.parameters.baseAsset.formula.code) + if (definition.tradingSystem.parameters.timeRange !== undefined) { + if (definition.tradingSystem.parameters.timeRange.code !== undefined) { + try { + let code = JSON.parse(definition.tradingSystem.parameters.timeRange.code) if (code.initialDatetime !== undefined) { initialDatetime = code.initialDatetime /* The second override occurs here, with the date explicitelly defined by the user */ } if (code.finalDatetime !== undefined) { finalDatetime = code.finalDatetime } + } catch (err) { + definition.tradingSystem.parameters.timeRange.error = err.message } } } @@ -195,34 +198,35 @@ async function readExecutionConfiguration(execution) { if (tradingSystem) { if (tradingSystem.parameters !== undefined) { if (tradingSystem.parameters.baseAsset !== undefined) { - if (tradingSystem.parameters.baseAsset.formula !== undefined) { - let receivedParameters - try { - receivedParameters = JSON.parse(tradingSystem.parameters.baseAsset.formula.code); + let code + try { + code = JSON.parse(tradingSystem.parameters.baseAsset.code); - if (receivedParameters.name !== undefined) { - baseAsset = receivedParameters.name; - if (baseAsset !== 'BTC' && baseAsset !== 'USDT') { - /* using BTC as default */ - baseAsset = 'BTC' - } + if (code.name !== undefined) { + baseAsset = code.name; + if (baseAsset !== 'BTC' && baseAsset !== 'USDT') { + /* using BTC as default */ + baseAsset = 'BTC' } - - if (baseAsset === 'BTC') { // NOTE: POLONIEX, the only exchange working so far, has Asset A and B inverted. We need to fix this. - if (receivedParameters.initialBalance !== undefined) { - process.env.INITIAL_BALANCE_ASSET_B = receivedParameters.initialBalance; - process.env.INITIAL_BALANCE_ASSET_A = 0 - } - } else { - if (receivedParameters.initialBalance !== undefined) { - process.env.INITIAL_BALANCE_ASSET_A = receivedParameters.initialBalance; - process.env.INITIAL_BALANCE_ASSET_B = 0 - } - } - } catch (err) { - process.env.INITIAL_BALANCE_ASSET_A = 0 // default - process.env.INITIAL_BALANCE_ASSET_B = 0.001 // default } + + if (baseAsset === 'BTC') { // NOTE: POLONIEX, the only exchange working so far, has Asset A and B inverted. We need to fix this. + if (code.initialBalance !== undefined) { + process.env.INITIAL_BALANCE_ASSET_B = code.initialBalance; + process.env.INITIAL_BALANCE_ASSET_A = 0 + } + } else { + if (code.initialBalance !== undefined) { + process.env.INITIAL_BALANCE_ASSET_A = code.initialBalance; + process.env.INITIAL_BALANCE_ASSET_B = 0 + } + } + } catch (err) { + definition.tradingSystem.parameters.baseAsset.error = err.message + + process.env.INITIAL_BALANCE_ASSET_A = 0 // default + process.env.INITIAL_BALANCE_ASSET_B = 0.001 // default + } } } diff --git a/User.Bot.js b/User.Bot.js index d646dd02a..88ddebd50 100644 --- a/User.Bot.js +++ b/User.Bot.js @@ -612,8 +612,7 @@ record.lastTradeROI + "," + record.beginRate + "," + record.endRate + "," + - record.exitType + "," + - record.stopRate + "]"; + record.exitType + "]"; if (separator === "") { separator = ","; } From 73b3685fe138b2af3588717b067e7809ae8603d9 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 24 Aug 2019 17:23:48 +0200 Subject: [PATCH 09/32] Removing the code of commons and bot from project --- Commons.js | 2976 ------------------- Integrations/{Strategy.js => Definition.js} | 14 +- Project.njsproj | 2 +- Run.js | 10 +- TradingEngineProcessMainLoop.js | 6 +- User.Bot.js | 674 ----- 6 files changed, 15 insertions(+), 3667 deletions(-) delete mode 100644 Commons.js rename Integrations/{Strategy.js => Definition.js} (73%) delete mode 100644 User.Bot.js diff --git a/Commons.js b/Commons.js deleted file mode 100644 index 2e6c26b92..000000000 --- a/Commons.js +++ /dev/null @@ -1,2976 +0,0 @@ -const strategy = require('./Integrations/Strategy') - -exports.newCommons = function newCommons(bot, logger, UTILITIES) { - - const { orderMessage } = require("@superalgos/mqservice") - - const { - MESSAGE_ENTITY, MESSAGE_TYPE, ORDER_CREATOR, ORDER_TYPE, - ORDER_OWNER, ORDER_DIRECTION, ORDER_STATUS, ORDER_EXIT_OUTCOME, - createMessage, getMessage, getExpandedMessage - } = orderMessage.newOrderMessage() - - const FULL_LOG = true; - const LOG_FILE_CONTENT = false; - - const MODULE_NAME = "Commons"; - const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; - const GMT_SECONDS = ':00.000 GMT+0000'; - - let thisObject = { - initializeData: initializeData, - runSimulation: runSimulation, - buildLRC: buildLRC, - buildPercentageBandwidthMap: buildPercentageBandwidthMap, - buildBollingerBandsMap: buildBollingerBandsMap, - buildBollingerChannelsArray: buildBollingerChannelsArray, - buildBollingerSubChannelsArray: buildBollingerSubChannelsArray, - buildCandles: buildCandles - }; - - let utilities = UTILITIES.newCloudUtilities(bot, logger); - - let LRCMap = new Map(); - let percentageBandwidthMap = new Map(); - let bollingerBandsMap = new Map(); - let bollingerChannelsArray = []; - let bollingerSubChannelsArray = []; - - let candles = []; - - return thisObject; - - function initializeData() { - - LRCMap = new Map(); - percentageBandwidthMap = new Map(); - bollingerBandsMap = new Map(); - bollingerChannelsArray = []; - bollingerSubChannelsArray = []; - - candles = []; - } - - async function runSimulation( - timePeriod, - currentDay, - startDate, - endDate, - interExecutionMemory, - assistant, - callback, - callBackFunction) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> Entering function."); } - - let processingDailyFiles = (currentDay !== undefined) - - let executionArray = []; - let recordsArray = []; - let conditionsArray = []; - let strategiesArray = []; - let tradesArray = []; - let lastObjectsArray = []; - - let definition = await strategy.getStrategy(); - let tradingSystem = definition.tradingSystem; - - /* Initial Default Values */ - - let initialDate = startDate; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> initialDate = " + initialDate); } - - const DEFAULT_BASE_ASSET_BALANCE = 1 - const DEFAULT_BASE_ASSET_MINIMUN_BALANCE = 0.5 - const DEFAULT_BASE_ASSET_MAXIMUN_BALANCE = 2 - - let initialBalanceA = DEFAULT_BASE_ASSET_BALANCE - let minimumBalanceA = DEFAULT_BASE_ASSET_MINIMUN_BALANCE - let maximumBalanceA = DEFAULT_BASE_ASSET_MAXIMUN_BALANCE - let initialBalanceB = 0 - let minimumBalanceB = 0 - let maximumBalanceB = 0 - let baseAsset = 'BTC' - - /* Parameters Processing */ - - if (tradingSystem.parameters !== undefined) { - if (tradingSystem.parameters.baseAsset !== undefined) { - if (tradingSystem.parameters.baseAsset.formula !== undefined) { - let receivedParameters - try { - receivedParameters = JSON.parse(tradingSystem.parameters.baseAsset.formula.code); - - if (receivedParameters.name !== undefined) { - baseAsset = receivedParameters.name; - if (baseAsset !== 'BTC' && baseAsset !== 'USDT') { - tradingSystem.parameters.baseAsset.formula.error = baseAsset + ' is not supported. Using default: BTC.' - baseAsset = 'BTC' - } - } - - if (baseAsset === 'BTC') { - if (receivedParameters.initialBalance !== undefined) { - initialBalanceA = receivedParameters.initialBalance; - initialBalanceB = 0 - } else { - initialBalanceA = DEFAULT_BASE_ASSET_BALANCE; - initialBalanceB = 0 - } - if (receivedParameters.minimumBalance !== undefined) { - minimumBalanceA = receivedParameters.minimumBalance; - minimumBalanceB = 0 - } else { - minimumBalanceA = DEFAULT_BASE_ASSET_MINIMUN_BALANCE; - minimumBalanceB = 0 - } - if (receivedParameters.maximumBalance !== undefined) { - maximumBalanceA = receivedParameters.maximumBalance; - maximumBalanceB = 0 - } else { - maximumBalanceA = DEFAULT_BASE_ASSET_MAXIMUN_BALANCE; - maximumBalanceB = 0 - } - } else { - if (receivedParameters.initialBalance !== undefined) { - initialBalanceB = receivedParameters.initialBalance; - initialBalanceA = 0 - } else { - initialBalanceB = DEFAULT_BASE_ASSET_BALANCE; - initialBalanceA = 0 - } - if (receivedParameters.minimumBalance !== undefined) { - minimumBalanceB = receivedParameters.minimumBalance; - minimumBalanceA = 0 - } else { - minimumBalanceB = DEFAULT_BASE_ASSET_MINIMUN_BALANCE; - minimumBalanceA = 0 - } - if (receivedParameters.maximumBalance !== undefined) { - maximumBalanceB = receivedParameters.maximumBalance; - maximumBalanceA = 0 - } else { - maximumBalanceB = DEFAULT_BASE_ASSET_MAXIMUN_BALANCE; - maximumBalanceA = 0 - } - } - } catch (err) { - tradingSystem.parameters.baseAsset.formula.error = err.message - } - } - } - } - - /* Slippage */ - let slippage = { // Default Values - positionRate: 0, - stopLoss: 0, - takeProfit: 0 - } - if (definition.tradingSystem !== undefined) { - if (definition.tradingSystem.parameters !== undefined) { - if (definition.tradingSystem.parameters.slippage !== undefined) { - if (definition.tradingSystem.parameters.slippage.code !== undefined) { - try { - let code = JSON.parse(definition.tradingSystem.parameters.slippage.code) - - if (code.positionRate !== undefined) { - slippage.positionRate = code.positionRate - } - if (code.stopLoss !== undefined) { - slippage.stopLoss = code.stopLoss - } - if (code.takeProfit !== undefined) { - slippage.takeProfit = code.takeProfit - } - - } catch (err) { - tradingSystem.parameters.slippage.error = err.message - } - } - } - } - } - - /* Fee Structure */ - let feeStructure = { // Default Values - maker: 0, - taker: 0 - } - - if (definition.tradingSystem !== undefined) { - if (definition.tradingSystem.parameters !== undefined) { - if (definition.tradingSystem.parameters.feeStructure !== undefined) { - if (definition.tradingSystem.parameters.feeStructure.code !== undefined) { - try { - let code = JSON.parse(definition.tradingSystem.parameters.feeStructure.code) - - if (code.maker !== undefined) { - feeStructure.maker = code.maker - } - if (code.taker !== undefined) { - feeStructure.taker = code.taker - } - - } catch (err) { - tradingSystem.parameters.feeStructure.error = err.message - } - } - } - } - } - - let timerToCloseStage = 0 - - /* Stop Loss Management */ - - const MIN_STOP_LOSS_VALUE = 1 // We can not let the stop be zero to avoid division by 0 error or infinity numbers as a result. - const MAX_STOP_LOSS_VALUE = Number.MAX_SAFE_INTEGER - let stopLoss = 0; - - /* Take Profit Management */ - - const MIN_TAKE_PROFIT_VALUE = 1 // We can not let the buy order be zero to avoid division by 0 error or infinity numbers as a result. - const MAX_TAKE_PROFIT_VALUE = Number.MAX_SAFE_INTEGER - let previousTakeProfit = 0; - let takeProfit = 0; - - /* Simulation Records */ - - let hitRatio = 0; - let ROI = 0; - let days = 0; - let positionDays = 0; - let anualizedRateOfReturn = 0; - let type = '""'; - let marketRate = 0; - let takePositionNow = false - let closePositionNow = false - - /* In some cases we need to know if we are positioned at the last candle of the calendar day, for that we need these variables. */ - - let lastInstantOfTheDay = currentDay.valueOf() + ONE_DAY_IN_MILISECONDS - 1; - let lastCandle = candles[candles.length - 1]; - - /* - The following counters need to survive multiple executions of the similator and keep themselves reliable. - This is challenging when the simulation is executed using Daily Files, since each execution means a new - day and these counters are meant to be kept along the whole market. - - To overcome this problem, we use the interExecutionMemory to record the values of the current execution - when finish. But there are a few details: - - 1. When the process is at the head of the market, it executes multple times at the same day. - 2. The same code serves execution from Market Files. - 3. In Daily Files we are receiving candles from the current day and previous day, so we need to take care of - not adding to the counters duplicate info when processing the day before candles. - - To overcome these challenges we record the values of the counters and variables on the interExecutionMemory only when - the day is complete and if we have a currentDay. That menas that for Market Files we will never use - interExecutionMemory. - */ - - /*Needed for statistics */ - let previousBalanceAssetA = 0; - let previousBalanceAssetB = 0; - - /* Position Management */ - let tradePositionRate = 0; - let tradePositionSize = 0; - - /* Strategy and Phase Management */ - let currentStrategyIndex = -1; - let strategyStage = 'No Stage'; - - let stopLossPhase = 0; - let stopLossStage = 'No Stage'; - - let takeProfitPhase = 0; - let takeProfitStage = 'No Stage'; - - /* These 2 objects will allow us to create separate files for each one of them. */ - - let currentStrategy = { - begin: 0, - end: 0, - status: 0, - number: 0, - beginRate: 0, - endRate: 0 - } - - let currentTrade = { - begin: 0, - end: 0, - status: 0, - profit: 0, - exitType: 0, - beginRate: 0, - endRate: 0 - } - - let balanceAssetA = initialBalanceA; - let balanceAssetB = initialBalanceB; - - let lastTradeProfitLoss = 0; - let profit = 0; - let lastTradeROI = 0; - - let roundtrips = 0; - let fails = 0; - let hits = 0; - let periods = 0; - let positionPeriods = 0; - - /* Message to the Simulation Executor */ - - let orderId = 0; - let messageId = 0; - - let yesterday = {}; - - /* Initialization */ - - yesterday.previousBalanceAssetA = previousBalanceAssetA - yesterday.previousBalanceAssetB = previousBalanceAssetB - - yesterday.tradePositionRate = tradePositionRate - yesterday.tradePositionSize = tradePositionSize - - yesterday.currentStrategyIndex = currentStrategyIndex - yesterday.strategyStage = strategyStage - - yesterday.stopLossPhase = stopLossPhase - yesterday.stopLossStage = stopLossStage - - yesterday.takeProfitPhase = takeProfitPhase - yesterday.takeProfitStage = takeProfitStage - - yesterday.currentStrategy = { - begin: 0, - end: 0, - status: 0, - number: 0, - beginRate: 0, - endRate: 0 - } - - yesterday.currentTrade = { - begin: 0, - end: 0, - status: 0, - profit: 0, - exitType: 0, - beginRate: 0, - endRate: 0 - } - - yesterday.balanceAssetA = balanceAssetA; - yesterday.balanceAssetB = balanceAssetB; - - yesterday.lastTradeProfitLoss = 0; - yesterday.profit = 0; - yesterday.lastTradeROI = 0; - - yesterday.roundtrips = 0; - yesterday.fails = 0; - yesterday.hits = 0; - yesterday.periods = 0; - yesterday.positionPeriods = 0; - - yesterday.orderId = 0; - yesterday.messageId = 0; - - yesterday.hitRatio = 0; - yesterday.ROI = 0; - yesterday.anualizedRateOfReturn = 0; - - if (interExecutionMemory.roundtrips === undefined) { - - /* Initialize the data structure we will use inter execution. */ - - interExecutionMemory.previousBalanceAssetA = previousBalanceAssetA - interExecutionMemory.previousBalanceAssetB = previousBalanceAssetB - - interExecutionMemory.tradePositionRate = tradePositionRate - interExecutionMemory.tradePositionSize = tradePositionSize - - interExecutionMemory.currentStrategyIndex = currentStrategyIndex - interExecutionMemory.strategyStage = strategyStage - - interExecutionMemory.stopLossPhase = stopLossPhase - interExecutionMemory.stopLossStage = stopLossStage - - interExecutionMemory.takeProfitPhase = takeProfitPhase - interExecutionMemory.takeProfitStage = takeProfitStage - - interExecutionMemory.currentStrategy = { - begin: 0, - end: 0, - status: 0, - number: 0, - beginRate: 0, - endRate: 0 - } - - interExecutionMemory.currentTrade = { - begin: 0, - end: 0, - status: 0, - profit: 0, - exitType: 0, - beginRate: 0, - endRate: 0 - } - - interExecutionMemory.balanceAssetA = balanceAssetA; - interExecutionMemory.balanceAssetB = balanceAssetB; - - interExecutionMemory.lastTradeProfitLoss = lastTradeProfitLoss; - interExecutionMemory.profit = profit; - interExecutionMemory.lastTradeROI = lastTradeROI; - - interExecutionMemory.roundtrips = 0; - interExecutionMemory.fails = 0; - interExecutionMemory.hits = 0; - interExecutionMemory.periods = 0; - interExecutionMemory.positionPeriods = 0; - - interExecutionMemory.orderId = 0; - interExecutionMemory.messageId = 0; - - interExecutionMemory.hitRatio = 0; - interExecutionMemory.ROI = 0; - interExecutionMemory.anualizedRateOfReturn = 0; - - } else { - - /* We get the initial values from the day previous to the candles we receive at the current execution */ - - previousBalanceAssetA = interExecutionMemory.previousBalanceAssetA - previousBalanceAssetB = interExecutionMemory.previousBalanceAssetB - - tradePositionRate = interExecutionMemory.tradePositionRate - tradePositionSize = interExecutionMemory.tradePositionSize - - currentStrategyIndex = interExecutionMemory.currentStrategyIndex - strategyStage = interExecutionMemory.strategyStage - - stopLossPhase = interExecutionMemory.stopLossPhase - stopLossStage = interExecutionMemory.stopLossStage - - takeProfitPhase = interExecutionMemory.takeProfitPhase - takeProfitStage = interExecutionMemory.takeProfitStage - - currentStrategy = { - begin: interExecutionMemory.currentStrategy.begin, - end: interExecutionMemory.currentStrategy.end, - status: interExecutionMemory.currentStrategy.status, - number: interExecutionMemory.currentStrategy.number, - beginRate: interExecutionMemory.currentStrategy.beginRate, - endRate: interExecutionMemory.currentStrategy.endRate - } - - currentTrade = { - begin: interExecutionMemory.currentTrade.begin, - end: interExecutionMemory.currentTrade.end, - status: interExecutionMemory.currentTrade.status, - profit: interExecutionMemory.currentTrade.profit, - exitType: interExecutionMemory.currentTrade.exitType, - beginRate: interExecutionMemory.currentTrade.beginRate, - endRate: interExecutionMemory.currentTrade.endRate - } - - if (currentDay.valueOf() >= startDate.valueOf() + ONE_DAY_IN_MILISECONDS) { // Only after the first day we start grabbing the balance from this memory. - - balanceAssetA = interExecutionMemory.balanceAssetA; - balanceAssetB = interExecutionMemory.balanceAssetB; - - yesterday.balanceAssetA = balanceAssetA; - yesterday.balanceAssetB = balanceAssetB; - - } - - lastTradeProfitLoss = interExecutionMemory.lastTradeProfitLoss; - profit = interExecutionMemory.profit; - lastTradeROI = interExecutionMemory.lastTradeROI; - - roundtrips = interExecutionMemory.roundtrips; - fails = interExecutionMemory.fails; - hits = interExecutionMemory.hits; - periods = interExecutionMemory.periods; - positionPeriods = interExecutionMemory.positionPeriods; - - orderId = interExecutionMemory.orderId; // to be deprecated - messageId = interExecutionMemory.messageId; // to be deprecated - - hitRatio = interExecutionMemory.hitRatio; - ROI = interExecutionMemory.ROI; - anualizedRateOfReturn = interExecutionMemory.anualizedRateOfReturn; - - /* For the case that any of these variables are not updated during the main loop, we need to store their value at the yesterday structure, otherwise it would be lost. */ - - yesterday.previousBalanceAssetA = previousBalanceAssetA - yesterday.previousBalanceAssetB = previousBalanceAssetB - - yesterday.tradePositionRate = tradePositionRate - yesterday.tradePositionSize = tradePositionSize - - yesterday.currentStrategyIndex = currentStrategyIndex; - yesterday.strategyStage = strategyStage; - - yesterday.stopLossPhase = stopLossPhase; - yesterday.stopLossStage = stopLossStage; - - yesterday.takeProfitPhase = takeProfitPhase; - yesterday.takeProfitStage = takeProfitStage; - - yesterday.currentStrategy = { - begin: currentStrategy.begin, - end: currentStrategy.end, - status: currentStrategy.status, - number: currentStrategy.number, - beginRate: currentStrategy.beginRate, - endRate: currentStrategy.endRate - } - - yesterday.currentTrade = { - begin: currentTrade.begin, - end: currentTrade.end, - status: currentTrade.status, - profit: currentTrade.profit, - exitType: currentTrade.exitType, - beginRate: currentTrade.beginRate, - endRate: currentTrade.endRate - } - - yesterday.lastTradeProfitLoss = lastTradeProfitLoss; - yesterday.profit = profit; - yesterday.lastTradeROI = lastTradeROI; - - yesterday.roundtrips = roundtrips; - yesterday.fails = fails; - yesterday.hits = hits; - yesterday.periods = periods; - yesterday.positionPeriods = positionPeriods; - - yesterday.hitRatio = hitRatio; - yesterday.ROI = ROI; - yesterday.anualizedRateOfReturn = anualizedRateOfReturn; - } - - /* Main Simulation Loop: We go thourgh all the candles at this time period. */ - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> Main Simulation Loop -> candles.length = " + candles.length); } - - let i - initializeLoop() - - function initializeLoop() { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> initializeLoop -> Entering function."); } - i = 0 - loop() - } - - function loop() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Entering function."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Processing candle # " + i); } - - let candle = candles[i]; - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Candle Begin @ " + (new Date(candle.begin)).toLocaleString()) } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Candle End @ " + (new Date(candle.end)).toLocaleString()) } - - let percentageBandwidth = percentageBandwidthMap.get(candle.begin); - let bollingerBand = bollingerBandsMap.get(candle.begin); - let bollingerChannel = getElement(bollingerChannelsArray, candle.begin, candle.end); - let bollingerSubChannel = getElement(bollingerSubChannelsArray, candle.begin, candle.end); - let positionedAtYesterday = (candle.end < currentDay.valueOf()) - - /* Assistant Info */ - - let ticker = { - bid: candle.close, - ask: candle.close, - last: candle.close - } - - //let LRC = LRCMap.get(candle.begin); - - /* If any of the needed indicators is missing, then that period is not calculated */ - - if (candle.end < initialDate.valueOf()) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle before the initialDate."); } - controlLoop(); - return - } - if (bollingerBand === undefined) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because Bollinger Band is undefined."); } - controlLoop(); - return - } - if (percentageBandwidth === undefined) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because %B is undefined."); } - controlLoop(); - return - } // percentageBandwidth might start after the first few candles. - if (bollingerChannel === undefined) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because Bollingeer Channel is undefined."); } - controlLoop(); - return - } - if (bollingerSubChannel === undefined) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because Bollingeer Sub Channel is undefined."); } - controlLoop(); - return - } - - //if (LRC === undefined) { controlLoop(); return} - - periods++; - days = periods * timePeriod / ONE_DAY_IN_MILISECONDS; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.periods = periods - } - - /* We skip the candle at the head of the market because i has not closed yet. */ - let candlesPerDay = ONE_DAY_IN_MILISECONDS / timePeriod - if (i === candles.length - 1) { - if ((candles.length < candlesPerDay) || (candles.length > candlesPerDay && candles.length < candlesPerDay * 2)) { - /*We are at the head of the market, thus we skip the last candle because it has not close yet. */ - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because it is the last one and has not been closed yet."); } - controlLoop(); - return - /* Note here that in the last candle of the first day or the second day it will use an incomplete candle and partially calculated indicators. - if we skip these two periods, then there will be a hole in the file since the last period will be missing. */ - } - } - - } else { // We are processing Market Files - if (i === candles.length - 1) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Skipping Candle because it is the last one and has not been closed yet."); } - controlLoop(); - return - } - } - - let lastObjects = { - candle: clone(candle), - //LRC: clone(LRC), - bollingerBand: clone(bollingerBand), - percentageBandwidth: clone(percentageBandwidth), - bollingerChannel: clone(bollingerChannel), - bollingerSubChannel: clone(bollingerSubChannel) - } - - function clone(obj) { - if (null == obj || "object" != typeof obj) return obj; - var copy = obj.constructor(); - for (var attr in obj) { - if (obj.hasOwnProperty(attr)) copy[attr] = obj[attr]; - } - return copy; - } - - lastObjects.candle.previous = undefined; - //lastObjects.LRC.previous = undefined; - lastObjects.bollingerBand.previous = undefined; - lastObjects.percentageBandwidth.previous = undefined; - lastObjects.bollingerChannel.previous = undefined; - lastObjects.bollingerSubChannel.previous = undefined; - - lastObjectsArray.push(lastObjects); - - if (lastObjectsArray.length > 5) { - lastObjectsArray.splice(0, 1); - } - - let conditions = new Map; // Here we store the conditions values that will be use in the simulator for decision making. - let formulas = new Map; - let conditionsArrayRecord = []; // These are the records that will be saved in a file for the plotter to consume. - let conditionsArrayValues = []; // Here we store the conditions values that will be written on file for the plotter. - let formulasErrors = []; // Here we store the errors produced by all phase formulas. - let formulasValues = []; // Here we store the values produced by all phase formulas. - - /* We define and evaluate all conditions to be used later during the simulation loop. */ - - conditionsArrayRecord.push(candle.begin); - conditionsArrayRecord.push(candle.end); - - evaluateConditionsAndFormulas(tradingSystem, conditions); - - function evaluateConditionsAndFormulas(tradingSystem, conditions) { - - for (let j = 0; j < tradingSystem.strategies.length; j++) { - - let strategy = tradingSystem.strategies[j]; - - let positionSize = 0 - let positionRate = 0 - - /* Continue with the rest of the formulas and conditions */ - - let triggerStage = strategy.triggerStage - - if (triggerStage !== undefined) { - - if (triggerStage.triggerOn !== undefined) { - - for (let k = 0; k < triggerStage.triggerOn.situations.length; k++) { - - let situation = triggerStage.triggerOn.situations[k]; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = j + '-' + 'triggerStage' + '-' + 'triggerOn' + '-' + k + '-' + m; - - if (condition.code !== undefined) { - newCondition(key, condition.code); - } - } - } - } - - if (triggerStage.triggerOff !== undefined) { - - for (let k = 0; k < triggerStage.triggerOff.situations.length; k++) { - - let situation = triggerStage.triggerOff.situations[k]; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = j + '-' + 'triggerStage' + '-' + 'triggerOff' + '-' + k + '-' + m; - - if (condition.code !== undefined) { - newCondition(key, condition.code); - } - } - } - } - - if (triggerStage.takePosition !== undefined) { - - for (let k = 0; k < triggerStage.takePosition.situations.length; k++) { - - let situation = triggerStage.takePosition.situations[k]; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = j + '-' + 'triggerStage' + '-' + 'takePosition' + '-' + k + '-' + m; - - if (condition.code !== undefined) { - newCondition(key, condition.code); - } - } - } - } - } - - let openStage = strategy.openStage - - if (openStage !== undefined) { - - /* Default Values*/ - if (baseAsset === 'BTC') { - positionSize = balanceAssetA; - positionRate = candle.close; - } else { - positionSize = balanceAssetB; - positionRate = candle.close; - } - - let initialDefinition = openStage.initialDefinition - - if (initialDefinition !== undefined) { - - if (tradePositionSize !== 0) { - positionSize = tradePositionSize - } else { - if (initialDefinition.positionSize !== undefined) { - if (initialDefinition.positionSize.formula !== undefined) { - try { - positionSize = eval(initialDefinition.positionSize.formula.code); - } catch (err) { - initialDefinition.positionSize.formula.error = err.message - } - if (isNaN(positionSize)) { - if (baseAsset === 'BTC') { - positionSize = balanceAssetA; - } else { - positionSize = balanceAssetB; - } - } else { - if (baseAsset === 'BTC') { - if (positionSize > balanceAssetA) { positionSize = balanceAssetA } - } else { - if (positionSize > balanceAssetB) { positionSize = balanceAssetB } - } - } - } - } - } - - if (tradePositionRate !== 0) { - positionRate = tradePositionRate - } else { - if (initialDefinition.positionRate !== undefined) { - if (initialDefinition.positionRate.formula !== undefined) { - try { - positionRate = eval(initialDefinition.positionRate.formula.code); - } catch (err) { - initialDefinition.positionRate.formula.error = err.message - } - if (isNaN(positionRate)) { - if (baseAsset === 'BTC') { - positionRate = candle.close; - } else { - positionRate = candle.close; - } - } - } - } - } - - if (initialDefinition.stopLoss !== undefined) { - - for (let p = 0; p < initialDefinition.stopLoss.phases.length; p++) { - - let phase = initialDefinition.stopLoss.phases[p]; - - /* Evaluate Formula */ - let formulaValue - let formulaError = '' - - if (phase.formula !== undefined) { - try { - formulaValue = eval(phase.formula.code); - if (formulaValue === Infinity) { - formulaError = "Formula evaluates to Infinity." - formulaValue = MAX_STOP_LOSS_VALUE - } - } catch (err) { - if (phase.formula.code.indexOf('previous') > 0 && err.message.indexOf('of undefined') > 0) { - /* - We are not going to set an error for the casess we are using previous and the error is that the indicator is undefined. - */ - } else { - formulaError = err.message - } - } - if (isNaN(formulaValue)) { formulaValue = 0; } - if (formulaValue < MIN_STOP_LOSS_VALUE) { - formulaValue = MIN_STOP_LOSS_VALUE - } - - formulasErrors.push('"' + formulaError + '"') - formulasValues.push(formulaValue) - let key = j + '-' + 'openStage' + '-' + 'initialDefinition' + '-' + 'stopLoss' + '-' + p; - formulas.set(key, formulaValue) - } - - /* next phase event */ - let nextPhaseEvent = phase.nextPhaseEvent; - if (nextPhaseEvent !== undefined) { - - for (let k = 0; k < nextPhaseEvent.situations.length; k++) { - - let situation = nextPhaseEvent.situations[k]; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = j + '-' + 'openStage' + '-' + 'initialDefinition' + '-' + 'stopLoss' + '-' + p + '-' + k + '-' + m; - - if (condition.code !== undefined) { - newCondition(key, condition.code); - } - } - } - } - } - } - - if (initialDefinition.takeProfit !== undefined) { - - for (let p = 0; p < initialDefinition.takeProfit.phases.length; p++) { - - let phase = initialDefinition.takeProfit.phases[p]; - - /* Evaluate Formula */ - let formulaValue - let formulaError = '' - - if (phase.formula !== undefined) { - try { - formulaValue = eval(phase.formula.code); - if (formulaValue === Infinity) { - formulaError = "Formula evaluates to Infinity." - formulaValue = MAX_TAKE_PROFIT_VALUE - } - } catch (err) { - if (phase.formula.code.indexOf('previous') > 0 && err.message.indexOf('of undefined') > 0) { - /* - We are not going to set an error for the casess we are using previous and the error is that the indicator is undefined. - */ - } else { - formulaError = err.message - } - } - if (isNaN(formulaValue)) { formulaValue = 0; } - if (formulaValue < MIN_TAKE_PROFIT_VALUE) { - formulaValue = MIN_TAKE_PROFIT_VALUE - } - - formulasErrors.push('"' + formulaError + '"') - formulasValues.push(formulaValue) - let key = j + '-' + 'openStage' + '-' + 'initialDefinition' + '-' + 'takeProfit' + '-' + p; - formulas.set(key, formulaValue) - } - - /* next phase event */ - let nextPhaseEvent = phase.nextPhaseEvent; - if (nextPhaseEvent !== undefined) { - - for (let k = 0; k < nextPhaseEvent.situations.length; k++) { - - let situation = nextPhaseEvent.situations[k]; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = j + '-' + 'openStage' + '-' + 'initialDefinition' + '-' + 'takeProfit' + '-' + p + '-' + k + '-' + m; - - if (condition.code !== undefined) { - newCondition(key, condition.code); - } - } - } - } - } - } - } - - strategy.positionSize = positionSize - strategy.positionRate = positionRate - } - - let manageStage = strategy.manageStage - - if (manageStage !== undefined) { - - if (manageStage.stopLoss !== undefined) { - - for (let p = 0; p < manageStage.stopLoss.phases.length; p++) { - - let phase = manageStage.stopLoss.phases[p]; - - /* Evaluate Formula */ - let formulaValue - let formulaError = '' - - if (phase.formula !== undefined) { - try { - formulaValue = eval(phase.formula.code); - if (formulaValue === Infinity) { - formulaError = "Formula evaluates to Infinity." - formulaValue = MAX_STOP_LOSS_VALUE - } - } catch (err) { - if (phase.formula.code.indexOf('previous') > 0 && err.message.indexOf('of undefined') > 0) { - /* - We are not going to set an error for the casess we are using previous and the error is that the indicator is undefined. - */ - } else { - formulaError = err.message - } - } - if (isNaN(formulaValue)) { formulaValue = 0; } - if (formulaValue < MIN_STOP_LOSS_VALUE) { - formulaValue = MIN_STOP_LOSS_VALUE - } - - formulasErrors.push('"' + formulaError + '"') - formulasValues.push(formulaValue) - let key = j + '-' + 'manageStage' + '-' + 'stopLoss' + '-' + p; - formulas.set(key, formulaValue) - } - - /* next phase event */ - let nextPhaseEvent = phase.nextPhaseEvent; - if (nextPhaseEvent !== undefined) { - - for (let k = 0; k < nextPhaseEvent.situations.length; k++) { - - let situation = nextPhaseEvent.situations[k]; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = j + '-' + 'manageStage' + '-' + 'stopLoss' + '-' + p + '-' + k + '-' + m; - - if (condition.code !== undefined) { - newCondition(key, condition.code); - } - } - } - } - } - } - - if (manageStage.takeProfit !== undefined) { - - for (let p = 0; p < manageStage.takeProfit.phases.length; p++) { - - let phase = manageStage.takeProfit.phases[p]; - - /* Evaluate Formula */ - let formulaValue - let formulaError = '' - - if (phase.formula !== undefined) { - try { - formulaValue = eval(phase.formula.code); - if (formulaValue === Infinity) { - formulaError = "Formula evaluates to Infinity." - formulaValue = MAX_TAKE_PROFIT_VALUE - } - } catch (err) { - if (phase.formula.code.indexOf('previous') > 0 && err.message.indexOf('of undefined') > 0) { - /* - We are not going to set an error for the casess we are using previous and the error is that the indicator is undefined. - */ - } else { - formulaError = err.message - } - } - if (isNaN(formulaValue)) { formulaValue = 0; } - if (formulaValue < MIN_TAKE_PROFIT_VALUE) { - formulaValue = MIN_TAKE_PROFIT_VALUE - } - - formulasErrors.push('"' + formulaError + '"') - formulasValues.push(formulaValue) - let key = j + '-' + 'manageStage' + '-' + 'takeProfit' + '-' + p; - formulas.set(key, formulaValue) - } - - /* next phase event */ - let nextPhaseEvent = phase.nextPhaseEvent; - if (nextPhaseEvent !== undefined) { - - for (let k = 0; k < nextPhaseEvent.situations.length; k++) { - - let situation = nextPhaseEvent.situations[k]; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = j + '-' + 'manageStage' + '-' + 'takeProfit' + '-' + p + '-' + k + '-' + m; - - if (condition.code !== undefined) { - newCondition(key, condition.code); - } - } - } - } - } - } - } - - function newCondition(key, node) { - - let condition; - let error = '' - let value - - try { - value = eval(node.code); - } catch (err) { - /* - One possible error is that the conditions references a .previous that is undefined. For this - reason and others, we will simply set the value to false. - */ - value = false - - if (node.code.indexOf('previous') > 0 && err.message.indexOf('of undefined') > 0) { - /* - We are not going to set an error for the casess we are using previous and the error is that the indicator is undefined. - */ - } else { - node.error = err.message - } - } - - condition = { - key: key, - value: value - }; - - conditions.set(condition.key, condition); - - if (condition.value) { - conditionsArrayValues.push(1); - } else { - conditionsArrayValues.push(0); - } - } - } - } - - /* Trigger On Conditions */ - if ( - strategyStage === 'No Stage' && - currentStrategyIndex === -1 - ) { - let minimumBalance - let maximumBalance - let balance - - if (baseAsset === 'BTC') { - balance = balanceAssetA - minimumBalance = minimumBalanceA - maximumBalance = maximumBalanceA - } else { - balance = balanceAssetB - minimumBalance = minimumBalanceB - maximumBalance = maximumBalanceB - } - - if (balance > minimumBalance && balance < maximumBalance) { - - /* - Here we need to pick a strategy, or if there is not suitable strategy for the current - market conditions, we pass until the next period. - - To pick a new strategy we will evaluate what we call the trigger on. Once we enter - into one strategy, we will ignore market conditions for others. However there is also - a strategy trigger off which can be hit before taking a position. If hit, we would - be outside a strategy again and looking for the condition to enter all over again. - - */ - - for (let j = 0; j < tradingSystem.strategies.length; j++) { - - let strategy = tradingSystem.strategies[j]; - - let triggerStage = strategy.triggerStage - - if (triggerStage !== undefined) { - - if (triggerStage.triggerOn !== undefined) { - - for (let k = 0; k < triggerStage.triggerOn.situations.length; k++) { - - let situation = triggerStage.triggerOn.situations[k]; - let passed = true; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = j + '-' + 'triggerStage' + '-' + 'triggerOn' + '-' + k + '-' + m; - - let value = false - if (conditions.get(key) !== undefined) { - value = conditions.get(key).value; - } - - if (value === false) { passed = false; } - } - - if (passed) { - - strategyStage = 'Trigger Stage'; - currentStrategyIndex = j; - currentStrategy.begin = candle.begin; - currentStrategy.beginRate = candle.min; - currentStrategy.endRate = candle.min; // In case the strategy does not get exited - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.strategyStage = strategyStage; - yesterday.currentStrategyIndex = currentStrategyIndex; - yesterday.currentStrategy.begin = currentStrategy.begin; - yesterday.currentStrategy.beginRate = currentStrategy.beginRate; - yesterday.currentStrategy.endRate = currentStrategy.endRate; - } - } - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Switching to Trigger Stage because conditions at Trigger On Event were met."); } - break; - } - } - } - } - } - } else { - let stopRunningDate = new Date(candle.begin) - if (balance < minimumBalance) { - tradingSystem.error = "Min Balance @ " + stopRunningDate.toUTCString() - } - if (balance > maximumBalance) { - tradingSystem.error = "Max Balance @ " + stopRunningDate.toUTCString() - } - } - } - - /* Trigger Off Condition */ - if (strategyStage === 'Trigger Stage') { - - let strategy = tradingSystem.strategies[currentStrategyIndex]; - - let triggerStage = strategy.triggerStage - - if (triggerStage !== undefined) { - - if (triggerStage.triggerOff !== undefined) { - - for (let k = 0; k < triggerStage.triggerOff.situations.length; k++) { - - let situation = triggerStage.triggerOff.situations[k]; - let passed = true; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = currentStrategyIndex + '-' + 'triggerStage' + '-' + 'triggerOff' + '-' + k + '-' + m; - - let value = false - if (conditions.get(key) !== undefined) { - value = conditions.get(key).value; - } - - if (value === false) { passed = false; } - } - - if (passed) { - - currentStrategy.number = currentStrategyIndex - currentStrategy.end = candle.end; - currentStrategy.endRate = candle.min; - currentStrategy.status = 1; - strategyStage = 'No Stage'; - currentStrategyIndex = -1; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.currentStrategy.number = currentStrategy.number; - yesterday.currentStrategy.end = currentStrategy.end; - yesterday.currentStrategy.endRate = currentStrategy.endRate; - yesterday.currentStrategy.status = currentStrategy.status; - yesterday.strategyStage = strategyStage; - yesterday.currentStrategyIndex = currentStrategyIndex; - } - } - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Switching to No Stage because conditions at the Trigger Off Event were met."); } - break; - } - } - } - } - } - - /* Take Position Condition */ - if (strategyStage === 'Trigger Stage') { - - let strategy = tradingSystem.strategies[currentStrategyIndex]; - - let triggerStage = strategy.triggerStage - - if (triggerStage !== undefined) { - - if (triggerStage.takePosition !== undefined) { - - for (let k = 0; k < triggerStage.takePosition.situations.length; k++) { - - let situation = triggerStage.takePosition.situations[k]; - let passed = true; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = currentStrategyIndex + '-' + 'triggerStage' + '-' + 'takePosition' + '-' + k + '-' + m; - - let value = false - if (conditions.get(key) !== undefined) { - value = conditions.get(key).value; - } - - if (value === false) { passed = false; } - } - - if (passed) { - - type = '"Take Position"'; - - strategyStage = 'Open Stage'; - stopLossStage = 'Open Stage'; - takeProfitStage = 'Open Stage'; - stopLossPhase = 1; - takeProfitPhase = 1; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.strategyStage = strategyStage; - yesterday.stopLossStage = stopLossStage; - yesterday.takeProfitStage = takeProfitStage; - yesterday.stopLossPhase = stopLossPhase; - yesterday.takeProfitPhase = takeProfitPhase; - } - } - - takePositionNow = true - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Conditions at the Take Position Event were met."); } - break; - } - } - } - } - } - - /* Stop Loss Management */ - if ( - (strategyStage === 'Open Stage' || strategyStage === 'Manage Stage') && - takePositionNow !== true - ) { - - checkStopPhases() - calculateStopLoss(); - - } - - function checkStopPhases() { - - let strategy = tradingSystem.strategies[currentStrategyIndex]; - - let openStage = strategy.openStage - let manageStage = strategy.manageStage - let parentNode - let j = currentStrategyIndex - let stageKey - let initialDefinitionKey = '' - let p - - if (stopLossStage === 'Open Stage' && openStage !== undefined) { - if (openStage.initialDefinition !== undefined) { - if (openStage.initialDefinition.stopLoss !== undefined) { - parentNode = openStage.initialDefinition - initialDefinitionKey = '-' + 'initialDefinition' - stageKey = 'openStage' - p = stopLossPhase - 1 - } - } - } - - if (stopLossStage === 'Manage Stage' && manageStage !== undefined) { - if (manageStage.stopLoss !== undefined) { - parentNode = manageStage - stageKey = 'manageStage' - p = stopLossPhase - 2 - } - } - - let phase = parentNode.stopLoss.phases[p]; - - let nextPhaseEvent = phase.nextPhaseEvent; - if (nextPhaseEvent !== undefined) { - - for (let k = 0; k < nextPhaseEvent.situations.length; k++) { - - let situation = nextPhaseEvent.situations[k]; - let passed = true; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = j + '-' + stageKey + initialDefinitionKey + '-' + 'stopLoss' + '-' + p + '-' + k + '-' + m; - - let value = false - if (conditions.get(key) !== undefined) { - value = conditions.get(key).value; - } - - if (value === false) { passed = false; } - } - - if (passed) { - - stopLossPhase++; - stopLossStage = 'Manage Stage' - if (takeProfitPhase > 1) { strategyStage = 'Manage Stage' } - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.stopLossPhase = stopLossPhase; - yesterday.stopLossStage = stopLossStage; - yesterday.strategyStage = strategyStage; - } - } - return; - } - } - } - } - - function calculateStopLoss() { - - let strategy = tradingSystem.strategies[currentStrategyIndex]; - let openStage = strategy.openStage - let manageStage = strategy.manageStage - let phase - let key - - if (stopLossStage === 'Open Stage' && openStage !== undefined) { - if (openStage.initialDefinition !== undefined) { - if (openStage.initialDefinition.stopLoss !== undefined) { - phase = openStage.initialDefinition.stopLoss.phases[stopLossPhase - 1]; - key = currentStrategyIndex + '-' + 'openStage' + '-' + 'initialDefinition' + '-' + 'stopLoss' + '-' + (stopLossPhase - 1); - } - } - } - - if (stopLossStage === 'Manage Stage' && manageStage !== undefined) { - if (manageStage.stopLoss !== undefined) { - phase = manageStage.stopLoss.phases[stopLossPhase - 2]; - key = currentStrategyIndex + '-' + 'manageStage' + '-' + 'stopLoss' + '-' + (stopLossPhase - 2); - } - } - - if (phase !== undefined) { - if (phase.formula !== undefined) { - stopLoss = formulas.get(key) - } - } - } - - /* Take Profit Management */ - if ( - (strategyStage === 'Open Stage' || strategyStage === 'Manage Stage') && - takePositionNow !== true - ) { - - checkTakeProfitPhases(); - calculateTakeProfit(); - - } - - function checkTakeProfitPhases() { - - let strategy = tradingSystem.strategies[currentStrategyIndex]; - - let openStage = strategy.openStage - let manageStage = strategy.manageStage - let parentNode - let j = currentStrategyIndex - let stageKey - let initialDefinitionKey = '' - let p - - if (takeProfitStage === 'Open Stage' && openStage !== undefined) { - if (openStage.initialDefinition !== undefined) { - if (openStage.initialDefinition.takeProfit !== undefined) { - parentNode = openStage.initialDefinition - initialDefinitionKey = '-' + 'initialDefinition' - stageKey = 'openStage' - p = takeProfitPhase - 1 - } - } - } - - if (takeProfitStage === 'Manage Stage' && manageStage !== undefined) { - if (manageStage.takeProfit !== undefined) { - parentNode = manageStage - stageKey = 'manageStage' - p = takeProfitPhase - 2 - } - } - - let phase = parentNode.takeProfit.phases[p]; - - let nextPhaseEvent = phase.nextPhaseEvent; - if (nextPhaseEvent !== undefined) { - - for (let k = 0; k < nextPhaseEvent.situations.length; k++) { - - let situation = nextPhaseEvent.situations[k]; - let passed = true; - - for (let m = 0; m < situation.conditions.length; m++) { - - let condition = situation.conditions[m]; - let key = j + '-' + stageKey + initialDefinitionKey + '-' + 'takeProfit' + '-' + p + '-' + k + '-' + m; - - let value = false - if (conditions.get(key) !== undefined) { - value = conditions.get(key).value; - } - - if (value === false) { passed = false; } - } - - if (passed) { - - takeProfitPhase++; - takeProfitStage = 'Manage Stage' - if (stopLossPhase > 1) { strategyStage = 'Manage Stage' } - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.takeProfitPhase = takeProfitPhase; - yesterday.takeProfitStage = takeProfitStage; - yesterday.strategyStage = strategyStage; - } - } - return; - } - } - } - } - - function calculateTakeProfit() { - - let strategy = tradingSystem.strategies[currentStrategyIndex]; - let openStage = strategy.openStage - let manageStage = strategy.manageStage - let phase - let key - - if (takeProfitStage === 'Open Stage' && openStage !== undefined) { - if (openStage.initialDefinition !== undefined) { - if (openStage.initialDefinition.takeProfit !== undefined) { - phase = openStage.initialDefinition.takeProfit.phases[takeProfitPhase - 1]; - key = currentStrategyIndex + '-' + 'openStage' + '-' + 'initialDefinition' + '-' + 'takeProfit' + '-' + (takeProfitPhase - 1); - } - } - } - - if (takeProfitStage === 'Manage Stage' && manageStage !== undefined) { - if (manageStage.takeProfit !== undefined) { - phase = manageStage.takeProfit.phases[takeProfitPhase - 2]; - key = currentStrategyIndex + '-' + 'manageStage' + '-' + 'takeProfit' + '-' + (takeProfitPhase - 2); - } - } - - if (phase !== undefined) { - if (phase.formula !== undefined) { - takeProfit = formulas.get(key) - } - } - } - - /* Keeping Position Counters Up-to-date */ - if ( - (strategyStage === 'Open Stage' || strategyStage === 'Manage Stage') - ) { - - if (takePositionNow === true) { - positionPeriods = 0 - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.positionPeriods = 0 - } - } - } - - positionPeriods++; - positionDays = positionPeriods * timePeriod / ONE_DAY_IN_MILISECONDS; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.positionPeriods = positionPeriods - } - } - } else { - positionPeriods = 0 - positionDays = 0 - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.positionPeriods = 0 - } - } - } - - /* Checking if Stop or Take Profit were hit */ - if ( - (strategyStage === 'Open Stage' || strategyStage === 'Manage Stage') && - takePositionNow !== true - ) { - let strategy = tradingSystem.strategies[currentStrategyIndex]; - - /* Checking what happened since the last execution. We need to know if the Stop Loss - or our Take Profit were hit. */ - - /* Stop Loss condition: Here we verify if the Stop Loss was hitted or not. */ - - if ((baseAsset === 'BTC' && candle.max >= stopLoss) || (baseAsset !== 'BTC' && candle.min <= stopLoss)) { - - /* - Hit Point Validation - - This prevents misscalculations when a formula places the stop loss in this case way beyond the market price. - If we take the stop loss value at those situation would be a huge distortion of facts. - */ - - if (baseAsset === 'BTC') { - if (stopLoss < candle.min) { stopLoss = candle.min } - } else { - if (stopLoss > candle.max) { stopLoss = candle.max } - } - - let slippedStopLoss = stopLoss - - /* Apply the Slippage */ - let slippageAmount = slippedStopLoss * slippage.stopLoss / 100 - - if (baseAsset === 'BTC') { - slippedStopLoss = slippedStopLoss + slippageAmount - } else { - slippedStopLoss = slippedStopLoss - slippageAmount - } - - let finalStopLoss = slippedStopLoss; - - let feePaid = 0 - - if (baseAsset === 'BTC') { - strategy.positionSize = balanceAssetB / finalStopLoss; - strategy.positionRate = finalStopLoss; - - feePaid = balanceAssetB / finalStopLoss * feeStructure.taker / 100 - - balanceAssetA = balanceAssetA + balanceAssetB / finalStopLoss - feePaid; - balanceAssetB = 0; - } else { - strategy.positionSize = balanceAssetA * finalStopLoss; - strategy.positionRate = finalStopLoss; - - feePaid = balanceAssetA * finalStopLoss * feeStructure.taker / 100 - - balanceAssetB = balanceAssetB + balanceAssetA * finalStopLoss - feePaid; - balanceAssetA = 0; - } - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.balanceAssetA = balanceAssetA; - yesterday.balanceAssetB = balanceAssetB; - } - } - - marketRate = finalStopLoss; - type = '"Close@StopLoss"'; - strategyStage = 'Close Stage'; - stopLossStage = 'No Stage'; - takeProfitStage = 'No Stage'; - currentTrade.end = candle.end; - currentTrade.status = 1; - currentTrade.exitType = 1; - currentTrade.endRate = finalStopLoss; - - closePositionNow = true; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.strategyStage = strategyStage; - yesterday.stopLossStage = stopLossStage; - yesterday.takeProfitStage = takeProfitStage; - yesterday.currentTrade.end = currentTrade.end; - yesterday.currentTrade.status = currentTrade.status; - yesterday.currentTrade.exitType = currentTrade.exitType; - yesterday.currentTrade.endRate = currentTrade.endRate; - } - } - } - - /* Take Profit condition: Here we verify if the Take Profit was hit or not. */ - - if ((baseAsset === 'BTC' && candle.min <= takeProfit) || (baseAsset !== 'BTC' && candle.max >= takeProfit)) { - - /* - Hit Point Validation: - - This prevents misscalculations when a formula places the take profit in this case way beyond the market price. - If we take the stop loss value at those situation would be a huge distortion of facts. - */ - - if (baseAsset === 'BTC') { - if (takeProfit > candle.max) { takeProfit = candle.max } - } else { - if (takeProfit < candle.min) { takeProfit = candle.min } - } - - let slippedTakeProfit = takeProfit - /* Apply the Slippage */ - let slippageAmount = slippedTakeProfit * slippage.takeProfit / 100 - - if (baseAsset === 'BTC') { - slippedTakeProfit = slippedTakeProfit + slippageAmount - } else { - slippedTakeProfit = slippedTakeProfit - slippageAmount - } - - let finalTakeProfit = slippedTakeProfit; - - let feePaid = 0 - - if (baseAsset === 'BTC') { - strategy.positionSize = balanceAssetB / finalTakeProfit; - strategy.positionRate = finalTakeProfit; - - feePaid = balanceAssetB / finalTakeProfit * feeStructure.taker / 100 - - balanceAssetA = balanceAssetA + balanceAssetB / finalTakeProfit - feePaid; - balanceAssetB = 0; - } else { - strategy.positionSize = balanceAssetA * finalTakeProfit; - strategy.positionRate = finalTakeProfit; - - feePaid = balanceAssetA * finalTakeProfit * feeStructure.taker / 100 - - balanceAssetB = balanceAssetB + balanceAssetA * finalTakeProfit - feePaid; - balanceAssetA = 0; - } - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.balanceAssetA = balanceAssetA; - yesterday.balanceAssetB = balanceAssetB; - - } - } - - marketRate = finalTakeProfit; - type = '"Close@TakeProfit"'; - strategyStage = 'Close Stage'; - stopLossStage = 'No Stage'; - takeProfitStage = 'No Stage'; - - currentTrade.end = candle.end; - currentTrade.status = 1; - currentTrade.exitType = 2; - currentTrade.endRate = finalTakeProfit; - - closePositionNow = true; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.strategyStage = strategyStage; - yesterday.stopLossStage = stopLossStage; - yesterday.takeProfitStage = takeProfitStage; - - yesterday.currentTrade.end = currentTrade.end; - yesterday.currentTrade.status = currentTrade.status; - yesterday.currentTrade.exitType = currentTrade.exitType; - yesterday.currentTrade.endRate = currentTrade.endRate; - } - } - } - } - - /* Taking a Position */ - if ( - takePositionNow === true - ) { - takePositionNow = false - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionNow -> Entering code block."); } - - /* Position size and rate */ - let strategy = tradingSystem.strategies[currentStrategyIndex]; - - tradePositionSize = strategy.positionSize; - tradePositionRate = strategy.positionRate; - - /* We take what was calculated at the formula and apply the slippage. */ - let slippageAmount = tradePositionRate * slippage.positionRate / 100 - - if (baseAsset === 'BTC') { - tradePositionRate = tradePositionRate - slippageAmount - } else { - tradePositionRate = tradePositionRate + slippageAmount - } - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.tradePositionSize = tradePositionSize; - yesterday.tradePositionRate = tradePositionRate; - } - } - - /* Update the trade record information. */ - currentTrade.begin = candle.begin; - currentTrade.beginRate = tradePositionRate; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.currentTrade.begin = currentTrade.begin; - yesterday.currentTrade.beginRate = currentTrade.beginRate; - } - } - - /* Check if we need to execute. */ - if (i > candles.length - 10) { /* Only at the last candles makes sense to check if we are in live mode or not.*/ - /* Check that we are in LIVE MODE */ - if (process.env.START_MODE === "live") { - /* We see if we need to put the actual order at the exchange. */ - if (interExecutionMemory.executionContext !== undefined) { - switch (interExecutionMemory.executionContext.status) { - case "Without a Position": { // We need to put the order because It was not put yet. - if (strategy.openStage !== undefined) { - if (strategy.openStage.openExecution !== undefined) { - putOpeningOrder() - return - } - } - break - } - case "Position Closed": { // Waiting for a confirmation that the position was closed. - if (strategy.openStage !== undefined) { - if (strategy.openStage.openExecution !== undefined) { - putOpeningOrder() - return - } - } - break - } - case "Taking Position": { // Waiting for a confirmation that the position was taken. - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionNow -> Exiting code block because status is Taking Position."); } - break - } - case "In a Position": { // This should mean that we already put the order at the exchange. - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionNow -> Exiting code block because status is In a Position."); } - break - } - } - } else { // The context does not exist so it means we are not in a position. - if (strategy.openStage !== undefined) { - if (strategy.openStage.openExecution !== undefined) { - putOpeningOrder() - return - } - } - } - } else { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionNow -> Not trading live."); } - } - } else { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionNow -> Not the last closed candle."); } - } - - takePositionAtSimulation() - return - - function putOpeningOrder() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> Entering function."); } - - /* We wont take a position unless we are withing the startDate and the endDate range */ - if (startDate !== undefined) { - if (candle.end < startDate.valueOf()) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> Not placing the trade at the exchange because current candle ends before the start date. -> startDate = " + startDate); } - takePositionAtSimulation() - return; - } - } - - /* This validation is disbled for now because we do not have the correct end date at this point. - if (endDate !== undefined) { - if (candle.begin > endDate.valueOf()) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> putOpeningOrder -> Not placing the trade at the exchange because current candle begins after the end date. -> endDate = " + endDate); } - takePositionAtSimulation() - return; - } - } - */ - - /* Checking the status of current positions */ - let positions = assistant.getPositions(); - if (positions.length > 0) { - let position = positions[positions.length - 1] // We are allways checking the the last position is not open. - if (position.status === 'open') { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> Not placing the trade at the exchange because the last position is still open. "); } - afterLoop(); - return - } - } - - /* Mechanism to avoid putting the same order over and over again at different executions of the simulation engine. */ - - if (interExecutionMemory.executionContext !== undefined) { - if (interExecutionMemory.executionContext.periods !== undefined) { - if (periods <= interExecutionMemory.executionContext.periods) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> Not placing the trade at the exchange because it was already placed at a previous execution."); } - takePositionAtSimulation() - return; - } - } - } - - /* We are not going to place orders based on outdated information. The next filter prevents firing orders when backtesting. */ - let today = new Date(Math.trunc((new Date().valueOf()) / ONE_DAY_IN_MILISECONDS) * ONE_DAY_IN_MILISECONDS) - let processDay = new Date(Math.trunc(currentDay.valueOf() / ONE_DAY_IN_MILISECONDS) * ONE_DAY_IN_MILISECONDS) - if (today.valueOf() !== processDay.valueOf()) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> Not placing the trade at the exchange because the current candle belongs to the previous day and that is considered simulation and not live trading."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> today = " + today); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> processDay = " + processDay); } - takePositionAtSimulation() - return; - } - - let openPositionRate - let amountA - let amountB - let positionDirection - let availableBalance = assistant.getAvailableBalance() - - if (baseAsset === 'BTC') { - positionDirection = "sell" - - openPositionRate = tradePositionRate - 100 // 100 is Provisional since we do have management of orders yet - - amountA = tradePositionSize * openPositionRate - amountB = tradePositionSize - - if (amountB > availableBalance.assetB) { // The assistant know what fees were paid. - amountB = availableBalance.assetB - } - } else { - positionDirection = "buy" - - openPositionRate = tradePositionRate + 100 // 100 is Provisional since we do have management of orders yet - - amountA = tradePositionSize - amountB = tradePositionSize / openPositionRate - - if (amountA > availableBalance.assetA) { // The assistant know what fees were paid. - amountA = availableBalance.assetA - } - } - - interExecutionMemory.executionContext = { - status: "Taking Position", - periods: periods, - } - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> Ready to put order."); } - assistant.putPosition(positionDirection, openPositionRate, amountA, amountB, onOrderPut) - - function onOrderPut(err) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> onOrderPut -> Entering function."); } - - try { - switch (err.result) { - case global.DEFAULT_OK_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> onOrderPut -> DEFAULT_OK_RESPONSE "); } - interExecutionMemory.executionContext = { - status: "In a Position", - periods: periods, - amountA: amountA, - amountB: amountB - } - takePositionAtSimulation() - return; - } - case global.DEFAULT_FAIL_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> onOrderPut -> DEFAULT_FAIL_RESPONSE "); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[ERROR] runSimulation -> loop -> putOpeningOrder -> onOrderPut -> Message = " + err.message); } - strategy.openStage.openExecution.error = err.message - afterLoop() - return; - } - case global.DEFAULT_RETRY_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> onOrderPut -> DEFAULT_RETRY_RESPONSE "); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[ERROR] runSimulation -> loop -> putOpeningOrder -> onOrderPut -> Message = " + err.message); } - strategy.openStage.openExecution.error = err.message - afterLoop() - return; - } - } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[ERROR] runSimulation -> loop -> putOpeningOrder -> onOrderPut -> Unexpected Response -> Message = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] runSimulation -> loop -> putOpeningOrder -> onOrderPut -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - } - } - } - - function takePositionAtSimulation() { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionAtSimulation -> Entering function."); } - - /* Continue with the simulation */ - calculateTakeProfit(); - calculateStopLoss(); - - marketRate = candle.close; - - previousBalanceAssetA = balanceAssetA; - previousBalanceAssetB = balanceAssetB; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.previousBalanceAssetA = previousBalanceAssetA; - yesterday.previousBalanceAssetB = previousBalanceAssetB; - } - } - - lastTradeProfitLoss = 0; - lastTradeROI = 0; - - let feePaid = 0 - - if (baseAsset === 'BTC') { - - feePaid = tradePositionSize * tradePositionRate * feeStructure.taker / 100 - - balanceAssetB = balanceAssetB + tradePositionSize * tradePositionRate - feePaid; - balanceAssetA = balanceAssetA - tradePositionSize; - } else { - - feePaid = tradePositionSize / tradePositionRate * feeStructure.taker / 100 - - balanceAssetA = balanceAssetA + tradePositionSize / tradePositionRate - feePaid; - balanceAssetB = balanceAssetB - tradePositionSize; - } - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.balanceAssetA = balanceAssetA; - yesterday.balanceAssetB = balanceAssetB; - - yesterday.lastTradeProfitLoss = lastTradeProfitLoss; - yesterday.lastTradeROI = lastTradeROI; - } - } - - addRecord(); - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> takePositionAtSimulation -> Exiting Loop Body after taking position at simulation."); } - controlLoop(); - return - } - } - - if (closePositionNow === true) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing a Position -> Entering code block."); } - - closePositionNow = false - - /* Position size and rate */ - let strategy = tradingSystem.strategies[currentStrategyIndex]; - - if (i > candles.length - 10) { /* Only at the last candles makes sense to check if we are in live mode or not.*/ - /* Check that we are in LIVE MODE */ - if (process.env.START_MODE === "live") { - /* We see if we need to put the actual order at the exchange. */ - if (interExecutionMemory.executionContext !== undefined) { - switch (interExecutionMemory.executionContext.status) { - case "Without a Position": { // No way to close anything at the exchange. - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing a Position -> Exiting code block because status is Without a Position."); } - break - } - case "In a Position": { // This should mean that we already put the order at the exchange. - if (strategy.closeStage !== undefined) { - if (strategy.closeStage.closeExecution !== undefined) { - putClosingOrder() - return - } - } - break - } - case "Closing Position": { // Waiting for a confirmation that the position was taken. - if (strategy.closeStage !== undefined) { - if (strategy.closeStage.closeExecution !== undefined) { - putClosingOrder() - return - } - } - break - } - - case "Position Closed": { // - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing a Position -> Exiting code block because status is Position Closed."); } - break - } - } - } - } - } else { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing a Position -> Not within the last 10 candles."); } - } - - closePositionAtSimulation() - return - - function putClosingOrder() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> putClosingOrder -> Entering function."); } - - /* Checking the status of current positions */ - let positions = assistant.getPositions(); - if (positions.length > 0) { - let position = positions[positions.length - 1] // We are allways checking the the last position is not open. - if (position.status === 'open') { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putClosingOrder -> Exiting function because status of last position is Open."); } - afterLoop(); - return - } - } - - /* Mechanism to avoid putting the same order over and over again at different executions of the simulation engine. */ - - if (interExecutionMemory.executionContext !== undefined) { - if (interExecutionMemory.executionContext.periods !== undefined) { - if (periods <= interExecutionMemory.executionContext.periods) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putClosingOrder -> Exiting function because this closing was already submited at a previous execution."); } - closePositionAtSimulation() - return; - } - } - } - - let closePositionRate - let amountA - let amountB - let positionDirection - let availableBalance = assistant.getAvailableBalance() - - if (baseAsset === 'BTC') { - positionDirection = "buy" - - closePositionRate = ticker.last + 100; // This is provisional and totally arbitrary, until we have a formula on the designer that defines this stuff. - - amountA = availableBalance.assetA - amountB = availableBalance.assetA / closePositionRate - - } else { - positionDirection = "sell" - - closePositionRate = ticker.last - 100; // This is provisional and totally arbitrary, until we have a formula on the designer that defines this stuff. - - amountA = availableBalance.assetB * closePositionRate - amountB = availableBalance.assetB - - } - - interExecutionMemory.executionContext = { - status: "Closing Position", - periods: periods, - } - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putClosingOrder -> About to close position at the exchange."); } - assistant.putPosition(positionDirection, closePositionRate, amountA, amountB, onOrderPut) - - function onOrderPut(err) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putClosingOrder -> onOrderPut -> Entering function."); } - - try { - switch (err.result) { - case global.DEFAULT_OK_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putClosingOrder -> onOrderPut -> DEFAULT_OK_RESPONSE "); } - interExecutionMemory.executionContext = { - status: "Position Closed", - periods: periods, - amountA: amountA, - amountB: amountB - } - closePositionAtSimulation() - return; - } - case global.DEFAULT_FAIL_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putClosingOrder -> onOrderPut -> DEFAULT_FAIL_RESPONSE "); } - /* We will assume that the problem is temporary, and expect that it will work at the next execution.*/ - strategy.closeStage.closeExecution.error = err.message - afterLoop() - return; - } - case global.DEFAULT_RETRY_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> putOpeningOrder -> onOrderPut -> DEFAULT_RETRY_RESPONSE "); } - strategy.closeStage.closeExecution.error = err.message - afterLoop() - return; - } - } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[ERROR] runSimulation -> loop -> putClosingOrder -> onOrderPut -> Unexpected Response -> Message = " + err.message); } - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] runSimulation -> loop -> putClosingOrder -> onOrderPut -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return - } - } - } - - function closePositionAtSimulation() { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> closePositionAtSimulation -> Entering function."); } - - roundtrips++; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.roundtrips = roundtrips - } - } - - if (baseAsset === 'BTC') { - lastTradeProfitLoss = balanceAssetA - previousBalanceAssetA; - lastTradeROI = lastTradeProfitLoss * 100 / tradePositionSize; - if (isNaN(lastTradeROI)) { lastTradeROI = 0; } - profit = balanceAssetA - initialBalanceA; - } else { - lastTradeProfitLoss = balanceAssetB - previousBalanceAssetB; - lastTradeROI = lastTradeProfitLoss * 100 / tradePositionSize; - if (isNaN(lastTradeROI)) { lastTradeROI = 0; } - profit = balanceAssetB - initialBalanceB; - } - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.lastTradeProfitLoss = lastTradeProfitLoss; - yesterday.profit = profit; - yesterday.lastTradeROI = lastTradeROI; - } - } - - currentTrade.lastTradeROI = lastTradeROI; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.currentTrade.lastTradeROI = currentTrade.lastTradeROI; - } - } - - if (lastTradeProfitLoss > 0) { - hits++; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.hits = hits - } - } - - } else { - fails++; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.fails = fails - } - } - } - - if (baseAsset === 'BTC') { - ROI = (initialBalanceA + profit) / initialBalanceA - 1; - hitRatio = hits / roundtrips; - anualizedRateOfReturn = ROI / days * 365; - } else { - ROI = (initialBalanceB + profit) / initialBalanceB - 1; - hitRatio = hits / roundtrips; - anualizedRateOfReturn = ROI / days * 365; - } - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.ROI = ROI; - yesterday.hitRatio = hitRatio; - yesterday.anualizedRateOfReturn = anualizedRateOfReturn; - } - } - - addRecord(); - - stopLoss = 0; - takeProfit = 0; - - tradePositionRate = 0; - tradePositionSize = 0; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.tradePositionSize = tradePositionSize; - yesterday.tradePositionRate = tradePositionRate; - } - } - - timerToCloseStage = candle.begin - stopLossStage = 'No Stage'; - takeProfitStage = 'No Stage'; - stopLossPhase = 0; - takeProfitPhase = 0; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.stopLossStage = stopLossStage; - yesterday.takeProfitStage = takeProfitStage; - yesterday.stopLossPhase = stopLossPhase; - yesterday.takeProfitPhase = takeProfitPhase; - } - } - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> closePositionAtSimulation -> Exiting Loop Body after closing position at simulation."); } - controlLoop(); - return - } - } - - /* Closing the Closing Stage */ - if (strategyStage === 'Close Stage') { - if (candle.begin - 5 * 60 * 1000 > timerToCloseStage) { - - currentStrategy.number = currentStrategyIndex - currentStrategy.end = candle.end; - currentStrategy.endRate = candle.min; - currentStrategy.status = 1; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - - yesterday.currentStrategy.number = currentStrategy.number; - yesterday.currentStrategy.end = currentStrategy.end; - yesterday.currentStrategy.endRate = currentStrategy.endRate; - yesterday.currentStrategy.status = currentStrategy.status; - } - } - - currentStrategyIndex = -1; - strategyStage = 'No Stage'; - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.currentStrategyIndex = currentStrategyIndex; - yesterday.strategyStage = strategyStage; - } - } - - timerToCloseStage = 0 - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing the Closing Stage -> Exiting Close Stage."); } - } else { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Closing the Closing Stage -> Waiting for timer."); } - } - } - - /* Not a buy or sell condition */ - - marketRate = candle.close; - addRecord(); - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> Exiting Loop Body after adding a record."); } - controlLoop(); - return - - function addRecord() { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> addRecord -> Entering function."); } - - // Since we are going to write the message to a file that the Simulation Executor is going to read, we use the abbreviations. - let messageType; - let message; - let simulationRecord; - let executionRecord; - let executionMessage; - - messageId++; - - if (strategyStage === 'Open Stage' || strategyStage === 'Manage Stage' || type === '"Close@TakeProfit"' || type === '"Close@StopLoss"') { - - if (type === '"Take Position"') { - messageType = MESSAGE_TYPE.Order; - orderId++; - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> addRecord -> Taking Position Now. "); } - - } else { - if (type === '"Close@TakeProfit"' || type === '"Close@StopLoss"') { - messageType = MESSAGE_TYPE.OrderClose; - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> loop -> addRecord -> Closing Position Now. "); } - - } else { - messageType = MESSAGE_TYPE.OrderUpdate; - } - } - - let exitOutcome = "" - if (messageType === MESSAGE_TYPE.OrderClose && type === '"Close@TakeProfit"') { - exitOutcome = "TP" - } - if (messageType === MESSAGE_TYPE.OrderClose && type === '"Close@StopLoss"') { - exitOutcome = "SL" - } - - executionMessage = createMessage( - messageId, - MESSAGE_ENTITY.SimulationEngine, - MESSAGE_ENTITY.SimulationExecutor, - messageType, - (new Date()).valueOf(), - orderId.toString(), - ORDER_CREATOR.SimulationEngine, - (new Date()).valueOf(), - ORDER_OWNER.User, - global.EXCHANGE_NAME, - "BTC_USDT", - 0, - ORDER_TYPE.Limit, - tradePositionRate, - stopLoss, - takeProfit, - ORDER_DIRECTION.Sell, - -1, - ORDER_STATUS.Signaled, - 0, - exitOutcome, - "") - - } - else { - - executionMessage = createMessage( - messageId, - MESSAGE_ENTITY.SimulationEngine, - MESSAGE_ENTITY.SimulationExecutor, - MESSAGE_TYPE.HeartBeat, - (new Date()).valueOf(), - "", - "", - 0, - "", - "", - "", - 0, - "", - 0, - 0, - 0, - "", - 0, - "", - 0, - "", - "") - } - - executionRecord = { - begin: candle.begin, - end: candle.end, - executionRecord: executionMessage - } - - executionArray.push(executionRecord) - - let strategyStageNumber - switch (strategyStage) { - case 'No Stage': { - strategyStageNumber = 0 - break - } - case 'Trigger Stage': { - strategyStageNumber = 1 - break - } - case 'Open Stage': { - strategyStageNumber = 2 - break - } - case 'Manage Stage': { - strategyStageNumber = 3 - break - } - case 'Close Stage': { - strategyStageNumber = 4 - break - } - } - - if (balanceAssetA === Infinity) { - balanceAssetA = Number.MAX_SAFE_INTEGER - } - - if (balanceAssetB === Infinity) { - balanceAssetB = Number.MAX_SAFE_INTEGER - } - - let quotedBaseAsset = '"' + baseAsset + '"' - - simulationRecord = { - begin: candle.begin, - end: candle.end, - type: type, - marketRate: marketRate, - amount: 1, - balanceA: balanceAssetA, - balanceB: balanceAssetB, - profit: profit, - lastTradeProfitLoss: lastTradeProfitLoss, - stopLoss: stopLoss, - roundtrips: roundtrips, - hits: hits, - fails: fails, - hitRatio: hitRatio, - ROI: ROI, - periods: periods, - days: days, - anualizedRateOfReturn: anualizedRateOfReturn, - positionRate: tradePositionRate, - lastTradeROI: lastTradeROI, - strategy: currentStrategyIndex, - strategyStageNumber: strategyStageNumber, - takeProfit: takeProfit, - stopLossPhase: stopLossPhase, - takeProfitPhase: takeProfitPhase, - executionRecord: executionMessage, - positionSize: tradePositionSize, - initialBalanceA: initialBalanceA, - minimumBalanceA: minimumBalanceA, - maximumBalanceA: maximumBalanceA, - initialBalanceB: initialBalanceB, - minimumBalanceB: minimumBalanceB, - maximumBalanceB: maximumBalanceB, - baseAsset: quotedBaseAsset, - positionPeriods: positionPeriods, - positionDays: positionDays - } - - recordsArray.push(simulationRecord); - - type = '""'; - - /* Prepare the information for the Conditions File */ - - conditionsArrayRecord.push(currentStrategyIndex); - conditionsArrayRecord.push(strategyStageNumber); - conditionsArrayRecord.push(stopLossPhase); - conditionsArrayRecord.push(takeProfitPhase); - conditionsArrayRecord.push(conditionsArrayValues); - conditionsArrayRecord.push(formulasErrors); - conditionsArrayRecord.push(formulasValues); - - conditionsArray.push(conditionsArrayRecord); - - /* Prepare the information for the Strategies File*/ - - if ( - (currentStrategy.begin !== 0 && currentStrategy.end !== 0) || - (currentStrategy.begin !== 0 && i === candles.length - 1 && lastCandle.end !== lastInstantOfTheDay) - ) { - - strategiesArray.push(currentStrategy); - - currentStrategy = { - begin: 0, - end: 0, - status: 0, - number: 0, - beginRate: 0, - endRate: 0 - } - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.currentStrategy = { - begin: 0, - end: 0, - status: 0, - number: 0, - beginRate: 0, - endRate: 0 - } - } - } - } - - /* Prepare the information for the Trades File */ - - if ( - (currentTrade.begin !== 0 && currentTrade.end !== 0) || - (currentTrade.begin !== 0 && i === candles.length - 1 && lastCandle.end !== lastInstantOfTheDay) - ) { - - currentTrade.profit = lastTradeProfitLoss; - - tradesArray.push(currentTrade); - - currentTrade = { - begin: 0, - end: 0, - status: 0, - lastTradeROI: 0, - exitType: 0, - beginRate: 0, - endRate: 0 - } - - if (processingDailyFiles) { - if (positionedAtYesterday) { - yesterday.currentTrade = { - begin: 0, - end: 0, - status: 0, - lastTradeROI: 0, - exitType: 0, - beginRate: 0, - endRate: 0 - } - } - } - } - } - } - - function controlLoop() { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> controlLoop -> Entering function."); } - i++ - if (i < candles.length) { - process.nextTick(loop) - } else { - afterLoop() - } - } - - function afterLoop() { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> afterLoop -> Entering function."); } - - /* - Before returning we need to see if we have to record some of our counters at the interExecutionMemory. - To do that, the condition to be met is that this execution must include all candles of the currentDay. - */ - - if (processingDailyFiles) { - - if (lastCandle.end === lastInstantOfTheDay) { - - interExecutionMemory.previousBalanceAssetA = yesterday.previousBalanceAssetA - interExecutionMemory.previousBalanceAssetB = yesterday.previousBalanceAssetB - - interExecutionMemory.tradePositionRate = yesterday.tradePositionRate - interExecutionMemory.tradePositionSize = yesterday.tradePositionSize - - interExecutionMemory.currentStrategyIndex = yesterday.currentStrategyIndex - interExecutionMemory.strategyStage = yesterday.strategyStage - - interExecutionMemory.stopLossPhase = yesterday.stopLossPhase - interExecutionMemory.stopLossStage = yesterday.stopLossStage - - interExecutionMemory.takeProfitPhase = yesterday.takeProfitPhase - interExecutionMemory.takeProfitStage = yesterday.takeProfitStage - - interExecutionMemory.currentStrategy = { - begin: yesterday.currentStrategy.begin, - end: yesterday.currentStrategy.end, - status: yesterday.currentStrategy.status, - number: yesterday.currentStrategy.number, - beginRate: yesterday.currentStrategy.beginRate, - endRate: yesterday.currentStrategy.endRate - } - - interExecutionMemory.currentTrade = { - begin: yesterday.currentTrade.begin, - end: yesterday.currentTrade.end, - status: yesterday.currentTrade.status, - profit: yesterday.currentTrade.profit, - exitType: yesterday.currentTrade.exitType, - beginRate: yesterday.currentTrade.beginRate, - endRate: yesterday.currentTrade.endRate - } - - interExecutionMemory.balanceAssetA = yesterday.balanceAssetA; - interExecutionMemory.balanceAssetB = yesterday.balanceAssetB; - interExecutionMemory.lastTradeProfitLoss = yesterday.lastTradeProfitLoss; - interExecutionMemory.profit = yesterday.profit; - interExecutionMemory.lastTradeROI = yesterday.lastTradeROI; - - interExecutionMemory.roundtrips = yesterday.roundtrips; - interExecutionMemory.fails = yesterday.fails; - interExecutionMemory.hits = yesterday.hits; - interExecutionMemory.periods = yesterday.periods; - interExecutionMemory.positionPeriods = yesterday.positionPeriods; - - interExecutionMemory.messageId = interExecutionMemory.messageId + yesterday.messageId; - interExecutionMemory.orderId = interExecutionMemory.orderId + yesterday.orderId; - - interExecutionMemory.hitRatio = yesterday.hitRatio; - interExecutionMemory.ROI = yesterday.ROI; - interExecutionMemory.anualizedRateOfReturn = yesterday.anualizedRateOfReturn; - } - } - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] runSimulation -> callback -> recordsArray.length = " + recordsArray.length); } - - callback(tradingSystem, executionArray, recordsArray, conditionsArray, strategiesArray, tradesArray, lastObjectsArray); - } - - - function getElement(pArray, begin, end) { - - let element; - - for (let i = 0; i < pArray.length; i++) { - - element = pArray[i]; - - if (begin >= element.begin && end <= element.end) { - return element - } - } - - return - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] runSimulation -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function buildLRC(dataFile, callBackFunction) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] buildLRC -> Entering function."); } - - try { - - let previous; - - for (let i = 0; i < dataFile.length; i++) { - - let LRC = { - begin: dataFile[i][0], - end: dataFile[i][1], - _15: dataFile[i][2], - _30: dataFile[i][3], - _60: dataFile[i][4] - }; - - if (previous !== undefined) { - - if (previous._15 > LRC._15) { LRC.direction15 = 'Down'; } - if (previous._15 < LRC._15) { LRC.direction15 = 'Up'; } - if (previous._15 === LRC._15) { LRC.direction15 = 'Side'; } - - if (previous._30 > LRC._30) { LRC.direction30 = 'Down'; } - if (previous._30 < LRC._30) { LRC.direction30 = 'Up'; } - if (previous._30 === LRC._30) { LRC.direction30 = 'Side'; } - - if (previous._60 > LRC._60) { LRC.direction60 = 'Down'; } - if (previous._60 < LRC._60) { LRC.direction60 = 'Up'; } - if (previous._60 === LRC._60) { LRC.direction60 = 'Side'; } - - } - - LRC.previous = previous; - - LRCMap.set(LRC.begin, LRC); - - previous = LRC; - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] buildLRC -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function buildPercentageBandwidthMap(dataFile, callBackFunction) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] buildPercentageBandwidthMap -> Entering function."); } - - try { - - let previous; - - for (let i = 0; i < dataFile.length; i++) { - - let percentageBandwidth = { - begin: dataFile[i][0], - end: dataFile[i][1], - value: dataFile[i][2], - movingAverage: dataFile[i][3], - bandwidth: dataFile[i][4] - }; - - if (previous !== undefined) { - - if (previous.movingAverage > percentageBandwidth.movingAverage) { percentageBandwidth.direction = 'Down'; } - if (previous.movingAverage < percentageBandwidth.movingAverage) { percentageBandwidth.direction = 'Up'; } - if (previous.movingAverage === percentageBandwidth.movingAverage) { percentageBandwidth.direction = 'Side'; } - - } - - percentageBandwidth.previous = previous; - - percentageBandwidthMap.set(percentageBandwidth.begin, percentageBandwidth); - - previous = percentageBandwidth; - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] buildPercentageBandwidthMap -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function buildBollingerBandsMap(dataFile, callBackFunction) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] buildBollingerBandsMap -> Entering function."); } - - try { - - let previous; - - for (let i = 0; i < dataFile.length; i++) { - - let bollingerBand = { - begin: dataFile[i][0], - end: dataFile[i][1], - movingAverage: dataFile[i][2], - standardDeviation: dataFile[i][3], - deviation: dataFile[i][4] - }; - - if (previous !== undefined) { - - if (previous.movingAverage > bollingerBand.movingAverage) { bollingerBand.direction = 'Down'; } - if (previous.movingAverage < bollingerBand.movingAverage) { bollingerBand.direction = 'Up'; } - if (previous.movingAverage === bollingerBand.movingAverage) { bollingerBand.direction = 'Side'; } - - } - - bollingerBand.previous = previous; - - bollingerBandsMap.set(bollingerBand.begin, bollingerBand); - - previous = bollingerBand; - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] buildBollingerBandsMap -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function buildBollingerChannelsArray(dataFile, callBackFunction) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] buildBollingerChannelsArray -> Entering function."); } - - try { - - let previous; - - for (let i = 0; i < dataFile.length; i++) { - - let bollingerChannel = { - begin: dataFile[i][0], - end: dataFile[i][1], - direction: dataFile[i][2], - period: dataFile[i][3] - }; - - bollingerChannel.previous = previous; - - bollingerChannelsArray.push(bollingerChannel); - - previous = bollingerChannel; - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] buildBollingerChannelsArray -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function buildBollingerSubChannelsArray(dataFile, callBackFunction) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] buildBollingerSubChannelsArray -> Entering function."); } - - try { - - let previous; - - for (let i = 0; i < dataFile.length; i++) { - - let bollingerSubChannel = { - begin: dataFile[i][0], - end: dataFile[i][1], - direction: dataFile[i][2], - slope: dataFile[i][3], - period: dataFile[i][4], - firstMovingAverage: dataFile[i][5], - lastMovingAverage: dataFile[i][6], - firstDeviation: dataFile[i][7], - lastDeviation: dataFile[i][8] - }; - - bollingerSubChannel.previous = previous; - - bollingerSubChannelsArray.push(bollingerSubChannel); - - previous = bollingerSubChannel; - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] buildBollingerSubChannelsArray -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function buildCandles(dataFile, callBackFunction) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] buildCandles -> Entering function."); } - - try { - - let previous; - - for (let i = 0; i < dataFile.length; i++) { - - let candle = { - open: undefined, - close: undefined, - min: 10000000000000, - max: 0, - begin: undefined, - end: undefined, - direction: undefined - }; - - candle.min = dataFile[i][0]; - candle.max = dataFile[i][1]; - - candle.open = dataFile[i][2]; - candle.close = dataFile[i][3]; - - candle.begin = dataFile[i][4]; - candle.end = dataFile[i][5]; - - if (candle.open > candle.close) { candle.direction = 'Down'; } - if (candle.open < candle.close) { candle.direction = 'Up'; } - if (candle.open === candle.close) { candle.direction = 'Side'; } - - candle.previous = previous; - - candles.push(candle); - - previous = candle; - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] buildCandles -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } -}; - - - diff --git a/Integrations/Strategy.js b/Integrations/Definition.js similarity index 73% rename from Integrations/Strategy.js rename to Integrations/Definition.js index a54b937b9..3dbbc46eb 100644 --- a/Integrations/Strategy.js +++ b/Integrations/Definition.js @@ -3,9 +3,9 @@ let fs = require('fs') const { promisify } = require('util') const readFileAsync = promisify(fs.readFile) -exports.getStrategy = async function () { +exports.getDefinition = async function () { try { - if (!global.STRATEGY) { + if (!global.DEFINITION) { let executionOnCloud = false if (process.env.ON_CLOUD !== undefined) { executionOnCloud = JSON.parse(process.env.ON_CLOUD) @@ -36,16 +36,16 @@ exports.getStrategy = async function () { throw new Error(strategizerResponse.data.errors[0].message) } - global.STRATEGY = strategizerResponse.data.data.strategizer_TradingSystemByFb.data + global.DEFINITION = strategizerResponse.data.data.strategizer_TradingSystemByFb.data } else { let fileLocation = process.env.INTER_PROCESS_FILES_PATH + '/definition.json' - let strategy = await readFileAsync(fileLocation, { encoding: 'utf8' }) - global.STRATEGY = JSON.parse(strategy) + let definition = readFile(fileLocation, { encoding: 'utf8' }) + global.DEFINITION = JSON.parse(definition) } } - return global.STRATEGY + return global.DEFINITION } catch (error) { - throw new Error('There has been an error getting the strategy: ', JSON.stringify(error)) + throw new Error('There has been an error getting the definition: ', JSON.stringify(error)) } } diff --git a/Project.njsproj b/Project.njsproj index 53091cd23..d4b026ae4 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -26,7 +26,7 @@ - + Code diff --git a/Run.js b/Run.js index befb62319..4270d3cd4 100644 --- a/Run.js +++ b/Run.js @@ -1,5 +1,7 @@ require('dotenv').config(); -const strategy = require('./Integrations/Strategy'); + +global.DEFINITION = require(process.env.INTER_PROCESS_FILES_PATH + '/Definition'); +const definition = global.DEFINITION const path = require('path'); global.SHALL_BOT_STOP = false @@ -75,9 +77,6 @@ async function sequenceExecution(currentStep) { execution.exchangeName ? global.EXCHANGE_NAME = execution.exchangeName : undefined; - - const definition = await strategy.getStrategy(); - if (definition) { if (definition.personalData) { if (definition.personalData.exchangeAccounts) { @@ -133,9 +132,6 @@ async function readExecutionConfiguration(execution) { let timePeriodFilter let botProcess - /* Try to get the begin and end dates from the Definition */ - let definition = await strategy.getStrategy() - /* Dates are taken initially from .env, but can be overwritten if they are defined by the user */ let initialDatetime = process.env.BEGIN_DATE_TIME let finalDatetime = process.env.END_DATE_TIME diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index b6dbc7661..1d59a68d9 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -11,6 +11,7 @@ const MULTI_PERIOD_MARKET = require(ROOT_DIR + 'MultiPeriodMarket'); const MULTI_PERIOD_DAILY = require(ROOT_DIR + 'MultiPeriodDaily'); const FILE_STORAGE = require('./Integrations/FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(parentLogger); const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); @@ -54,7 +55,7 @@ } USER_BOT_MODULE = {} - USER_BOT_MODULE.newUserBot = require(ROOT_DIR + 'User.Bot').newUserBot // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. + USER_BOT_MODULE.newUserBot = require(process.env.BOTS_PATH + '/aamasters/AAMasters/bots/AAJason-Trading-Engine-Bot/Multi-Period-Daily/User.Bot').newUserBot // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. //USER_BOT_MODULE.newUserBot = eval(text); // Use this for production filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; @@ -69,8 +70,9 @@ callBackFunction(global.DEFAULT_OK_RESPONSE); return; } + COMMONS_MODULE = {} - COMMONS_MODULE.newCommons = require(ROOT_DIR + 'Commons').newCommons // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. + COMMONS_MODULE.newCommons = require(process.env.BOTS_PATH + '/aamasters/AAMasters/bots/AAJason-Trading-Engine-Bot/Commons').newCommons // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. //COMMONS_MODULE.newCommons = eval(text); // Use this for production callBackFunction(global.DEFAULT_OK_RESPONSE); diff --git a/User.Bot.js b/User.Bot.js deleted file mode 100644 index 88ddebd50..000000000 --- a/User.Bot.js +++ /dev/null @@ -1,674 +0,0 @@ -exports.newUserBot = function newUserBot(bot, logger, COMMONS, UTILITIES, fileStorage) { - - const FULL_LOG = true; - const LOG_FILE_CONTENT = false; - - const MODULE_NAME = "User Bot"; - - const EXECUTION_FOLDER_NAME = "Trading-Execution"; - const SIMULATED_RECORDS_FOLDER_NAME = "Trading-Simulation"; - const CONDITIONS_FOLDER_NAME = "Simulation-Conditions"; - const STRATEGIES_FOLDER_NAME = "Simulation-Strategies"; - const TRADES_FOLDER_NAME = "Simulation-Trades"; - - const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; - - const commons = COMMONS.newCommons(bot, logger, UTILITIES); - - thisObject = { - initialize: initialize, - start: start - }; - - let utilities = UTILITIES.newCloudUtilities(bot, logger); - - let dataDependencies; - let assistant; - - return thisObject; - - function initialize(pDataDependencies, callBackFunction, pAssistant) { - - try { - - logger.fileName = MODULE_NAME; - logger.initialize(); - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } - - dataDependencies = pDataDependencies; - assistant = pAssistant; - - callBackFunction(global.DEFAULT_OK_RESPONSE); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function start(dataFiles, timePeriod, outputPeriodLabel, currentDay, startDate, endDate, interExecutionMemory, callBackFunction) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); } - - let market = global.MARKET; - let dataFile; - - let recordsArray - let conditionsArray - let strategiesArray - let tradesArray - let lastObjectsArray - - let tradingSystem = {}; - - commons.initializeData(); - - for (let i = 0; i < dataDependencies.config.length; i++) { - - let dependency = dataDependencies.config[i]; - dataFile = dataFiles[i]; - - switch (i) { - /* - case 0: { - commons.buildLRC(dataFile, callBackFunction); - break; - }*/ - case 0: { - commons.buildPercentageBandwidthMap(dataFile, callBackFunction); - break; - } - case 1: { - commons.buildBollingerBandsMap(dataFile, callBackFunction); - break; - } - case 2: { - commons.buildBollingerChannelsArray(dataFile, callBackFunction); - break; - } - case 3: { - commons.buildBollingerSubChannelsArray(dataFile, callBackFunction); - break; - } - case 4: { - commons.buildCandles(dataFile, callBackFunction); - break; - } - } - } - - commons.runSimulation( - timePeriod, - currentDay, - startDate, - endDate, - interExecutionMemory, - assistant, - writeFiles, - callBackFunction) - - function writeFiles(pTradingSystem, pExecutionArray, pRecordsArray, pConditionsArray, pStrategiesArray, pTradesArray, pLastObjectsArray) { - - tradingSystem = pTradingSystem - executionArray = pExecutionArray - recordsArray = pRecordsArray - conditionsArray = pConditionsArray - strategiesArray = pStrategiesArray - tradesArray = pTradesArray - lastObjectsArray = pLastObjectsArray - - writeExecutionFile() - } - - function writeExecutionFile() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeExecutionFile -> Entering function."); } - - let separator = ""; - let fileRecordCounter = 0; - - let fileContent = ""; - - for (let i = 0; i < executionArray.length; i++) { - - let record = executionArray[i]; - - /* Will only add to the file the records of the current day */ - - if (record.begin < currentDay.valueOf()) { continue; } - - fileContent = fileContent + separator + '[' + - record.begin + "," + - record.end + "," + - JSON.stringify(record.executionRecord) + "]"; - - if (separator === "") { separator = ","; } - - fileRecordCounter++; - - } - - fileContent = "[" + fileContent + "]"; - - let dateForPath = currentDay.getUTCFullYear() + '/' + utilities.pad(currentDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(currentDay.getUTCDate(), 2); - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePathRoot = bot.devTeam + "/" + bot.codeName + "." + bot.version.major + "." + bot.version.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + EXECUTION_FOLDER_NAME + "/" + "Multi-Period-Daily" + "/" + outputPeriodLabel + "/" + dateForPath; - filePath += '/' + fileName - - fileStorage.createTextFile(bot.devTeam, filePath, fileContent + '\n', onFileCreated); - - function onFileCreated(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeExecutionFile -> onFileCreated -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> writeExecutionFile -> onFileCreated -> fileContent = " + fileContent); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> writeExecutionFile -> onFileCreated -> err = " + err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> writeExecutionFile -> onFileCreated -> filePath = " + filePath); - logger.write(MODULE_NAME, "[ERROR] start -> writeExecutionFile -> onFileCreated -> market = " + market.assetA + "_" + market.assetB); - - callBackFunction(err); - return; - - } - - writeFinalResults(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeExecutionFile -> onFileCreated -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeExecutionFile -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function writeFinalResults() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFinalResults -> Entering function."); } - - if ( // We only write this file if we are at the head of the market. - Math.trunc(currentDay.valueOf() / ONE_DAY_IN_MILISECONDS) * ONE_DAY_IN_MILISECONDS - !== - Math.trunc((new Date()).valueOf() / ONE_DAY_IN_MILISECONDS) * ONE_DAY_IN_MILISECONDS - ) { - writeRecordsFile(); - return; - } - - let separator = ""; - let fileRecordCounter = 0; - - let fileContent = ""; - - for (let i = 0; i < recordsArray.length; i++) { - - let record = recordsArray[i]; - - /* Will only add to the file the records of the current day */ - - if (record.begin < currentDay.valueOf()) { continue; } - - fileContent = fileContent + separator + '[' + - record.begin + "," + - record.end + "," + - record.type + "," + - record.marketRate + "," + - record.amount + "," + - record.balanceA + "," + - record.balanceB + "," + - record.profit + "," + - record.lastTradeProfitLoss + "," + - record.stopLoss + "," + - record.roundtrips + "," + - record.hits + "," + - record.fails + "," + - record.hitRatio + "," + - record.ROI + "," + - record.periods + "," + - record.days + "," + - record.anualizedRateOfReturn + "," + - record.positionRate + "," + - record.lastTradeROI + "," + - record.strategy + "," + - record.strategyStageNumber + "," + - record.takeProfit + "," + - record.stopLossPhase + "," + - record.takeProfitPhase + "," + - JSON.stringify(record.executionRecord) + "," + - record.positionSize + "," + - record.initialBalanceA + "," + - record.minimumBalanceA + "," + - record.maximumBalanceA + "," + - record.initialBalanceB + "," + - record.minimumBalanceB + "," + - record.maximumBalanceB + "," + - record.baseAsset + "," + - record.positionPeriods + "," + - record.positionDays + "]"; - - if (separator === "") { separator = ","; } - - fileRecordCounter++; - - } - - fileContent = "[" + fileContent + "]"; - - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePathRoot = bot.devTeam + "/" + bot.codeName + "." + bot.version.major + "." + bot.version.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + SIMULATED_RECORDS_FOLDER_NAME + "/" + "Multi-Period-Daily" + "/" + outputPeriodLabel; - filePath += '/' + fileName - - fileStorage.createTextFile(bot.devTeam, filePath, fileContent + '\n', onFileCreated); - - function onFileCreated(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFinalResults -> onFileCreated -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> writeFinalResults -> onFileCreated -> fileContent = " + fileContent); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> writeFinalResults -> onFileCreated -> err = " + err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> writeFinalResults -> onFileCreated -> filePath = " + filePath); - logger.write(MODULE_NAME, "[ERROR] start -> writeFinalResults -> onFileCreated -> market = " + market.assetA + "_" + market.assetB); - - callBackFunction(err); - return; - - } - - writeRecordsFile(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeFinalResults -> onFileCreated -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeFinalResults -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function writeRecordsFile() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeRecordsFile -> Entering function."); } - - let separator = ""; - let fileRecordCounter = 0; - - let fileContent = ""; - - for (let i = 0; i < recordsArray.length; i++) { - - let record = recordsArray[i]; - - /* Will only add to the file the records of the current day */ - - if (record.begin < currentDay.valueOf()) { continue; } - - fileContent = fileContent + separator + '[' + - record.begin + "," + - record.end + "," + - record.type + "," + - record.marketRate + "," + - record.amount + "," + - record.balanceA + "," + - record.balanceB + "," + - record.profit + "," + - record.lastTradeProfitLoss + "," + - record.stopLoss + "," + - record.roundtrips + "," + - record.hits + "," + - record.fails + "," + - record.hitRatio + "," + - record.ROI + "," + - record.periods + "," + - record.days + "," + - record.anualizedRateOfReturn + "," + - record.positionRate + "," + - record.lastTradeROI + "," + - record.strategy + "," + - record.strategyStageNumber + "," + - record.takeProfit + "," + - record.stopLossPhase + "," + - record.takeProfitPhase + "," + - JSON.stringify(record.executionRecord) + "," + - record.positionSize + "," + - record.initialBalanceA + "," + - record.minimumBalanceA + "," + - record.maximumBalanceA + "," + - record.initialBalanceB + "," + - record.minimumBalanceB + "," + - record.maximumBalanceB + "," + - record.baseAsset + "," + - record.positionPeriods + "," + - record.positionDays + "]"; - - if (separator === "") { separator = ","; } - - fileRecordCounter++; - - } - - fileContent = "[" + fileContent + "]"; - - let dateForPath = currentDay.getUTCFullYear() + '/' + utilities.pad(currentDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(currentDay.getUTCDate(), 2); - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePathRoot = bot.devTeam + "/" + bot.codeName + "." + bot.version.major + "." + bot.version.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + SIMULATED_RECORDS_FOLDER_NAME + "/" + "Multi-Period-Daily" + "/" + outputPeriodLabel + "/" + dateForPath; - filePath += '/' + fileName - - fileStorage.createTextFile(bot.devTeam, filePath, fileContent + '\n', onFileCreated); - - function onFileCreated(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeRecordsFile -> onFileCreated -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> writeRecordsFile -> onFileCreated -> fileContent = " + fileContent); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> writeRecordsFile -> onFileCreated -> err = " + err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> writeRecordsFile -> onFileCreated -> filePath = " + filePath); - logger.write(MODULE_NAME, "[ERROR] start -> writeRecordsFile -> onFileCreated -> market = " + market.assetA + "_" + market.assetB); - - callBackFunction(err); - return; - - } - - writeConditionsFile(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeRecordsFile -> onFileCreated -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeRecordsFile -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function writeConditionsFile() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeConditionsFile -> Entering function."); } - - let separator = ""; - let fileRecordCounter = 0; - - let fileContent = ""; - - for (let i = 0; i < conditionsArray.length; i++) { - - let record = conditionsArray[i]; - - /* Will only add to the file the records of the current day */ - - if (record.begin < currentDay.valueOf()) { continue; } - - let conditions = ""; - let conditionsSeparator = ""; - - /* Will only add to the file the records of the current day */ - - if (record.begin < currentDay.valueOf()) { continue; } - - for (let j = 0; j < record.length - 3; j++) { - conditions = conditions + conditionsSeparator + record[j]; - if (conditionsSeparator === "") { conditionsSeparator = ","; } - } - - conditions = conditions + conditionsSeparator + '[' + record[record.length - 3] + ']'; // The last item contains an Array of condition values. - conditions = conditions + conditionsSeparator + '[' + record[record.length - 2] + ']'; // The last item contains an Array of formulaErrors. - conditions = conditions + conditionsSeparator + '[' + record[record.length - 1] + ']'; // The last item contains an Array of formulaValues. - - fileContent = fileContent + separator + '[' + conditions + ']'; - - if (separator === "") { separator = ","; } - - fileRecordCounter++; - - } - - fileContent = "[" + fileContent + "]"; - fileContent = "[" + JSON.stringify(tradingSystem) + "," + JSON.stringify(lastObjectsArray) + "," + fileContent + "]"; - - let dateForPath = currentDay.getUTCFullYear() + '/' + utilities.pad(currentDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(currentDay.getUTCDate(), 2); - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePathRoot = bot.devTeam + "/" + bot.codeName + "." + bot.version.major + "." + bot.version.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + CONDITIONS_FOLDER_NAME + "/" + "Multi-Period-Daily" + "/" + outputPeriodLabel + "/" + dateForPath; - filePath += '/' + fileName - - fileStorage.createTextFile(bot.devTeam, filePath, fileContent + '\n', onFileCreated); - - function onFileCreated(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeConditionsFile -> onFileCreated -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> writeConditionsFile -> onFileCreated -> fileContent = " + fileContent); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> writeConditionsFile -> onFileCreated -> err = " + err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> writeConditionsFile -> onFileCreated -> filePath = " + filePath); - logger.write(MODULE_NAME, "[ERROR] start -> writeConditionsFile -> onFileCreated -> market = " + market.assetA + "_" + market.assetB); - - callBackFunction(err); - return; - - } - - writeStrategiesFile(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeConditionsFile -> onFileCreated -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeConditionsFile -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function writeStrategiesFile() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStrategiesFile -> Entering function."); } - - let separator = ""; - let fileRecordCounter = 0; - - let fileContent = ""; - - for (let i = 0; i < strategiesArray.length; i++) { - let record = strategiesArray[i]; - - /* Will only add to the file the records of the current day. In this case since objects can span more than one day, we add all of the objects that ends - at the current date. */ - - if (record.end < currentDay.valueOf()) { continue; } - - fileContent = fileContent + separator + '[' + - record.begin + "," + - record.end + "," + - record.status + "," + - record.number + "," + - record.beginRate + "," + - record.endRate + "]"; - - if (separator === "") { separator = ","; } - - fileRecordCounter++; - - } - - fileContent = "[" + fileContent + "]"; - - let dateForPath = currentDay.getUTCFullYear() + '/' + utilities.pad(currentDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(currentDay.getUTCDate(), 2); - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePathRoot = bot.devTeam + "/" + bot.codeName + "." + bot.version.major + "." + bot.version.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + STRATEGIES_FOLDER_NAME + "/" + "Multi-Period-Daily" + "/" + outputPeriodLabel + "/" + dateForPath; - filePath += '/' + fileName - - fileStorage.createTextFile(bot.devTeam, filePath, fileContent + '\n', onFileCreated); - - function onFileCreated(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStrategiesFile -> onFileCreated -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStrategiesFile -> onFileCreated -> fileContent = " + fileContent); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> writeStrategiesFile -> onFileCreated -> err = " + err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> writeStrategiesFile -> onFileCreated -> filePath = " + filePath); - logger.write(MODULE_NAME, "[ERROR] start -> writeStrategiesFile -> onFileCreated -> market = " + market.assetA + "_" + market.assetB); - - callBackFunction(err); - return; - - } - - writeTradesFile(); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeStrategiesFile -> onFileCreated -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeStrategiesFile -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function writeTradesFile() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeTradesFile -> Entering function."); } - - let separator = ""; - let fileRecordCounter = 0; - - let fileContent = ""; - - for (let i = 0; i < tradesArray.length; i++) { - - let record = tradesArray[i]; - - /* Will only add to the file the records of the current day. In this case since objects can span more than one day, we add all of the objects that ends - at the current date. */ - - if (record.end < currentDay.valueOf()) { continue; } - if (record.stopRate === undefined) { record.stopRate = 0} - - fileContent = fileContent + separator + '[' + - record.begin + "," + - record.end + "," + - record.status + "," + - record.lastTradeROI + "," + - record.beginRate + "," + - record.endRate + "," + - record.exitType + "]"; - - if (separator === "") { separator = ","; } - - fileRecordCounter++; - - } - - fileContent = "[" + fileContent + "]"; - - let dateForPath = currentDay.getUTCFullYear() + '/' + utilities.pad(currentDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(currentDay.getUTCDate(), 2); - let fileName = '' + market.assetA + '_' + market.assetB + '.json'; - - let filePathRoot = bot.devTeam + "/" + bot.codeName + "." + bot.version.major + "." + bot.version.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + bot.dataSetVersion; - let filePath = filePathRoot + "/Output/" + TRADES_FOLDER_NAME + "/" + "Multi-Period-Daily" + "/" + outputPeriodLabel + "/" + dateForPath; - filePath += '/' + fileName - - fileStorage.createTextFile(bot.devTeam, filePath, fileContent + '\n', onFileCreated); - - function onFileCreated(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeTradesFile -> onFileCreated -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> writeTradesFile -> onFileCreated -> fileContent = " + fileContent); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[ERROR] start -> writeTradesFile -> onFileCreated -> err = " + err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> writeTradesFile -> onFileCreated -> filePath = " + filePath); - logger.write(MODULE_NAME, "[ERROR] start -> writeTradesFile -> onFileCreated -> market = " + market.assetA + "_" + market.assetB); - - callBackFunction(err); - return; - - } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeTradesFile -> onFileCreated -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeTradesFile -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } -}; From 885fea80e28dc2aece0f620ae36c9d5be69e37d2 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 25 Aug 2019 13:59:09 +0200 Subject: [PATCH 10/32] Multi Period Process Created --- MultiPeriod.js | 782 ++++++++++++++++++++++++++++++++ Project.njsproj | 3 + Root.js | 2 +- Run.js | 4 +- TradingEngineProcessMainLoop.js | 8 +- 5 files changed, 795 insertions(+), 4 deletions(-) create mode 100644 MultiPeriod.js diff --git a/MultiPeriod.js b/MultiPeriod.js new file mode 100644 index 000000000..92c46f5df --- /dev/null +++ b/MultiPeriod.js @@ -0,0 +1,782 @@ +exports.newMultiPeriod = function newMultiPeriod(bot, logger, COMMONS, UTILITIES, USER_BOT_MODULE, COMMONS_MODULE) { + + const FULL_LOG = true; + const LOG_FILE_CONTENT = false; + const MODULE_NAME = "Multi Period"; + const GMT_SECONDS = ':00.000 GMT+0000'; + const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000; + + thisObject = { + initialize: initialize, + start: start + }; + + let utilities = UTILITIES.newCloudUtilities(logger); + + let statusDependencies; + let dataDependencies; + let storages = []; + let dataFiles = []; + + let usertBot; + + const FILE_STORAGE = require('./Integrations/FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(logger); + + let processConfig; + + return thisObject; + + function initialize(pProcessConfig, pStatusDependencies, pDataDependencies, pAssistant, callBackFunction) { + + try { + + logger.fileName = MODULE_NAME; + logger.initialize(); + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } + + statusDependencies = pStatusDependencies; + dataDependencies = pDataDependencies; + processConfig = pProcessConfig; + + for (let i = 0; i < dataDependencies.config.length; i++) { + + let key; + let storage; + let dependency = dataDependencies.config[i]; + + key = dependency.devTeam + "-" + + dependency.bot + "-" + + dependency.product + "-" + + dependency.dataSet + "-" + + dependency.dataSetVersion + + storage = dataDependencies.dataSets.get(key); + + storages.push(storage); + + } + + usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, fileStorage); + usertBot.initialize(dataDependencies, callBackFunction, pAssistant); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function start(callBackFunction) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); } + + let market = global.MARKET; + + /* Context Variables */ + + let contextVariables = { + lastFile: undefined, // Datetime of the last file files sucessfully produced by this process. + dateBeginOfMarket: undefined, // Datetime of the first trade file in the whole market history. + dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process. + }; + + let previousDay; // Holds the date of the previous day relative to the processing date. + + let interExecutionMemoryArray; + + getContextVariables(); + + function getContextVariables() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Entering function."); } + + let thisReport; + let reportKey; + let statusReport; + + if (processConfig.framework.startDate.fixedDate !== undefined) { + + /* The starting date is fixed, we will start from there. */ + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> We have got a user defined startDate. -> startDate = " + processConfig.framework.startDate.fixedDate); } + contextVariables.dateBeginOfMarket = new Date(processConfig.framework.startDate.fixedDate); + + } else { + + /* + We look first for the bot who knows the begining of the marke in order to get when the market starts. + */ + + let botWhoKnowsTheBeginingOfTheMarket = statusDependencies.config[processConfig.framework.startDate.takeItFromStatusDependency]; + + reportKey = botWhoKnowsTheBeginingOfTheMarket.devTeam + "-" + botWhoKnowsTheBeginingOfTheMarket.bot + "-" + botWhoKnowsTheBeginingOfTheMarket.process + "-" + "dataSet.V1"; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } + + statusReport = statusDependencies.statusReports.get(reportKey); + + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + thisReport = statusDependencies.statusReports.get(reportKey).file; + + if (thisReport.lastFile === undefined) { + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); + logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet."); + + let customOK = { + result: global.CUSTOM_OK_RESPONSE.result, + message: "Dependency does not exist." + } + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); + callBackFunction(customOK); + return; + } + + contextVariables.dateBeginOfMarket = new Date(thisReport.lastFile.year + "-" + thisReport.lastFile.month + "-" + thisReport.lastFile.days + " " + thisReport.lastFile.hours + ":" + thisReport.lastFile.minutes + GMT_SECONDS); + + } + + if (processConfig.framework.endDate.fixedDate !== undefined) { + + /* The ending date is fixed, we will end there. */ + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> We have got a user defined endDate. -> endDate = " + processConfig.framework.endDate.fixedDate); } + contextVariables.dateEndOfMarket = new Date(processConfig.framework.endDate.fixedDate); + + } else { + + /* + Here we get the status report from the bot who knows which is the end of the market. + */ + + let botWhoKnowsTheEndOfTheMarket = statusDependencies.config[processConfig.framework.endDate.takeItFromStatusDependency]; + + reportKey = botWhoKnowsTheEndOfTheMarket.devTeam + "-" + botWhoKnowsTheEndOfTheMarket.bot + "-" + botWhoKnowsTheEndOfTheMarket.process + "-" + "dataSet.V1"; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } + + statusReport = statusDependencies.statusReports.get(reportKey); + + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + thisReport = statusDependencies.statusReports.get(reportKey).file; + + if (thisReport.lastFile === undefined) { + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey); + + let customOK = { + result: global.CUSTOM_OK_RESPONSE.result, + message: "Dependency not ready." + } + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); + callBackFunction(customOK); + return; + } + + contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf()); + } + + /* Finally we get our own Status Report. */ + + reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } + + statusReport = statusDependencies.statusReports.get(reportKey); + + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + + thisReport = statusDependencies.statusReports.get(reportKey).file; + + if (thisReport.lastFile !== undefined) { + + if (bot.hasTheBotJustStarted === true && processConfig.framework.startDate.resumeExecution === false) { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Starting from the begining because bot has just started and resume execution was true."); } + startFromBegining(); + return; + } + + contextVariables.lastFile = new Date(thisReport.lastFile); + interExecutionMemoryArray = thisReport.interExecutionMemoryArray; + + processTimePeriods(); + return; + + } else { + + /* + In the case when there is no status report, we assume like the last processed file is the one on the date of Begining of Market. + */ + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Starting from the begining of the market because own status report not found or lastFile was undefined."); } + startFromBegining(); + return; + } + + function startFromBegining() { + + contextVariables.lastFile = new Date(contextVariables.dateBeginOfMarket.getUTCFullYear() + "-" + (contextVariables.dateBeginOfMarket.getUTCMonth() + 1) + "-" + contextVariables.dateBeginOfMarket.getUTCDate() + " " + "00:00" + GMT_SECONDS); + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> startFromBegining -> contextVariables.lastFile = " + contextVariables.lastFile); } + + /* + The first time the process is running is the right time to create the data structure that is going to be shared across different executions. + This data structure has one object per each timePeriod. + */ + + interExecutionMemoryArray = []; + + for (let i = 0; i < global.dailyFilePeriods.length; i++) { + let emptyObject = {}; + interExecutionMemoryArray.push(emptyObject); + } + + processTimePeriods(); + + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> err = " + err.stack); + if (err.message === "Cannot read property 'file' of undefined") { + logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Check the bot configuration to see if all of its statusDependencies declarations are correct. "); + logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys)); + } + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function processTimePeriods() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> Entering function."); } + + try { + + let n; + let botNeverRan = true; + + bot.multiPeriodDailyProcessDatetime = new Date(contextVariables.lastFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. + + advanceTime(); + + function advanceTime() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> Entering function."); } + + bot.multiPeriodDailyProcessDatetime = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() + ONE_DAY_IN_MILISECONDS); + previousDay = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() - ONE_DAY_IN_MILISECONDS); + + console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " " + " bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime.toISOString()); + + if (global.FULL_LOG === 'true') { + logger.newInternalLoop(bot.codeName, bot.process); + } + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> previousDay = " + previousDay); } + + /* Validation that we are not going past the head of the market. */ + + if (bot.multiPeriodDailyProcessDatetime.valueOf() > contextVariables.dateEndOfMarket.valueOf()) { + + const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> " + logText); } + + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + + } + + periodsLoop(); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> advanceTime -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoop -> Entering function."); } + + /* + + We will iterate through all posible periods. + + */ + + n = 0 // loop Variable representing each possible period as defined at the periods array. + + periodsLoopBody(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoop -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsLoopBody() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Entering function."); } + + const timePeriod = global.dailyFilePeriods[n][0]; + const outputPeriodLabel = global.dailyFilePeriods[n][1]; + + if (processConfig.framework.validPeriods !== undefined) { + let validPeriod = false; + for (let i = 0; i < processConfig.framework.validPeriods.length; i++) { + let period = processConfig.framework.validPeriods[i]; + if (period === outputPeriodLabel) { validPeriod = true } + } + if (validPeriod === false) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Discarding period for not being listed as a valid period. -> outputPeriodLabel = " + outputPeriodLabel); } + periodsControlLoop(); + return; + } + } + + let timePeriodFilter = global.EXECUTION_CONFIG.timePeriodFilter + if (timePeriodFilter !== undefined) { + if (timePeriodFilter.indexOf(outputPeriodLabel) === -1) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Discarding period for being filtered out. -> outputPeriodLabel = " + outputPeriodLabel); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Discarding period for being filtered out. -> timePeriodFilter = " + timePeriodFilter); } + periodsControlLoop(); + return; + } + } + + let dependencyIndex = 0; + dataFiles = []; + + dependencyLoopBody(); + + function dependencyLoopBody() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } + + let dependency = dataDependencies.config[dependencyIndex]; + let storage = storages[dependencyIndex]; + + let previousFile; + let currentFile; + + if (bot.multiPeriodDailyProcessDatetime.valueOf() > contextVariables.dateBeginOfMarket.valueOf()) { + getPreviousFile(); + } else { + previousFile = []; + getCurrentFile() + } + + function getPreviousFile() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } + + let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2); + let filePath + if (dependency.dataSet === "Multi-Period-Daily") { + filePath = dependency.product + '/' + dependency.dataSet + "/" + outputPeriodLabel + "/" + dateForPath; + } else { + filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath; + } + let fileName = market.assetA + '_' + market.assetB + ".json"; + + storage.getTextFile(filePath, fileName, onFileReceived); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); } + + if ((err.message === "File does not exist." && botNeverRan === true) || err.code === 'The specified key does not exist.') { + + /* + Sometimes one of the dependencies of an indicator for some reasons are not calculated from the begining of the market. + When that happens we can not find those files. What we do in this situation is to move the time fordward until we can find + all the dependencies and the first run of the bot is successful. + + After that, we will not accept more missing files on any of the dependencies, and if any is missing we will abort the processing. + */ + logger.write(MODULE_NAME, "[WARN] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found."); + + advanceTime(); + return; + } + + if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); + callBackFunction(err); + return; + } + + previousFile = JSON.parse(text); + + getCurrentFile(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function getCurrentFile() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } + + let dateForPath = bot.multiPeriodDailyProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCDate(), 2); + let filePath + if (dependency.dataSet === "Multi-Period-Daily") { + filePath = dependency.product + '/' + dependency.dataSet + "/" + outputPeriodLabel + "/" + dateForPath; + } else { + filePath = dependency.product + '/' + dependency.dataSet + "/" + dateForPath; + } + let fileName = market.assetA + '_' + market.assetB + ".json"; + + storage.getTextFile(filePath, fileName, onFileReceived); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); } + + if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> filePath = " + filePath); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> fileName = " + fileName); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return; + } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); + callBackFunction(err); + return; + } + + currentFile = JSON.parse(text); + + let dataFile = previousFile.concat(currentFile); + + dataFiles.push(dataFile); + dependencyControlLoop(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoop -> dependencyLoopBody -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function dependencyControlLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> dependencyControlLoop -> Entering function."); } + + dependencyIndex++; + + if (dependencyIndex < dataDependencies.config.length) { + + dependencyLoopBody(); + + } else { + + callTheBot(); + + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> dependencyControlLoop -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function callTheBot() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> Entering function."); } + + const timePeriod = global.dailyFilePeriods[n][0]; + const outputPeriodLabel = global.dailyFilePeriods[n][1]; + + usertBot.start( + dataFiles, + timePeriod, + outputPeriodLabel, + bot.multiPeriodDailyProcessDatetime, + contextVariables.dateBeginOfMarket, + contextVariables.dateEndOfMarket, + interExecutionMemoryArray[n], + onBotFinished); + + function onBotFinished(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> onBotFinished -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + callBackFunction(err); + return; + } + + botNeverRan = false; + periodsControlLoop(); + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> onBotFinished -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsControlLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsControlLoop -> Entering function."); } + + n++; + + if (n < global.dailyFilePeriods.length) { + + periodsLoopBody(); + + } else { + + n = 0; + + writeDataRanges(onWritten); + + function onWritten(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> controlLoop -> onWritten -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> controlLoop -> onWritten -> err = " + err.stack); + callBackFunction(err); + return; + } + + writeStatusReport(bot.multiPeriodDailyProcessDatetime, advanceTime); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> controlLoop -> onWritten -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsControlLoop -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function writeDataRanges(callBack) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRanges -> Entering function."); } + + let productIndex = 0; + productLoopBody(); + + function productLoopBody() { + + let folderName = bot.products[productIndex].codeName; + + writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodDailyProcessDatetime, folderName, controlLoop); + } + + function controlLoop() { + + productIndex++; + + if (productIndex < bot.products.length) { + productLoopBody(); + } else { + callBack(global.DEFAULT_OK_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> writeDataRanges -> err = " + err.stack); + callBack(global.DEFAULT_FAIL_RESPONSE); + } + + } + + function writeDataRange(pBegin, pEnd, pProductFolder, callBack) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> Entering function."); } + + let dataRange = { + begin: pBegin.valueOf(), + end: pEnd.valueOf() + }; + + let fileContent = JSON.stringify(dataRange); + + let fileName = '/Data.Range.' + market.assetA + '_' + market.assetB + '.json'; + let filePath = bot.filePathRoot + "/Output/" + pProductFolder + "/" + bot.process + fileName; + + fileStorage.createTextFile(global.DEV_TEAM, filePath, fileContent + '\n', onFileCreated); + + function onFileCreated(err) { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> onFileCreated -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write(MODULE_NAME, "[ERROR] start -> writeDataRange -> onFileCreated -> err = " + err.stack); + callBack(err); + return; + } + + if (LOG_FILE_CONTENT === true) { + logger.write(MODULE_NAME, "[INFO] start -> writeDataRange -> onFileCreated -> Content written = " + fileContent); + } + + callBack(global.DEFAULT_OK_RESPONSE); + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> writeDataRange -> err = " + err.stack); + callBack(global.DEFAULT_FAIL_RESPONSE); + } + } + + function writeStatusReport(lastFileDate, callBack) { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> lastFileDate = " + lastFileDate); } + + try { + + let reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; + let thisReport = statusDependencies.statusReports.get(reportKey); + + thisReport.file.lastExecution = bot.currentDaytime; + thisReport.file.lastFile = lastFileDate; + thisReport.file.interExecutionMemoryArray = interExecutionMemoryArray; + thisReport.save(callBack); + + bot.hasTheBotJustStarted = false; + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function pad(str, max) { + str = str.toString(); + return str.length < max ? pad(" " + str, max) : str; + } +}; diff --git a/Project.njsproj b/Project.njsproj index d4b026ae4..a6dd72273 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -27,6 +27,9 @@ + + Code + Code diff --git a/Root.js b/Root.js index f4b80a256..2e6c123c6 100644 --- a/Root.js +++ b/Root.js @@ -235,7 +235,7 @@ if (processConfig.framework !== undefined) { - if (processConfig.framework.name === "Multi-Period-Daily" || processConfig.framework.name === "Multi-Period-Market") { + if (processConfig.framework.name === "Multi-Period-Daily" || processConfig.framework.name === "Multi-Period-Market" || processConfig.framework.name === "Multi-Period") { processConfig.framework.startDate.resumeExecution = false; if (processConfig.startMode.noTime !== undefined) { if (processConfig.startMode.noTime.run === "true") { diff --git a/Run.js b/Run.js index 4270d3cd4..389ea570d 100644 --- a/Run.js +++ b/Run.js @@ -159,13 +159,13 @@ async function readExecutionConfiguration(execution) { if (definition.simulationParams.timePeriodDailyArray !== undefined) { if (definition.simulationParams.timePeriodDailyArray.length === 1) { timePeriodFilter = definition.simulationParams.timePeriodDailyArray[0] - botProcess = "Multi-Period-Daily" + botProcess = "Multi-Period" } } if (definition.simulationParams.timePeriodMarketArray !== undefined) { if (definition.simulationParams.timePeriodMarketArray.length === 1) { timePeriodFilter = definition.simulationParams.timePeriodMarketArray[0] - botProcess = "Multi-Period-Market" + botProcess = "Multi-Period" } } } diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 1d59a68d9..18ba4f06b 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -10,6 +10,7 @@ const MULTI_PERIOD_MARKET = require(ROOT_DIR + 'MultiPeriodMarket'); const MULTI_PERIOD_DAILY = require(ROOT_DIR + 'MultiPeriodDaily'); + const MULTI_PERIOD = require(ROOT_DIR + 'MultiPeriod'); const FILE_STORAGE = require('./Integrations/FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(parentLogger); @@ -55,7 +56,7 @@ } USER_BOT_MODULE = {} - USER_BOT_MODULE.newUserBot = require(process.env.BOTS_PATH + '/aamasters/AAMasters/bots/AAJason-Trading-Engine-Bot/Multi-Period-Daily/User.Bot').newUserBot // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. + USER_BOT_MODULE.newUserBot = require(process.env.BOTS_PATH + '/aamasters/AAMasters/bots/AAJason-Trading-Engine-Bot/Multi-Period/User.Bot').newUserBot // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. //USER_BOT_MODULE.newUserBot = eval(text); // Use this for production filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; @@ -487,6 +488,11 @@ intitializeProcessFramework(); break; } + case 'Multi-Period': { + processFramework = MULTI_PERIOD.newMultiPeriod(bot, logger, COMMONS_MODULE, UTILITIES, USER_BOT_MODULE, COMMONS_MODULE); + intitializeProcessFramework(); + break; + } default: { logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeAssistant -> onInizialized -> Process Framework not Supported."); logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeAssistant -> onInizialized -> Process Framework Name = " + processConfig.framework.name); From 18d4a05b97e718545d37291fa982135b55e4738a Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 25 Aug 2019 20:52:07 +0200 Subject: [PATCH 11/32] Multi Period working for Daily Files --- MultiPeriod.js | 153 +++++++++++++++++++++++++------------------------ 1 file changed, 79 insertions(+), 74 deletions(-) diff --git a/MultiPeriod.js b/MultiPeriod.js index 92c46f5df..d92808f4e 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -17,6 +17,7 @@ let dataDependencies; let storages = []; let dataFiles = []; + let multiPeriodDataFiles = new Map(); let usertBot; @@ -73,6 +74,9 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); } + let currentTimePeriod + let currentOutputPeriodLabel + let market = global.MARKET; /* Context Variables */ @@ -200,7 +204,7 @@ /* Finally we get our own Status Report. */ - reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; + reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period" + "-" + "dataSet.V1"; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> reportKey = " + reportKey); } statusReport = statusDependencies.statusReports.get(reportKey); @@ -374,11 +378,9 @@ let timePeriodFilter = global.EXECUTION_CONFIG.timePeriodFilter if (timePeriodFilter !== undefined) { - if (timePeriodFilter.indexOf(outputPeriodLabel) === -1) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Discarding period for being filtered out. -> outputPeriodLabel = " + outputPeriodLabel); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Discarding period for being filtered out. -> timePeriodFilter = " + timePeriodFilter); } - periodsControlLoop(); - return; + if (timePeriodFilter.indexOf(outputPeriodLabel) > 0 -1) { + currentTimePeriod = global.dailyFilePeriods[n][0]; + currentOutputPeriodLabel = global.dailyFilePeriods[n][1]; } } @@ -521,6 +523,7 @@ let dataFile = previousFile.concat(currentFile); dataFiles.push(dataFile); + dependencyControlLoop(); } @@ -556,7 +559,10 @@ } else { - callTheBot(); + let mapKey = global.dailyFilePeriods[n][1]; + multiPeriodDataFiles.set(mapKey, dataFiles) + + periodsControlLoop(); } } @@ -566,51 +572,6 @@ } } - function callTheBot() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> Entering function."); } - - const timePeriod = global.dailyFilePeriods[n][0]; - const outputPeriodLabel = global.dailyFilePeriods[n][1]; - - usertBot.start( - dataFiles, - timePeriod, - outputPeriodLabel, - bot.multiPeriodDailyProcessDatetime, - contextVariables.dateBeginOfMarket, - contextVariables.dateEndOfMarket, - interExecutionMemoryArray[n], - onBotFinished); - - function onBotFinished(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> onBotFinished -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - callBackFunction(err); - return; - } - - botNeverRan = false; - periodsControlLoop(); - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> onBotFinished -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - } - catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> callTheBot -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } } catch (err) { logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> err = " + err.stack); @@ -634,27 +595,8 @@ n = 0; - writeDataRanges(onWritten); + callTheBot(); - function onWritten(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> controlLoop -> onWritten -> Entering function."); } - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> controlLoop -> onWritten -> err = " + err.stack); - callBackFunction(err); - return; - } - - writeStatusReport(bot.multiPeriodDailyProcessDatetime, advanceTime); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> controlLoop -> onWritten -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } } } catch (err) { @@ -663,6 +605,69 @@ } } + function callTheBot() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> callTheBot -> Entering function."); } + + usertBot.start( + multiPeriodDataFiles, + currentTimePeriod, + currentOutputPeriodLabel, + bot.multiPeriodDailyProcessDatetime, + contextVariables.dateBeginOfMarket, + contextVariables.dateEndOfMarket, + interExecutionMemoryArray[n], + onBotFinished); + + function onBotFinished(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> callTheBot -> onBotFinished -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + callBackFunction(err); + return; + } + + botNeverRan = false; + + writeDataRanges(onWritten); + + function onWritten(err) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> callTheBot -> onBotFinished -> onWritten -> Entering function."); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); + callBackFunction(err); + return; + } + + writeStatusReport(bot.multiPeriodDailyProcessDatetime, advanceTime); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> onBotFinished -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } } catch (err) { logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> err = " + err.stack); @@ -718,7 +723,7 @@ let fileContent = JSON.stringify(dataRange); let fileName = '/Data.Range.' + market.assetA + '_' + market.assetB + '.json'; - let filePath = bot.filePathRoot + "/Output/" + pProductFolder + "/" + bot.process + fileName; + let filePath = bot.filePathRoot + "/Output/" + pProductFolder + "/" + 'Multi-Period-Daily' + fileName; fileStorage.createTextFile(global.DEV_TEAM, filePath, fileContent + '\n', onFileCreated); @@ -752,7 +757,7 @@ try { - let reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period-Daily" + "-" + "dataSet.V1"; + let reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period" + "-" + "dataSet.V1"; let thisReport = statusDependencies.statusReports.get(reportKey); thisReport.file.lastExecution = bot.currentDaytime; From 1f84be75f39cfa85d24b1ba1b4b60bc20eb6dd51 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 26 Aug 2019 17:18:12 +0200 Subject: [PATCH 12/32] Multi Period fully tested --- Integrations/FileStorage.js | 4 +- MultiPeriod.js | 422 ++++++++++++++++++++++++++++-------- 2 files changed, 332 insertions(+), 94 deletions(-) diff --git a/Integrations/FileStorage.js b/Integrations/FileStorage.js index a2071deb7..e9c7011cd 100644 --- a/Integrations/FileStorage.js +++ b/Integrations/FileStorage.js @@ -64,7 +64,7 @@ exports.newFileStorage = function newFileStorage(logger) { } logInfo('getTextFile: ' + fileLocation) - console.log('getTextFile: ' + fileLocation) + //console.log('getTextFile: ' + fileLocation) let fileContent = await readFileAsync(fileLocation) callBackFunction(global.DEFAULT_OK_RESPONSE, fileContent.toString()) @@ -130,7 +130,7 @@ exports.newFileStorage = function newFileStorage(logger) { } logInfo('createTextFile: ' + fileLocation) - console.log('createTextFile: ' + fileLocation) + //console.log('createTextFile: ' + fileLocation) let directoryPath = fileLocation.substring(0, fileLocation.lastIndexOf('/') + 1); mkDirByPathSync(directoryPath) diff --git a/MultiPeriod.js b/MultiPeriod.js index d92808f4e..fa414ff94 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -60,7 +60,7 @@ } usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE, UTILITIES, fileStorage); - usertBot.initialize(dataDependencies, callBackFunction, pAssistant); + usertBot.initialize(callBackFunction, pAssistant); } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); @@ -78,6 +78,7 @@ let currentOutputPeriodLabel let market = global.MARKET; + let botNeverRan = true; /* Context Variables */ @@ -89,7 +90,7 @@ let previousDay; // Holds the date of the previous day relative to the processing date. - let interExecutionMemoryArray; + let interExecutionMemoryArray = []; getContextVariables(); @@ -235,7 +236,7 @@ contextVariables.lastFile = new Date(thisReport.lastFile); interExecutionMemoryArray = thisReport.interExecutionMemoryArray; - processTimePeriods(); + processTimePeriodsMarketFiles(); return; } else { @@ -266,7 +267,7 @@ interExecutionMemoryArray.push(emptyObject); } - processTimePeriods(); + processTimePeriodsMarketFiles(); } @@ -280,62 +281,21 @@ } } - function processTimePeriods() { + function processTimePeriodsMarketFiles() { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> Entering function."); } try { let n; - let botNeverRan = true; - bot.multiPeriodDailyProcessDatetime = new Date(contextVariables.lastFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. - - advanceTime(); - - function advanceTime() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> Entering function."); } - - bot.multiPeriodDailyProcessDatetime = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() + ONE_DAY_IN_MILISECONDS); - previousDay = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() - ONE_DAY_IN_MILISECONDS); - - console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " " + " bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime.toISOString()); - - if (global.FULL_LOG === 'true') { - logger.newInternalLoop(bot.codeName, bot.process); - } - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> previousDay = " + previousDay); } - - /* Validation that we are not going past the head of the market. */ - - if (bot.multiPeriodDailyProcessDatetime.valueOf() > contextVariables.dateEndOfMarket.valueOf()) { - - const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> " + logText); } - - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - - } - - periodsLoop(); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> advanceTime -> err = " + err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } + periodsLoop(); function periodsLoop() { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoop -> Entering function."); } /* @@ -349,7 +309,7 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -358,7 +318,239 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> Entering function."); } + + const timePeriod = global.marketFilesPeriods[n][0]; + const outputPeriodLabel = global.marketFilesPeriods[n][1]; + + let dependencyIndex = 0; + dataFiles = []; + + let timePeriodFilter = global.EXECUTION_CONFIG.timePeriodFilter + if (timePeriodFilter !== undefined) { + if (timePeriodFilter.indexOf(outputPeriodLabel) > -1) { + currentTimePeriod = global.marketFilesPeriods[n][0]; + currentOutputPeriodLabel = global.marketFilesPeriods[n][1]; + } + } + + dependencyLoopBody(); + + function dependencyLoopBody() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } + + let dependency = dataDependencies.config[dependencyIndex]; + let storage = storages[dependencyIndex]; + + getFile(); + + function getFile() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> Entering function."); } + + let fileName = market.assetA + '_' + market.assetB + ".json"; + + if (dependency.dataSet !== "Multi-Period-Market") { + + dependencyControlLoop(); + return + } + + let filePath = dependency.product + '/' + dependency.dataSet + "/" + outputPeriodLabel; + + storage.getTextFile(filePath, fileName, onFileReceived); + + function onFileReceived(err, text) { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> text = " + text); } + + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); + callBackFunction(err); + return; + } + + let dataFile = JSON.parse(text); + dataFiles.push(dataFile); + + dependencyControlLoop(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> onFileReceived -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> dependencyLoopBody -> getFile -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoop -> dependencyLoopBody -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function dependencyControlLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> dependencyControlLoop -> Entering function."); } + + dependencyIndex++; + + if (dependencyIndex < dataDependencies.config.length) { + + dependencyLoopBody(); + + } else { + + let mapKey = global.marketFilesPeriods[n][1]; + multiPeriodDataFiles.set(mapKey, dataFiles) + + periodsControlLoop(); + + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> dependencyControlLoop -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsLoopBody -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsControlLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsMarketFiles -> periodsControlLoop -> Entering function."); } + + n++; + + if (n < global.marketFilesPeriods.length) { + + periodsLoopBody(); + + } else { + + processTimePeriodsDailyFiles(); + + } + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> periodsControlLoop -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsMarketFiles -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function processTimePeriodsDailyFiles() { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> Entering function."); } + + try { + + let n = 0; + + if (currentTimePeriod) { + callTheBot() + return + } + + bot.multiPeriodDailyProcessDatetime = new Date(contextVariables.lastFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. + + advanceTime(); + + function advanceTime() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> advanceTime -> Entering function."); } + + bot.multiPeriodDailyProcessDatetime = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() + ONE_DAY_IN_MILISECONDS); + previousDay = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() - ONE_DAY_IN_MILISECONDS); + + console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " " + " bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime.toISOString()); + + if (global.FULL_LOG === 'true') { + logger.newInternalLoop(bot.codeName, bot.process); + } + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> advanceTime -> bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> advanceTime -> previousDay = " + previousDay); } + + /* Validation that we are not going past the head of the market. */ + + if (bot.multiPeriodDailyProcessDatetime.valueOf() > contextVariables.dateEndOfMarket.valueOf()) { + + const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> advanceTime -> " + logText); } + + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + + } + + periodsLoop(); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> advanceTime -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsLoop() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoop -> Entering function."); } + + /* + + We will iterate through all posible periods. + + */ + + n = 0 // loop Variable representing each possible period as defined at the periods array. + + periodsLoopBody(); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoop -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function periodsLoopBody() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> Entering function."); } const timePeriod = global.dailyFilePeriods[n][0]; const outputPeriodLabel = global.dailyFilePeriods[n][1]; @@ -370,7 +562,7 @@ if (period === outputPeriodLabel) { validPeriod = true } } if (validPeriod === false) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Discarding period for not being listed as a valid period. -> outputPeriodLabel = " + outputPeriodLabel); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> Discarding period for not being listed as a valid period. -> outputPeriodLabel = " + outputPeriodLabel); } periodsControlLoop(); return; } @@ -378,7 +570,7 @@ let timePeriodFilter = global.EXECUTION_CONFIG.timePeriodFilter if (timePeriodFilter !== undefined) { - if (timePeriodFilter.indexOf(outputPeriodLabel) > 0 -1) { + if (timePeriodFilter.indexOf(outputPeriodLabel) > -1) { currentTimePeriod = global.dailyFilePeriods[n][0]; currentOutputPeriodLabel = global.dailyFilePeriods[n][1]; } @@ -393,7 +585,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> Entering function."); } let dependency = dataDependencies.config[dependencyIndex]; let storage = storages[dependencyIndex]; @@ -412,10 +604,17 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> Entering function."); } + + if (dependency.dataSet === "Multi-Period-Market") { + + dependencyControlLoop(); + return + } let dateForPath = previousDay.getUTCFullYear() + '/' + utilities.pad(previousDay.getUTCMonth() + 1, 2) + '/' + utilities.pad(previousDay.getUTCDate(), 2); let filePath + if (dependency.dataSet === "Multi-Period-Daily") { filePath = dependency.product + '/' + dependency.dataSet + "/" + outputPeriodLabel + "/" + dateForPath; } else { @@ -429,8 +628,8 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> text = " + text); } if ((err.message === "File does not exist." && botNeverRan === true) || err.code === 'The specified key does not exist.') { @@ -441,7 +640,7 @@ After that, we will not accept more missing files on any of the dependencies, and if any is missing we will abort the processing. */ - logger.write(MODULE_NAME, "[WARN] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found."); + logger.write(MODULE_NAME, "[WARN] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> Skipping day because file " + filePath + "/" + fileName + " was not found."); advanceTime(); return; @@ -449,14 +648,14 @@ if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_RETRY_RESPONSE); return; } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); callBackFunction(err); return; } @@ -467,13 +666,13 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -482,7 +681,13 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } + + if (dependency.dataSet === "Multi-Period-Market") { + + dependencyControlLoop(); + return + } let dateForPath = bot.multiPeriodDailyProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCDate(), 2); let filePath @@ -499,21 +704,21 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); } - if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> Entering function."); } + if (LOG_FILE_CONTENT === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> text = " + text); } if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> filePath = " + filePath); - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> fileName = " + fileName); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> filePath = " + filePath); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> fileName = " + fileName); callBackFunction(global.DEFAULT_RETRY_RESPONSE); return; } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); callBackFunction(err); return; } @@ -528,19 +733,19 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoop -> dependencyLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoop -> dependencyLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -549,7 +754,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> dependencyControlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> dependencyControlLoop -> Entering function."); } dependencyIndex++; @@ -567,14 +772,14 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> dependencyControlLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> dependencyControlLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsLoopBody -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -583,7 +788,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsControlLoop -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsControlLoop -> Entering function."); } n++; @@ -600,7 +805,7 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsControlLoop -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> periodsControlLoop -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -609,7 +814,7 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> callTheBot -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> callTheBot -> Entering function."); } usertBot.start( multiPeriodDataFiles, @@ -625,52 +830,65 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> callTheBot -> onBotFinished -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> Entering function."); } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - callBackFunction(err); return; } botNeverRan = false; - writeDataRanges(onWritten); + if (currentTimePeriod > global.dailyFilePeriods[0][0]) { + writeMarketStatusReport(onMarketStatusReport); + } else { + writeDataRanges(onWritten); + } function onWritten(err) { try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> callTheBot -> onBotFinished -> onWritten -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onWritten -> Entering function."); } if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); callBackFunction(err); return; } - writeStatusReport(bot.multiPeriodDailyProcessDatetime, advanceTime); + writeDailyStatusReport(bot.multiPeriodDailyProcessDatetime, advanceTime); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onWritten -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function onMarketStatusReport() { + try { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> Entering function."); } + callBackFunction(global.DEFAULT_OK_RESPONSE); + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> onBotFinished -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> callTheBot -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> callTheBot -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -750,10 +968,10 @@ } } - function writeStatusReport(lastFileDate, callBack) { + function writeDailyStatusReport(lastFileDate, callBack) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> Entering function."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeStatusReport -> lastFileDate = " + lastFileDate); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDailyStatusReport -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeDailyStatusReport -> lastFileDate = " + lastFileDate); } try { @@ -768,10 +986,30 @@ bot.hasTheBotJustStarted = false; } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> writeStatusReport -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> writeDailyStatusReport -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } + + function writeMarketStatusReport(callBack) { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> writeMarketStatusReport -> Entering function."); } + + try { + + let reportKey = bot.devTeam + "-" + bot.codeName + "-" + "Multi-Period" + "-" + "dataSet.V1"; + let thisReport = statusDependencies.statusReports.get(reportKey); + + thisReport.file.lastExecution = bot.processDatetime; + thisReport.save(callBack); + + } + catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> writeMarketStatusReport -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + } catch (err) { From 0074b7ce0f943ff206b550553def420bff1cf7ed Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 29 Aug 2019 19:56:21 +0200 Subject: [PATCH 13/32] Removing Temporary code --- SensorBotProcessMainLoop.js | 2 ++ TradingEngineProcessMainLoop.js | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index 8a6312951..37ac495e7 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -51,6 +51,7 @@ return; } USER_BOT_MODULE = {} + USER_BOT_MODULE.newUserBot = eval(text); // TODO This needs to be changed function filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; @@ -66,6 +67,7 @@ return; } COMMONS_MODULE = {} + COMMONS_MODULE.newCommons = eval(text); // TODO This needs to be changed function callBackFunction(global.DEFAULT_OK_RESPONSE); diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 18ba4f06b..c1a6a7ecc 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -56,8 +56,8 @@ } USER_BOT_MODULE = {} - USER_BOT_MODULE.newUserBot = require(process.env.BOTS_PATH + '/aamasters/AAMasters/bots/AAJason-Trading-Engine-Bot/Multi-Period/User.Bot').newUserBot // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. - //USER_BOT_MODULE.newUserBot = eval(text); // Use this for production + //USER_BOT_MODULE.newUserBot = require(process.env.BOTS_PATH + '/aamasters/AAMasters/bots/AAJason-Trading-Engine-Bot/Multi-Period/User.Bot').newUserBot // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. + USER_BOT_MODULE.newUserBot = eval(text); // Use this for production filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; filePath += "/Commons.js" @@ -73,8 +73,8 @@ } COMMONS_MODULE = {} - COMMONS_MODULE.newCommons = require(process.env.BOTS_PATH + '/aamasters/AAMasters/bots/AAJason-Trading-Engine-Bot/Commons').newCommons // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. - //COMMONS_MODULE.newCommons = eval(text); // Use this for production + //COMMONS_MODULE.newCommons = require(process.env.BOTS_PATH + '/aamasters/AAMasters/bots/AAJason-Trading-Engine-Bot/Commons').newCommons // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. + COMMONS_MODULE.newCommons = eval(text); // Use this for production callBackFunction(global.DEFAULT_OK_RESPONSE); } From 0dfe957db734c0794ca8024bbd30f33625489c0a Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 5 Sep 2019 18:53:32 +0200 Subject: [PATCH 14/32] IPC Event Handler Implemented and working well --- Assistant.js | 71 ++++++++--------- EventHandler.js | 143 ++++++++++++++++++++++++++++++++++ MultiPeriod.js | 4 + Project.njsproj | 3 + Root.js | 27 ------- Run.js | 77 +++++++++++++++--- SensorBotProcessMainLoop.js | 4 +- TradingBotProcessMainLoop.js | 5 -- package-lock.json | 33 ++++++++ package.json | 1 + sequence - all processes.json | 57 ++------------ sequence.json | 10 +-- 12 files changed, 299 insertions(+), 136 deletions(-) create mode 100644 EventHandler.js diff --git a/Assistant.js b/Assistant.js index 75ac8c03a..293a657fd 100644 --- a/Assistant.js +++ b/Assistant.js @@ -63,7 +63,8 @@ } case 'Backtest': { - getMarketRateFromIndicator(); + // getMarketRateFromIndicator(); NOTE: This path is disabled since for now we do not allow ticker in simulation. Also it allow us to ship a dataset without bruce + validateExchangeSyncronicity(); break; } @@ -112,13 +113,13 @@ validateExchangeSyncronicity(); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] initialize -> getMarketRateFromExchange -> onTicker -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] initialize -> getMarketRateFromExchange -> onTicker -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] initialize -> getMarketRateFromExchange -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] initialize -> getMarketRateFromExchange -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -156,12 +157,12 @@ let candleArray; if (err.result === global.CUSTOM_FAIL_RESPONSE.result || err.code === "The specified key does not exist." || err.message === "File does not exist.") { // Just past midnight, this file will not exist for a couple of minutes. - + logger.write(MODULE_NAME, "[WARN] initialize -> getMarketRateFromIndicator -> onFileReceived -> err = " + JSON.stringify(err)); logger.write(MODULE_NAME, "[WARN] initialize -> getMarketRateFromIndicator -> This could happen when there are still holes on trades and the process needs to catch up. "); callBackFunction(global.DEFAULT_FAIL_RESPONSE); return; - + } if (err.result === global.DEFAULT_OK_RESPONSE.result) { @@ -211,7 +212,7 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] initialize -> getMarketRateFromIndicator -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] initialize -> getMarketRateFromIndicator -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -253,13 +254,13 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] initialize -> validateExchangeSyncronicity -> onDone -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] initialize -> validateExchangeSyncronicity -> onDone -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] initialize -> onDone -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] initialize -> onDone -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -313,14 +314,14 @@ callBackFunction(global.DEFAULT_OK_RESPONSE); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] initialize -> calculateProfits -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] initialize -> calculateProfits -> err = " + err.stack); callBack(global.DEFAULT_FAIL_RESPONSE); return; } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -399,7 +400,7 @@ } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] getPositionsAtExchange -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] getPositionsAtExchange -> err = " + err.stack); callBack(global.DEFAULT_FAIL_RESPONSE); } } @@ -561,17 +562,17 @@ if (thisPosition.id === pPositionId) { - let feeRate = 0.002; // Default backtesting fee simulation + let feeRate = 0.002; // Default backtesting fee simulation - let trade = { - id: Math.trunc(Math.random(1) * 1000000), - type: thisPosition.type, - rate: thisPosition.rate.toString(), + let trade = { + id: Math.trunc(Math.random(1) * 1000000), + type: thisPosition.type, + rate: thisPosition.rate.toString(), amountA: thisObject.truncDecimals(thisPosition.amountA).toString(), amountB: thisObject.truncDecimals(thisPosition.amountB).toString(), fee: thisObject.truncDecimals(feeRate).toString(), - date: (new Date()).valueOf() - } + date: (new Date()).valueOf() + } trades.push(trade); @@ -626,12 +627,12 @@ break; } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> getPositionTradesAtExchange -> onResponse -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> getPositionTradesAtExchange -> onResponse -> err = " + err.stack); callBack(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> getPositionTradesAtExchange -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> getPositionTradesAtExchange -> err = " + err.stack); callBack(global.DEFAULT_FAIL_RESPONSE); } } @@ -656,7 +657,7 @@ let sumRate = 0; for (let k = 0; k < pTrades.length; k++) { - let trade = pTrades[k]; + let trade = pTrades[k]; sumAssetA = sumAssetA + thisObject.truncDecimals(trade.amountA); sumAssetB = sumAssetB + thisObject.truncDecimals(trade.amountB); sumRate = sumRate + thisObject.truncDecimals(trade.rate); @@ -704,7 +705,7 @@ next(); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> confirmOrderWasExecuted -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> confirmOrderWasExecuted -> err = " + err.stack); callBack(global.DEFAULT_FAIL_RESPONSE); return; } @@ -726,8 +727,8 @@ let sumAssetA = 0; let sumAssetB = 0; - for (let k = 0; k < pTrades.length; k++) { - let trade = pTrades[k]; + for (let k = 0; k < pTrades.length; k++) { + let trade = pTrades[k]; sumAssetA = sumAssetA + thisObject.truncDecimals(trade.amountA); sumAssetB = sumAssetB + thisObject.truncDecimals(trade.amountB); } @@ -749,7 +750,7 @@ */ let exchangeParam = exchangeAPI.getMaxDecimalPositions(); - let minValue = '0.' + (1).toPrecision(exchangeParam-1).split('.').reverse().join(''); + let minValue = '0.' + (1).toPrecision(exchangeParam - 1).split('.').reverse().join(''); let exchangePrecision = parseFloat(parseFloat(minValue).toFixed(exchangeParam)); if (Math.abs(position.amountA - sumAssetA) > exchangePrecision || Math.abs(position.amountB - sumAssetB) > exchangePrecision) { @@ -790,7 +791,7 @@ next(); } catch (err) { - logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> confirmOrderWasPartiallyExecuted -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> confirmOrderWasPartiallyExecuted -> err = " + err.stack); callBack(global.DEFAULT_FAIL_RESPONSE); return; } @@ -857,7 +858,7 @@ context.executionContext.balance.assetA = thisObject.truncDecimals(context.executionContext.balance.assetA + assetA); context.executionContext.balance.assetB = thisObject.truncDecimals(context.executionContext.balance.assetB - assetB); - context.executionContext.availableBalance.assetA = thisObject.truncDecimals(context.executionContext.availableBalance.assetA + assetA); + context.executionContext.availableBalance.assetA = thisObject.truncDecimals(context.executionContext.availableBalance.assetA + assetA); /* Not the available balance for asset B is not affected since it was already reduced when the order was placed. */ @@ -865,14 +866,14 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> applyTradesToContext -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> applyTradesToContext -> err = " + err.stack); callBack(global.DEFAULT_FAIL_RESPONSE); return; } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> loopBody -> err = " + err.stack); callBack(global.DEFAULT_FAIL_RESPONSE); return; } @@ -911,7 +912,7 @@ } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] ordersExecutionCheck -> err = " + err.stack); callBack(global.DEFAULT_FAIL_RESPONSE); } } @@ -1075,12 +1076,12 @@ break; } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] putPosition -> onResponse -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] putPosition -> onResponse -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] putPosition -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] putPosition -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -1199,12 +1200,12 @@ break; } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] movePosition -> onResponse -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] movePosition -> onResponse -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } } catch (err) { - logger.write(MODULE_NAME, "[ERROR] movePosition -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] movePosition -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } @@ -1330,7 +1331,7 @@ let runIndex = context.statusReport.runs.length - 1; context.newHistoryRecord.messageRelevance = pRelevance; - context.newHistoryRecord.messageTitle = bot.startMode + "." + runIndex +": "+ pTitle; + context.newHistoryRecord.messageTitle = bot.startMode + "." + runIndex + ": " + pTitle; context.newHistoryRecord.messageBody = pBody; } diff --git a/EventHandler.js b/EventHandler.js new file mode 100644 index 000000000..401a40a49 --- /dev/null +++ b/EventHandler.js @@ -0,0 +1,143 @@ +exports.newEventHandler = function newEventHandler() { + let thisObject = { + initialize: initialize, + createEventHandler: createEventHandler, + deleteEventHandler: deleteEventHandler, + listenToEvent: listenToEvent, + stopListening: stopListening, + raiseEvent: raiseEvent + } + + let ipc = require('node-ipc'); + let eventListeners = new Map() + let responseWaiters = new Map() + + ipc.config.id = 'Clone Executor'; + ipc.config.retry = 1500; + + return thisObject + + function initialize(callBackFunction) { + + ipc.connectTo( + 'world', + function () { + ipc.of.world.on( + 'connect', + function () { + ipc.log('## Connected to Superalgos Event Server ##'.rainbow, ipc.config.delay); + + let eventCommand = { + action: 'greetings', + from: 'Clone Executor' + } + + sendCommand(eventCommand) + + callBackFunction() + } + ); + ipc.of.world.on( + 'disconnect', + function () { + ipc.log('Disconnected from Superalgos Event Server '.notice); + } + ); + ipc.of.world.on( + 'message', // here is where all incomming messages are processed. + function (data) { + ipc.log('Got a message from Superalgos Event Server: '.debug, data); + + let message = JSON.parse(data) + + if (message.action === 'Event Raised') { + let handler = eventListeners.get(message.eventHandlerName + '-' + message.eventType) + if (handler) { + handler(message) + } + return + } + + if (message.action === 'Event Server Response') { + let handler = responseWaiters.get(message.callerId) + if (handler) { + handler(message) + } + return + } + } + ); + } + ); + } + + function sendCommand(command, responseCallBack, eventsCallBack) { + + if (command.action === 'listenToEvent') { + eventListeners.set(command.eventHandlerName + '-' + command.eventType, eventsCallBack) + } + if (command.callerId && responseCallBack) { + responseWaiters.set(command.callerId, responseCallBack) + } + + ipc.of.world.emit( + 'message', + JSON.stringify(command) + ) + } + + function createEventHandler(eventHandlerName, callerId, responseCallBack) { + let eventCommand = { + action: 'createEventHandler', + eventHandlerName: eventHandlerName, + callerId: callerId + } + + sendCommand(eventCommand, responseCallBack) + } + + function deleteEventHandler(eventHandlerName, callerId, responseCallBack) { + let eventCommand = { + action: 'deleteEventHandler', + eventHandlerName: eventHandlerName, + callerId: callerId + } + + sendCommand(eventCommand, responseCallBack) + } + + function listenToEvent(eventHandlerName, eventType, extraData, callerId, responseCallBack, eventsCallBack) { + eventCommand = { + action: 'listenToEvent', + eventHandlerName: eventHandlerName, + eventType: eventType, + extraData: extraData, + callerId: callerId + } + + sendCommand(eventCommand, responseCallBack, eventsCallBack) + } + + function stopListening(eventHandlerName, eventSubscriptionId, callerId, responseCallBack) { + eventCommand = { + action: 'stopListening', + eventHandlerName: eventHandlerName, + eventSubscriptionId: eventSubscriptionId, + callerId: callerId + } + + sendCommand(eventCommand, responseCallBack) + } + + function raiseEvent(eventHandlerName, eventType, event, callerId, responseCallBack) { + let eventCommand = { + action: 'raiseEvent', + eventHandlerName: eventHandlerName, + eventType: eventType, + event: event, + callerId: callerId + } + + sendCommand(eventCommand, responseCallBack) + } +} \ No newline at end of file diff --git a/MultiPeriod.js b/MultiPeriod.js index fa414ff94..39745a489 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -984,6 +984,10 @@ thisReport.save(callBack); bot.hasTheBotJustStarted = false; + + /* Emit event that signals that this process finished */ + global.eventHandler.raiseEvent('Jason-Multi-Period', 'Status Report Updated', { lastProcessedDay: lastFileDate }) + } catch (err) { logger.write(MODULE_NAME, "[ERROR] start -> writeDailyStatusReport -> err = " + err.stack); diff --git a/Project.njsproj b/Project.njsproj index a6dd72273..8c8f00478 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -24,6 +24,9 @@ Code + + Code + diff --git a/Root.js b/Root.js index 2e6c123c6..1e031b898 100644 --- a/Root.js +++ b/Root.js @@ -1,32 +1,5 @@ exports.newRoot = function newRoot() { - /* Callbacks default responses. */ - - global.DEFAULT_OK_RESPONSE = { - result: "Ok", - message: "Operation Succeeded" - }; - - global.DEFAULT_FAIL_RESPONSE = { - result: "Fail", - message: "Operation Failed" - }; - - global.DEFAULT_RETRY_RESPONSE = { - result: "Retry", - message: "Retry Later" - }; - - global.CUSTOM_OK_RESPONSE = { - result: "Ok, but check Message", - message: "Custom Message" - }; - - global.CUSTOM_FAIL_RESPONSE = { - result: "Fail Because", - message: "Custom Message" - }; - /* This is the Execution Datetime */ global.EXECUTION_DATETIME = new Date(); diff --git a/Run.js b/Run.js index 389ea570d..6156a8432 100644 --- a/Run.js +++ b/Run.js @@ -1,3 +1,55 @@ +/* Callbacks default responses. */ + +global.DEFAULT_OK_RESPONSE = { + result: "Ok", + message: "Operation Succeeded" +}; + +global.DEFAULT_FAIL_RESPONSE = { + result: "Fail", + message: "Operation Failed" +}; + +global.DEFAULT_RETRY_RESPONSE = { + result: "Retry", + message: "Retry Later" +}; + +global.CUSTOM_OK_RESPONSE = { + result: "Ok, but check Message", + message: "Custom Message" +}; + +global.CUSTOM_FAIL_RESPONSE = { + result: "Fail Because", + message: "Custom Message" +}; + +const EVENT_HANDLER_MODULE = require('./EventHandler.js'); +global.eventHandler = EVENT_HANDLER_MODULE.newEventHandler() + +global.eventHandler.initialize(bootLoader) + +function bootLoader() { + + global.eventHandler.createEventHandler('Jason-Multi-Period') + + tryToListenToCockpit() + + function tryToListenToCockpit() { + + global.eventHandler.listenToEvent('Cockpit-Restart-Simulation', 'Simulation Started', undefined, 'Clone Executor Boot Loader', onResponse, startSequence) + + function onResponse(message) { + if (message.result !== global.DEFAULT_OK_RESPONSE.result) { + setTimeout(tryToListenToCockpit, 3000) + } + } + } +} + +/* Old Run.js code follows... */ + require('dotenv').config(); global.DEFINITION = require(process.env.INTER_PROCESS_FILES_PATH + '/Definition'); @@ -5,7 +57,7 @@ const definition = global.DEFINITION const path = require('path'); global.SHALL_BOT_STOP = false -global.AT_BREAKPOINT = false // This is used only when running at the browser. + global.FULL_LOG = process.env.FULL_LOG /* Default parameters can be changed by the execution configuration */ @@ -40,18 +92,21 @@ if (process.env.RUN_SEQUENCE !== undefined) { isRunSequence = JSON.parse(process.env.RUN_SEQUENCE) } -if (isRunSequence) { - process.on('message', message => { - if (message === 'STOP') { - runClonExecutor = false; - console.log("[INFO] CloneExecutor -> Clone Executor Stopped."); - } - }); - sequenceExecution(sequenceStep) -} else { - readExecutionConfiguration() +function startSequence () { + if (isRunSequence) { + process.on('message', message => { + if (message === 'STOP') { + runClonExecutor = false; + console.log("[INFO] CloneExecutor -> Clone Executor Stopped."); + } + }); + sequenceExecution(sequenceStep) + } else { + readExecutionConfiguration() + } } + async function sequenceExecution(currentStep) { let execution = sequenceList[currentStep]; diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index 37ac495e7..73c03c2ed 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -51,7 +51,9 @@ return; } USER_BOT_MODULE = {} - + if (bot.process === "Hole-Fixing") { + //USER_BOT_MODULE.newUserBot = require(process.env.BOTS_PATH + '/aamasters/AAMasters/bots/AACharly-Sensor-Bot/Hole-Fixing/User.Bot').newUserBot + } USER_BOT_MODULE.newUserBot = eval(text); // TODO This needs to be changed function filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; diff --git a/TradingBotProcessMainLoop.js b/TradingBotProcessMainLoop.js index 63ba92fc9..6db2d7c29 100644 --- a/TradingBotProcessMainLoop.js +++ b/TradingBotProcessMainLoop.js @@ -260,11 +260,6 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> bot.processDatetime = " + bot.processDatetime.toISOString()); } - if (global.AT_BREAKPOINT === true) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Plot Breakpoint Hit."); } - - } /* High level log entry */ diff --git a/package-lock.json b/package-lock.json index b2391e70d..400d53cef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -202,6 +202,11 @@ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.0.0.tgz", "integrity": "sha512-30xVGqjLjiUOArT4+M5q9sYdvuR4riM6yK9wMcas9Vbp6zZa+ocC9dp6QoftuhTPhFAiLK/0C5Ni2nou/Bk8lg==" }, + "easy-stack": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/easy-stack/-/easy-stack-1.0.0.tgz", + "integrity": "sha1-EskbMIWjfwuqM26UhurEv5Tj54g=" + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -224,6 +229,11 @@ "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz", "integrity": "sha512-zoB603vQReOFvTg5xMl9I1P2PnHsHQQKTEowsKKD7nseUfJq6UWzK+4YtlWUO1nhiQUxe6XMkk+JleSZD1NZFA==" }, + "event-pubsub": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/event-pubsub/-/event-pubsub-4.3.0.tgz", + "integrity": "sha512-z7IyloorXvKbFx9Bpie2+vMJKKx1fH1EN5yiTfp8CiLOTptSYy1g8H4yDpGlEdshL1PBiFtBHepF2cNsqeEeFQ==" + }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -344,6 +354,19 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "js-message": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/js-message/-/js-message-1.0.5.tgz", + "integrity": "sha1-IwDSSxrwjondCVvBpMnJz8uJLRU=" + }, + "js-queue": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/js-queue/-/js-queue-2.0.0.tgz", + "integrity": "sha1-NiITz4YPRo8BJfxslqvBdCUx+Ug=", + "requires": { + "easy-stack": "1.0.0" + } + }, "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", @@ -435,6 +458,16 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "node-ipc": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/node-ipc/-/node-ipc-9.1.1.tgz", + "integrity": "sha512-FAyICv0sIRJxVp3GW5fzgaf9jwwRQxAKDJlmNFUL5hOy+W4X/I5AypyHoq0DXXbo9o/gt79gj++4cMr4jVWE/w==", + "requires": { + "event-pubsub": "4.3.0", + "js-message": "1.0.5", + "js-queue": "2.0.0" + } + }, "nonce": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/nonce/-/nonce-1.0.4.tgz", diff --git a/package.json b/package.json index c676322d5..d13992b3a 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "@superalgos/mqservice": "^0.0.17", "axios": "^0.19.0", "dotenv": "^8.0.0", + "node-ipc": "^9.1.1", "winston": "^3.2.1" } } diff --git a/sequence - all processes.json b/sequence - all processes.json index 20650bd26..0ae6028e3 100644 --- a/sequence - all processes.json +++ b/sequence - all processes.json @@ -14,26 +14,26 @@ { "devTeam": "AAMasters", "bot": "AACharly", - "mode": "noTime", + "mode": "oneMonth", "resumeExecution": true, "type": "Sensor", "exchangeName": "Poloniex", "process": "Hole-Fixing", "startYear": 2019, - "endYear": 2021, - "month": 1 + "endYear": 2019, + "month": 9 }, { "devTeam": "AAMasters", "bot": "AABruce", - "mode": "noTime", + "mode": "oneMonth", "resumeExecution": true, "type": "Indicator", "exchangeName": "Poloniex", "process": "Single-Period-Daily", "startYear": 2019, "endYear": 2021, - "month": 1 + "month": 9 }, { "devTeam": "AAMasters", @@ -141,52 +141,5 @@ "process": "Multi-Period-Daily", "beginDatetime": "2019-08-01T08:00:00.000Z", "timePeriod": "05-min" - }, - { - "devTeam": "AAVikings", - "bot": "AAGauss", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAVikings", - "bot": "AAGauss", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Market", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "vikingsalerts", - "bot": "simulator-bot-vikingsalerts", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "beginDatetime": "2019-08-01T08:00:00.000Z", - "timePeriod": "05-min" - }, - { - "devTeam": "vikingsalerts", - "bot": "bot-vikingsalerts", - "mode": "live", - "resumeExecution": true, - "type": "Trading", - "exchangeName": "Poloniex", - "process": "Trading-Process", - "beginDatetime": "2019-08-01T08:00:00.000Z", - "timePeriod": "05-min", - "dataSet": "Multi-Period-Daily" } ] diff --git a/sequence.json b/sequence.json index 10ca0ec47..0ae6028e3 100644 --- a/sequence.json +++ b/sequence.json @@ -14,26 +14,26 @@ { "devTeam": "AAMasters", "bot": "AACharly", - "mode": "noTime", + "mode": "oneMonth", "resumeExecution": true, "type": "Sensor", "exchangeName": "Poloniex", "process": "Hole-Fixing", "startYear": 2019, - "endYear": 2021, - "month": 1 + "endYear": 2019, + "month": 9 }, { "devTeam": "AAMasters", "bot": "AABruce", - "mode": "noTime", + "mode": "oneMonth", "resumeExecution": true, "type": "Indicator", "exchangeName": "Poloniex", "process": "Single-Period-Daily", "startYear": 2019, "endYear": 2021, - "month": 1 + "month": 9 }, { "devTeam": "AAMasters", From 10c631b5b41c2b6f6addd127aa0cd8ab7d67c9b7 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 6 Sep 2019 07:29:29 +0200 Subject: [PATCH 15/32] Web Sockets Implemented with one example of system events working --- MultiPeriod.js | 2 +- Run.js | 8 ++++---- EventHandler.js => SystemEventHandler.js | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename EventHandler.js => SystemEventHandler.js (98%) diff --git a/MultiPeriod.js b/MultiPeriod.js index 39745a489..ab5ede991 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -986,7 +986,7 @@ bot.hasTheBotJustStarted = false; /* Emit event that signals that this process finished */ - global.eventHandler.raiseEvent('Jason-Multi-Period', 'Status Report Updated', { lastProcessedDay: lastFileDate }) + global.systemEventHandler.raiseEvent('Jason-Multi-Period', 'Status Report Updated', { lastProcessedDay: lastFileDate }) } catch (err) { diff --git a/Run.js b/Run.js index 6156a8432..5b47e5875 100644 --- a/Run.js +++ b/Run.js @@ -26,19 +26,19 @@ global.CUSTOM_FAIL_RESPONSE = { }; const EVENT_HANDLER_MODULE = require('./EventHandler.js'); -global.eventHandler = EVENT_HANDLER_MODULE.newEventHandler() +global.systemEventHandler = EVENT_HANDLER_MODULE.newSystemEventHandler() -global.eventHandler.initialize(bootLoader) +global.systemEventHandler.initialize(bootLoader) function bootLoader() { - global.eventHandler.createEventHandler('Jason-Multi-Period') + global.systemEventHandler.createEventHandler('Jason-Multi-Period') tryToListenToCockpit() function tryToListenToCockpit() { - global.eventHandler.listenToEvent('Cockpit-Restart-Simulation', 'Simulation Started', undefined, 'Clone Executor Boot Loader', onResponse, startSequence) + global.systemEventHandler.listenToEvent('Cockpit-Restart-Simulation', 'Simulation Started', undefined, 'Clone Executor Boot Loader', onResponse, startSequence) function onResponse(message) { if (message.result !== global.DEFAULT_OK_RESPONSE.result) { diff --git a/EventHandler.js b/SystemEventHandler.js similarity index 98% rename from EventHandler.js rename to SystemEventHandler.js index 401a40a49..d7bf033d0 100644 --- a/EventHandler.js +++ b/SystemEventHandler.js @@ -1,4 +1,4 @@ -exports.newEventHandler = function newEventHandler() { +exports.newSystemEventHandler = function newSystemEventHandler() { let thisObject = { initialize: initialize, createEventHandler: createEventHandler, From 83d88758c5a58acb086eda1a32e8e5d572aa11a2 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 6 Sep 2019 08:33:24 +0200 Subject: [PATCH 16/32] System Event Handler Working --- Project.njsproj | 2 +- Run.js | 2 +- SystemEventHandler.js | 9 ++------- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Project.njsproj b/Project.njsproj index 8c8f00478..f1c1237bd 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -24,7 +24,7 @@ Code - + Code diff --git a/Run.js b/Run.js index 5b47e5875..1a1891aeb 100644 --- a/Run.js +++ b/Run.js @@ -25,7 +25,7 @@ global.CUSTOM_FAIL_RESPONSE = { message: "Custom Message" }; -const EVENT_HANDLER_MODULE = require('./EventHandler.js'); +const EVENT_HANDLER_MODULE = require('./SystemEventHandler.js'); global.systemEventHandler = EVENT_HANDLER_MODULE.newSystemEventHandler() global.systemEventHandler.initialize(bootLoader) diff --git a/SystemEventHandler.js b/SystemEventHandler.js index d7bf033d0..a5d231b2b 100644 --- a/SystemEventHandler.js +++ b/SystemEventHandler.js @@ -92,7 +92,6 @@ eventHandlerName: eventHandlerName, callerId: callerId } - sendCommand(eventCommand, responseCallBack) } @@ -102,30 +101,27 @@ eventHandlerName: eventHandlerName, callerId: callerId } - sendCommand(eventCommand, responseCallBack) } function listenToEvent(eventHandlerName, eventType, extraData, callerId, responseCallBack, eventsCallBack) { - eventCommand = { + let eventCommand = { action: 'listenToEvent', eventHandlerName: eventHandlerName, eventType: eventType, extraData: extraData, callerId: callerId } - sendCommand(eventCommand, responseCallBack, eventsCallBack) } function stopListening(eventHandlerName, eventSubscriptionId, callerId, responseCallBack) { - eventCommand = { + let eventCommand = { action: 'stopListening', eventHandlerName: eventHandlerName, eventSubscriptionId: eventSubscriptionId, callerId: callerId } - sendCommand(eventCommand, responseCallBack) } @@ -137,7 +133,6 @@ event: event, callerId: callerId } - sendCommand(eventCommand, responseCallBack) } } \ No newline at end of file From e3e24d2dd5e1201dfff194a62485ec1bca5071a5 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 7 Sep 2019 14:15:48 +0200 Subject: [PATCH 17/32] Created and destroyed by Backend Server --- Project.njsproj | 10 ++- SystemEventHandler.js | 138 ------------------------------------------ Run.js => server.js | 26 +++++++- 3 files changed, 28 insertions(+), 146 deletions(-) delete mode 100644 SystemEventHandler.js rename Run.js => server.js (95%) diff --git a/Project.njsproj b/Project.njsproj index f1c1237bd..2d9bfbc32 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -6,14 +6,15 @@ {ae0a3fe2-7021-4cc2-9922-a887ca5cdd7c} ShowAllFiles - Run.js + server.js .\ . {3AF33F2E-1136-4D97-BBB7-1795711AC8B8};{349c5851-65df-11da-9384-00065b846f21};{9092AA53-FB77-4645-B42D-1CCCA6BD08BD} 14.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) Project - Trading-Process + + C:\Program Files\nodejs\node.exe False Debug|Any CPU @@ -24,9 +25,6 @@ Code - - Code - @@ -67,7 +65,7 @@ Code - + Code diff --git a/SystemEventHandler.js b/SystemEventHandler.js deleted file mode 100644 index a5d231b2b..000000000 --- a/SystemEventHandler.js +++ /dev/null @@ -1,138 +0,0 @@ -exports.newSystemEventHandler = function newSystemEventHandler() { - let thisObject = { - initialize: initialize, - createEventHandler: createEventHandler, - deleteEventHandler: deleteEventHandler, - listenToEvent: listenToEvent, - stopListening: stopListening, - raiseEvent: raiseEvent - } - - let ipc = require('node-ipc'); - let eventListeners = new Map() - let responseWaiters = new Map() - - ipc.config.id = 'Clone Executor'; - ipc.config.retry = 1500; - - return thisObject - - function initialize(callBackFunction) { - - ipc.connectTo( - 'world', - function () { - ipc.of.world.on( - 'connect', - function () { - ipc.log('## Connected to Superalgos Event Server ##'.rainbow, ipc.config.delay); - - let eventCommand = { - action: 'greetings', - from: 'Clone Executor' - } - - sendCommand(eventCommand) - - callBackFunction() - } - ); - ipc.of.world.on( - 'disconnect', - function () { - ipc.log('Disconnected from Superalgos Event Server '.notice); - } - ); - ipc.of.world.on( - 'message', // here is where all incomming messages are processed. - function (data) { - ipc.log('Got a message from Superalgos Event Server: '.debug, data); - - let message = JSON.parse(data) - - if (message.action === 'Event Raised') { - let handler = eventListeners.get(message.eventHandlerName + '-' + message.eventType) - if (handler) { - handler(message) - } - return - } - - if (message.action === 'Event Server Response') { - let handler = responseWaiters.get(message.callerId) - if (handler) { - handler(message) - } - return - } - } - ); - } - ); - } - - function sendCommand(command, responseCallBack, eventsCallBack) { - - if (command.action === 'listenToEvent') { - eventListeners.set(command.eventHandlerName + '-' + command.eventType, eventsCallBack) - } - if (command.callerId && responseCallBack) { - responseWaiters.set(command.callerId, responseCallBack) - } - - ipc.of.world.emit( - 'message', - JSON.stringify(command) - ) - } - - function createEventHandler(eventHandlerName, callerId, responseCallBack) { - let eventCommand = { - action: 'createEventHandler', - eventHandlerName: eventHandlerName, - callerId: callerId - } - sendCommand(eventCommand, responseCallBack) - } - - function deleteEventHandler(eventHandlerName, callerId, responseCallBack) { - let eventCommand = { - action: 'deleteEventHandler', - eventHandlerName: eventHandlerName, - callerId: callerId - } - sendCommand(eventCommand, responseCallBack) - } - - function listenToEvent(eventHandlerName, eventType, extraData, callerId, responseCallBack, eventsCallBack) { - let eventCommand = { - action: 'listenToEvent', - eventHandlerName: eventHandlerName, - eventType: eventType, - extraData: extraData, - callerId: callerId - } - sendCommand(eventCommand, responseCallBack, eventsCallBack) - } - - function stopListening(eventHandlerName, eventSubscriptionId, callerId, responseCallBack) { - let eventCommand = { - action: 'stopListening', - eventHandlerName: eventHandlerName, - eventSubscriptionId: eventSubscriptionId, - callerId: callerId - } - sendCommand(eventCommand, responseCallBack) - } - - function raiseEvent(eventHandlerName, eventType, event, callerId, responseCallBack) { - let eventCommand = { - action: 'raiseEvent', - eventHandlerName: eventHandlerName, - eventType: eventType, - event: event, - callerId: callerId - } - sendCommand(eventCommand, responseCallBack) - } -} \ No newline at end of file diff --git a/Run.js b/server.js similarity index 95% rename from Run.js rename to server.js index 1a1891aeb..6958dcdf1 100644 --- a/Run.js +++ b/server.js @@ -25,10 +25,12 @@ global.CUSTOM_FAIL_RESPONSE = { message: "Custom Message" }; -const EVENT_HANDLER_MODULE = require('./SystemEventHandler.js'); +console.log('NEW INSTANCE OF BACKEND SEVER RUNNING') + +const EVENT_HANDLER_MODULE = require('../Libraries/SystemEventsClient/SystemEventHandler.js'); global.systemEventHandler = EVENT_HANDLER_MODULE.newSystemEventHandler() -global.systemEventHandler.initialize(bootLoader) +global.systemEventHandler.initialize('Clone Executor', bootLoader) function bootLoader() { @@ -48,6 +50,24 @@ function bootLoader() { } } +/* + +We read the first string sent as an argument when the process was created by the Backend Server. Ther we will find the information of the identity +of this backend process and know exactly what to run within this server instance. + +*/ + +let backendProcessDefinition = process.argv[2] +if (backendProcessDefinition !== undefined) { + console.log('RECIBI ESTA DEFINICION: ' + backendProcessDefinition) + backendProcessDefinition = JSON.parse(backendProcessDefinition) +} else { + /* Debugging Definition goes here */ + backendProcessDefinition = { + + } +} + /* Old Run.js code follows... */ require('dotenv').config(); @@ -97,6 +117,8 @@ function startSequence () { process.on('message', message => { if (message === 'STOP') { runClonExecutor = false; + global.systemEventHandler.finalize() + global.systemEventHandler = undefined console.log("[INFO] CloneExecutor -> Clone Executor Stopped."); } }); From 3c3aa19caaf8dd1bc82071a086e0991637403286 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 8 Sep 2019 10:33:19 +0200 Subject: [PATCH 18/32] Working with Charly --- MultiPeriod.js | 2 +- SensorBotProcessMainLoop.js | 10 +++ sequence.json | 5 +- server.js | 137 +++++++++++++++++++++++------------- 4 files changed, 102 insertions(+), 52 deletions(-) diff --git a/MultiPeriod.js b/MultiPeriod.js index ab5ede991..56cd5480e 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -986,7 +986,7 @@ bot.hasTheBotJustStarted = false; /* Emit event that signals that this process finished */ - global.systemEventHandler.raiseEvent('Jason-Multi-Period', 'Status Report Updated', { lastProcessedDay: lastFileDate }) + global.SYSTEM_EVENT_HANDLER.raiseEvent('Jason-Multi-Period', 'Status Report Updated', { lastProcessedDay: lastFileDate }) } catch (err) { diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index 73c03c2ed..0173d8611 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -122,6 +122,16 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Entering function."); } + /* Loop Heartbeat sent to the UI */ + //hearBeat() + function hearBeat() { + let key = global.USER_DEFINITION.name + '-' + global.USER_DEFINITION.type + let event = { + seconds: (new Date()).getSeconds() + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) + } + /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ const UTILITIES = require(ROOT_DIR + 'CloudUtilities'); diff --git a/sequence.json b/sequence.json index 0ae6028e3..c0db3b13f 100644 --- a/sequence.json +++ b/sequence.json @@ -6,10 +6,7 @@ "resumeExecution": true, "type": "Sensor", "exchangeName": "Poloniex", - "process": "Live-Trades", - "startYear": 2019, - "endYear": 2021, - "month": 1 + "process": "Live-Trades" }, { "devTeam": "AAMasters", diff --git a/server.js b/server.js index 6958dcdf1..7126cb938 100644 --- a/server.js +++ b/server.js @@ -25,30 +25,28 @@ global.CUSTOM_FAIL_RESPONSE = { message: "Custom Message" }; -console.log('NEW INSTANCE OF BACKEND SEVER RUNNING') +/* Process Events */ -const EVENT_HANDLER_MODULE = require('../Libraries/SystemEventsClient/SystemEventHandler.js'); -global.systemEventHandler = EVENT_HANDLER_MODULE.newSystemEventHandler() +process.on('uncaughtException', function (err) { + console.log('[INFO] server -> uncaughtException -> err.message = ' + err.message) + console.log('[INFO] server -> uncaughtException -> err.stack = ' + err.stack) + process.exit(1) +}) -global.systemEventHandler.initialize('Clone Executor', bootLoader) +process.on('unhandledRejection', (reason, p) => { + console.log('[INFO] server -> unhandledRejection -> reason = ' + JSON.stringify(reason)) + console.log('[INFO] server -> unhandledRejection -> p = ' + JSON.stringify(p)) + process.exit(1) +}) -function bootLoader() { +process.on('exit', function (code) { + console.log('[INFO] server -> process.on.exit -> About to exit -> code = ' + code) +}) - global.systemEventHandler.createEventHandler('Jason-Multi-Period') +/* Local Variables */ - tryToListenToCockpit() - - function tryToListenToCockpit() { - - global.systemEventHandler.listenToEvent('Cockpit-Restart-Simulation', 'Simulation Started', undefined, 'Clone Executor Boot Loader', onResponse, startSequence) - - function onResponse(message) { - if (message.result !== global.DEFAULT_OK_RESPONSE.result) { - setTimeout(tryToListenToCockpit, 3000) - } - } - } -} +let sequenceList = [] +let heartBeatInterval /* @@ -56,17 +54,71 @@ We read the first string sent as an argument when the process was created by the of this backend process and know exactly what to run within this server instance. */ +global.USER_DEFINITION = process.argv[2] -let backendProcessDefinition = process.argv[2] -if (backendProcessDefinition !== undefined) { - console.log('RECIBI ESTA DEFINICION: ' + backendProcessDefinition) - backendProcessDefinition = JSON.parse(backendProcessDefinition) -} else { - /* Debugging Definition goes here */ - backendProcessDefinition = { +if (global.USER_DEFINITION !== undefined) { + console.log('[INFO] server -> global.USER_DEFINITION = ' + global.USER_DEFINITION) + try { + global.USER_DEFINITION = JSON.parse(global.USER_DEFINITION) + } catch (err) { + console.log('[ERROR] server -> global.USER_DEFINITION -> ' + err.stack) + } +} +/* +else { + let argument = '{"type":"Backend Process","name":"New Backend Process","bot":{"type":"Sensor","botProcesses":[{"type":"Bot Process","name":"New Bot Process","code":{"devTeam":"AAMasters","bot":"AACharly","mode":"noTime","resumeExecution":true,"type":"Sensor","exchangeName":"Poloniex","process":"Live-Trades"}}]}}' + try { + global.USER_DEFINITION = JSON.parse(argument) + } catch (err) { + console.log(err.stack) } } +*/ + +const EVENT_HANDLER_MODULE = require('../Libraries/SystemEventsClient/SystemEventHandler.js'); +global.SYSTEM_EVENT_HANDLER = EVENT_HANDLER_MODULE.newSystemEventHandler() + +global.SYSTEM_EVENT_HANDLER.initialize('Clone Executor', bootLoader) + +function bootLoader() { + + global.SYSTEM_EVENT_HANDLER.createEventHandler(global.USER_DEFINITION.name) + + /* + tryToListenToCockpit() + + function tryToListenToCockpit() { + + global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Simulation', 'Simulation Started', undefined, 'Clone Executor Boot Loader', onResponse, startSequence) + + function onResponse(message) { + if (message.result !== global.DEFAULT_OK_RESPONSE.result) { + setTimeout(tryToListenToCockpit, 3000) + } + } + } + */ + + for (let i = 0; i < global.USER_DEFINITION.bot.botProcesses.length; i++) { + let code = global.USER_DEFINITION.bot.botProcesses[i].code + sequenceList.push(code) + } + + /* Heartbeat sent to the UI */ + heartBeatInterval = setInterval(hearBeat, 1000) + + function hearBeat() { + let key = global.USER_DEFINITION.name + '-' + global.USER_DEFINITION.type + let event = { + seconds: (new Date()).getSeconds() + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) + } + + startSequence() +} + /* Old Run.js code follows... */ @@ -85,23 +137,10 @@ global.EXCHANGE_NAME = process.env.EXCHANGE_NAME global.MARKET = { assetA: 'USDT', assetB: 'BTC' } global.CLONE_EXECUTOR = { codeName: 'AACloud', version: '1.1' } -process.on('uncaughtException', function (err) { - console.log('[INFO] Run -> uncaughtException -> err.message = ' + err.message) - console.log('[INFO] Run -> uncaughtException -> err.stack = ' + err.stack) - process.exit(1) -}) -process.on('unhandledRejection', (reason, p) => { - console.log('[INFO] Run -> unhandledRejection -> reason = ' + JSON.stringify(reason)) - console.log('[INFO] Run -> unhandledRejection -> p = ' + JSON.stringify(p)) - process.exit(1) -}) -process.on('exit', function (code) { - console.log('[INFO] Run -> process.on.exit -> About to exit -> code = ' + code) -}) -let sequenceList = require('./sequence'); + let isRunSequence = false; let sequenceStep = 0; let processedSteps = new Map(); @@ -117,8 +156,12 @@ function startSequence () { process.on('message', message => { if (message === 'STOP') { runClonExecutor = false; - global.systemEventHandler.finalize() - global.systemEventHandler = undefined + + /* Cleaning Before Exiting. */ + clearInterval(heartBeatInterval) + + global.SYSTEM_EVENT_HANDLER.finalize() + global.SYSTEM_EVENT_HANDLER = undefined console.log("[INFO] CloneExecutor -> Clone Executor Stopped."); } }); @@ -191,7 +234,7 @@ function onExecutionFinish(result, finishStepKey) { setTimeout(function () { if (runClonExecutor) { console.log("[INFO] onExecutionFinish -> New round for sequence execution started."); - sequenceList = require('./sequence'); // We read again the sequence after every loop + sequenceStep = 0; processedSteps = new Map(); notFirstSequence = true; @@ -204,7 +247,7 @@ function onExecutionFinish(result, finishStepKey) { async function readExecutionConfiguration(execution) { try { - console.log("[INFO] Run -> readExecutionConfiguration -> Entering function. "); + console.log("[INFO] server -> readExecutionConfiguration -> Entering function. "); let timePeriodFilter let botProcess @@ -442,7 +485,7 @@ function getTimePeriod(timePeriod) { timePeriodMap.set('01-min', 60000) return timePeriodMap.get(timePeriod) } catch (error) { - console.log('[WARN] Run -> readExecutionConfiguration -> getTimePeriod -> Error: ', error) + console.log('[WARN] server -> readExecutionConfiguration -> getTimePeriod -> Error: ', error) } } else { return undefined @@ -450,7 +493,7 @@ function getTimePeriod(timePeriod) { } function startRoot() { - console.log('[INFO] Run -> startRoot -> Entering function. ') + console.log('[INFO] server -> startRoot -> Entering function. ') const ROOT_DIR = './' const ROOT_MODULE = require(ROOT_DIR + 'Root') @@ -467,7 +510,7 @@ function startRoot() { root.initialize(UI_COMMANDS, onInitialized) function onInitialized() { - console.log('[INFO] Run -> startRoot -> onInitialized -> Entering function. ') + console.log('[INFO] server -> startRoot -> onInitialized -> Entering function. ') root.start(onExecutionFinish) } From e1fe07b380178b6e4172c9f48864cbf6702c4eee Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 8 Sep 2019 12:07:33 +0200 Subject: [PATCH 19/32] Latest Refactorings --- IndicatorBotProcessMainLoop.js | 2 +- SensorBotProcessMainLoop.js | 2 +- server.js | 15 +++++++-------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 52707db55..8a57bbf59 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -2,7 +2,7 @@ const ROOT_DIR = './'; - const MODULE_NAME = "Indicator Bot Process Main Loop"; + const MODULE_NAME = "Indicator Process Main Loop"; const FULL_LOG = true; let USER_BOT_MODULE; diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index 0173d8611..42f9a8c46 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -2,7 +2,7 @@ const ROOT_DIR = './'; - const MODULE_NAME = "Sensor Bot Process Main Loop"; + const MODULE_NAME = "Sensor Process Main Loop"; const FULL_LOG = true; let USER_BOT_MODULE; diff --git a/server.js b/server.js index 7126cb938..5cddc51fd 100644 --- a/server.js +++ b/server.js @@ -50,8 +50,8 @@ let heartBeatInterval /* -We read the first string sent as an argument when the process was created by the Backend Server. Ther we will find the information of the identity -of this backend process and know exactly what to run within this server instance. +We read the first string sent as an argument when the process was created by the Task Manager. Ther we will find the information of the identity +of this Task and know exactly what to run within this server instance. */ global.USER_DEFINITION = process.argv[2] @@ -65,16 +65,15 @@ if (global.USER_DEFINITION !== undefined) { } } -/* -else { - let argument = '{"type":"Backend Process","name":"New Backend Process","bot":{"type":"Sensor","botProcesses":[{"type":"Bot Process","name":"New Bot Process","code":{"devTeam":"AAMasters","bot":"AACharly","mode":"noTime","resumeExecution":true,"type":"Sensor","exchangeName":"Poloniex","process":"Live-Trades"}}]}}' +else { // I use this section to debug in standalone mode. + let argument = '{"type":"Task","name":"Extract Trades from Poloniex","bot":{"type":"Sensor","name":"Charly","processes":[{"type":"Process","name":"Live-Trades","code":{"devTeam":"AAMasters","bot":"AACharly","mode":"noTime","resumeExecution":true,"type":"Sensor","exchangeName":"Poloniex","process":"Live-Trades"}},{"type":"Process","name":"Hole-Fixing","code":{"devTeam":"AAMasters","bot":"AACharly","mode":"oneMonth","resumeExecution":true,"type":"Sensor","exchangeName":"Poloniex","process":"Hole-Fixing","startYear":2019,"endYear":2019,"month":9}}]}}' try { global.USER_DEFINITION = JSON.parse(argument) } catch (err) { console.log(err.stack) } } -*/ + const EVENT_HANDLER_MODULE = require('../Libraries/SystemEventsClient/SystemEventHandler.js'); global.SYSTEM_EVENT_HANDLER = EVENT_HANDLER_MODULE.newSystemEventHandler() @@ -100,8 +99,8 @@ function bootLoader() { } */ - for (let i = 0; i < global.USER_DEFINITION.bot.botProcesses.length; i++) { - let code = global.USER_DEFINITION.bot.botProcesses[i].code + for (let i = 0; i < global.USER_DEFINITION.bot.processes.length; i++) { + let code = global.USER_DEFINITION.bot.processes[i].code sequenceList.push(code) } From 9327a385bcbb4e71174dcff9b566d006ec84793d Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 8 Sep 2019 14:04:03 +0200 Subject: [PATCH 20/32] Sensors with hartbeat working --- SensorBotProcessMainLoop.js | 22 ++++++++++++++-------- TaskServer.sln | 25 +++++++++++++++++++++++++ server.js | 30 +++++++++++++++--------------- 3 files changed, 54 insertions(+), 23 deletions(-) create mode 100644 TaskServer.sln diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index 42f9a8c46..a025be6d0 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -123,14 +123,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Entering function."); } /* Loop Heartbeat sent to the UI */ - //hearBeat() - function hearBeat() { - let key = global.USER_DEFINITION.name + '-' + global.USER_DEFINITION.type - let event = { - seconds: (new Date()).getSeconds() - } - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) - } + hearBeat() /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ @@ -550,6 +543,10 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Entering function."); } if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> nextWaitTime = " + nextWaitTime); } + /* We show we reached the end of the loop. */ + + hearBeat() + /* Here we check if we must stop the loop gracefully. */ shallWeStop(onStop, onContinue); @@ -701,6 +698,15 @@ callBackFunction(err); } } + + function hearBeat() { + let key = global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].name + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].type + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].id + + let event = { + seconds: (new Date()).getSeconds() + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) + } } catch (err) { diff --git a/TaskServer.sln b/TaskServer.sln new file mode 100644 index 000000000..a21837e88 --- /dev/null +++ b/TaskServer.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.27130.2027 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}") = "Project", "Project.njsproj", "{AE0A3FE2-7021-4CC2-9922-A887CA5CDD7C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {AE0A3FE2-7021-4CC2-9922-A887CA5CDD7C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AE0A3FE2-7021-4CC2-9922-A887CA5CDD7C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AE0A3FE2-7021-4CC2-9922-A887CA5CDD7C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AE0A3FE2-7021-4CC2-9922-A887CA5CDD7C}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {9128DF06-8CA5-497C-8BF7-B924571C5394} + EndGlobalSection +EndGlobal diff --git a/server.js b/server.js index 5cddc51fd..424e895cf 100644 --- a/server.js +++ b/server.js @@ -28,19 +28,19 @@ global.CUSTOM_FAIL_RESPONSE = { /* Process Events */ process.on('uncaughtException', function (err) { - console.log('[INFO] server -> uncaughtException -> err.message = ' + err.message) - console.log('[INFO] server -> uncaughtException -> err.stack = ' + err.stack) + console.log('[INFO] Task Server -> server -> uncaughtException -> err.message = ' + err.message) + console.log('[INFO] Task Server -> server -> uncaughtException -> err.stack = ' + err.stack) process.exit(1) }) process.on('unhandledRejection', (reason, p) => { - console.log('[INFO] server -> unhandledRejection -> reason = ' + JSON.stringify(reason)) - console.log('[INFO] server -> unhandledRejection -> p = ' + JSON.stringify(p)) + console.log('[INFO] Task Server -> server -> unhandledRejection -> reason = ' + JSON.stringify(reason)) + console.log('[INFO] Task Server -> server -> unhandledRejection -> p = ' + JSON.stringify(p)) process.exit(1) }) process.on('exit', function (code) { - console.log('[INFO] server -> process.on.exit -> About to exit -> code = ' + code) + console.log('[INFO] Task Server -> server -> process.on.exit -> About to exit -> code = ' + code) }) /* Local Variables */ @@ -57,11 +57,11 @@ of this Task and know exactly what to run within this server instance. global.USER_DEFINITION = process.argv[2] if (global.USER_DEFINITION !== undefined) { - console.log('[INFO] server -> global.USER_DEFINITION = ' + global.USER_DEFINITION) + console.log('[INFO] Task Server -> server -> global.USER_DEFINITION = ' + global.USER_DEFINITION) try { global.USER_DEFINITION = JSON.parse(global.USER_DEFINITION) } catch (err) { - console.log('[ERROR] server -> global.USER_DEFINITION -> ' + err.stack) + console.log('[ERROR] Task Server -> server -> global.USER_DEFINITION -> ' + err.stack) } } @@ -108,7 +108,7 @@ function bootLoader() { heartBeatInterval = setInterval(hearBeat, 1000) function hearBeat() { - let key = global.USER_DEFINITION.name + '-' + global.USER_DEFINITION.type + let key = global.USER_DEFINITION.name + '-' + global.USER_DEFINITION.type + '-' + global.USER_DEFINITION.id let event = { seconds: (new Date()).getSeconds() } @@ -141,7 +141,7 @@ global.CLONE_EXECUTOR = { codeName: 'AACloud', version: '1.1' } let isRunSequence = false; -let sequenceStep = 0; +global.sequenceStep = 0; let processedSteps = new Map(); let notFirstSequence = false; let runClonExecutor = true; @@ -227,14 +227,14 @@ async function sequenceExecution(currentStep) { } function onExecutionFinish(result, finishStepKey) { - if (sequenceStep < sequenceList.length && runClonExecutor) { + if (global.sequenceStep < sequenceList.length && runClonExecutor) { sequenceExecution(sequenceStep); } else { setTimeout(function () { if (runClonExecutor) { console.log("[INFO] onExecutionFinish -> New round for sequence execution started."); - sequenceStep = 0; + global.sequenceStep = 0; processedSteps = new Map(); notFirstSequence = true; sequenceExecution(sequenceStep); @@ -246,7 +246,7 @@ function onExecutionFinish(result, finishStepKey) { async function readExecutionConfiguration(execution) { try { - console.log("[INFO] server -> readExecutionConfiguration -> Entering function. "); + console.log("[INFO] Task Server -> server -> readExecutionConfiguration -> Entering function. "); let timePeriodFilter let botProcess @@ -484,7 +484,7 @@ function getTimePeriod(timePeriod) { timePeriodMap.set('01-min', 60000) return timePeriodMap.get(timePeriod) } catch (error) { - console.log('[WARN] server -> readExecutionConfiguration -> getTimePeriod -> Error: ', error) + console.log('[WARN] Task Server -> server -> readExecutionConfiguration -> getTimePeriod -> Error: ', error) } } else { return undefined @@ -492,7 +492,7 @@ function getTimePeriod(timePeriod) { } function startRoot() { - console.log('[INFO] server -> startRoot -> Entering function. ') + console.log('[INFO] Task Server -> server -> startRoot -> Entering function. ') const ROOT_DIR = './' const ROOT_MODULE = require(ROOT_DIR + 'Root') @@ -509,7 +509,7 @@ function startRoot() { root.initialize(UI_COMMANDS, onInitialized) function onInitialized() { - console.log('[INFO] server -> startRoot -> onInitialized -> Entering function. ') + console.log('[INFO] Task Server -> server -> startRoot -> onInitialized -> Entering function. ') root.start(onExecutionFinish) } From 6b707a2048ddd35e12a9f1ee14d676ed1f4ad447 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 10 Sep 2019 08:00:31 +0200 Subject: [PATCH 21/32] Status Report waiting for events, version 1 --- IndicatorBotProcessMainLoop.js | 16 +++++++ StatusReport.js | 75 ++++++++++++++++++++++++++++++++- TradingEngineProcessMainLoop.js | 16 +++++++ server.js | 23 ++++++++-- 4 files changed, 126 insertions(+), 4 deletions(-) diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 8a57bbf59..b1bdfc5d1 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -120,6 +120,9 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Entering function."); } + /* Loop Heartbeat sent to the UI */ + hearBeat() + /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ const UTILITIES = require(ROOT_DIR + 'CloudUtilities'); @@ -791,6 +794,10 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Entering function."); } if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> nextWaitTime = " + nextWaitTime); } + /* We show we reached the end of the loop. */ + + hearBeat() + /* Here we check if we must stop the loop gracefully. */ shallWeStop(onStop, onContinue); @@ -916,6 +923,15 @@ callBackFunction(err); } } + + function hearBeat() { + let key = global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].name + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].type + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].id + + let event = { + seconds: (new Date()).getSeconds() + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) + } } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] run -> err = "+ err.stack); clearInterval(fixedTimeLoopIntervalHandle); diff --git a/StatusReport.js b/StatusReport.js index 106dfa3a4..1c6d4e497 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -54,7 +54,76 @@ timePath = "/" + year + "/" + month; } - callBackFunction(global.DEFAULT_OK_RESPONSE); + + if (owner.waitUntilNextUpdate === true) { + /* This forces this process to wait until the process that this one depends on, updates its status report. */ + + let key = owner.devTeam + "-" + owner.bot + "-" + owner.process + let callerId = bot.devTeam + "-" + bot.codeName + "-" + bot.process + + let eventSubscriptionIdStatusReport + let eventSubscriptionIdProcessTerminated + + eventSubscriptionIdStatusReport = global.SYSTEM_EVENT_HANDLER.listenToEvent(key, 'Status Report Updated', undefined, callerId, responseCallBackStatusReport, eventsCallBackStatusReport) + + function responseCallBackStatusReport(message) { + if (message.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write(MODULE_NAME, "[ERROR] initialize -> Could not register event listener for event 'Status Report Updated' -> Message = " + message); + stopListeningAll() + let event = { + reason: 'This process depends on ' + key + ' which is currently not running.' + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated', event) + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function eventsCallBackStatusReport() { + /* We continue the normal flow after we learn the dependent process has updated its status report. */ + stopListeningAll() + callBackFunction(global.DEFAULT_OK_RESPONSE); + } + + eventSubscriptionIdProcessTerminated = global.SYSTEM_EVENT_HANDLER.listenToEvent(key, 'Process Terminated', undefined, callerId, responseCallBackProcessTerminated, eventsCallBackProcessTerminated) + + function responseCallBackProcessTerminated(message) { + if (message.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write(MODULE_NAME, "[ERROR] initialize -> Could not register event listener for event 'Process Terminated' -> Message = " + message); + stopListeningAll() + let event = { + reason: 'This process depends on ' + key + ' which is currently not running.' + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated', event) + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function eventsCallBackProcessTerminated() { + /* If the process we depend on to start was terminated, there is no point to continue waiting. We need to cloese everything al terminate this process too. */ + logger.write(MODULE_NAME, "[ERROR] initialize -> Received event 'Process Terminated' from a dependency we are waiting for. -> Message = " + message); + + stopListeningAll() + let event = { + reason: 'This process depends on ' + key + ' which has been terminated.' + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated', event) + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + + function stopListeningAll() { + if (eventSubscriptionIdStatusReport) { + stopListening(key, 'Status Report Updated', eventSubscriptionIdStatusReport) + } + if (eventSubscriptionIdProcessTerminated) { + stopListening(key, 'Process Terminated', eventSubscriptionIdStatusReport) + } + } + + } else { + /* In this case, the Status Report does not depends on a process which needs to wait for. */ + callBackFunction(global.DEFAULT_OK_RESPONSE); + } + } catch (err) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); @@ -199,6 +268,10 @@ logger.write(MODULE_NAME, "[INFO] save -> onFileCreated -> Content written = " + fileContent); } + /* Here we raise the event stating that this status report was updated. */ + let key = bot.devTeam + "-" + bot.codeName + "-" + bot.process + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Status Report Updated') + callBackFunction(global.DEFAULT_OK_RESPONSE); return; } diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index c1a6a7ecc..a8df4efef 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -125,6 +125,9 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Entering function."); } + /* Loop Heartbeat sent to the UI */ + hearBeat() + /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ const UTILITIES = require(ROOT_DIR + 'CloudUtilities'); @@ -870,6 +873,10 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Entering function."); } if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> nextWaitTime = " + nextWaitTime); } + /* We show we reached the end of the loop. */ + + hearBeat() + /* Here we check if we must stop the loop gracefully. */ shallWeStop(onStop, onContinue); @@ -996,6 +1003,15 @@ callBackFunction(err); } } + + function hearBeat() { + let key = global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].name + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].type + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].id + + let event = { + seconds: (new Date()).getSeconds() + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) + } } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] run -> err = "+ err.stack); clearInterval(fixedTimeLoopIntervalHandle); diff --git a/server.js b/server.js index 424e895cf..95a2b25f8 100644 --- a/server.js +++ b/server.js @@ -78,11 +78,11 @@ else { // I use this section to debug in standalone mode. const EVENT_HANDLER_MODULE = require('../Libraries/SystemEventsClient/SystemEventHandler.js'); global.SYSTEM_EVENT_HANDLER = EVENT_HANDLER_MODULE.newSystemEventHandler() -global.SYSTEM_EVENT_HANDLER.initialize('Clone Executor', bootLoader) +global.SYSTEM_EVENT_HANDLER.initialize('Task Server', bootLoader) function bootLoader() { - global.SYSTEM_EVENT_HANDLER.createEventHandler(global.USER_DEFINITION.name) + //global.SYSTEM_EVENT_HANDLER.createEventHandler(global.USER_DEFINITION.name) /* tryToListenToCockpit() @@ -101,7 +101,14 @@ function bootLoader() { for (let i = 0; i < global.USER_DEFINITION.bot.processes.length; i++) { let code = global.USER_DEFINITION.bot.processes[i].code + + /* Add to the execution sequence list. */ sequenceList.push(code) + + /* Create the event handler for each process. This event handlers are where the status reports updated events are raised. */ + + let key = code.devTeam + "-" + code.bot + "-" + code.process + global.SYSTEM_EVENT_HANDLER.createEventHandler(key) } /* Heartbeat sent to the UI */ @@ -153,12 +160,22 @@ if (process.env.RUN_SEQUENCE !== undefined) { function startSequence () { if (isRunSequence) { process.on('message', message => { - if (message === 'STOP') { + if (message === 'Stop this Task') { runClonExecutor = false; /* Cleaning Before Exiting. */ clearInterval(heartBeatInterval) + for (let i = 0; i < global.USER_DEFINITION.bot.processes.length; i++) { + let code = global.USER_DEFINITION.bot.processes[i].code + + /* Delete the event handler for each process. */ + + let key = code.devTeam + "-" + code.codeName + "-" + code.process + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated') + global.SYSTEM_EVENT_HANDLER.deleteEventHandler(key) + } + global.SYSTEM_EVENT_HANDLER.finalize() global.SYSTEM_EVENT_HANDLER = undefined console.log("[INFO] CloneExecutor -> Clone Executor Stopped."); From 4a079accbf070151abd382bb384b736aa17b243a Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 10 Sep 2019 09:43:42 +0200 Subject: [PATCH 22/32] Status Reports Events Syncronized even whe dependent is run first that dependency --- StatusReport.js | 47 ++++++++--------------------------------------- server.js | 7 ++++++- 2 files changed, 14 insertions(+), 40 deletions(-) diff --git a/StatusReport.js b/StatusReport.js index 1c6d4e497..46c4b8e05 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -61,15 +61,13 @@ let key = owner.devTeam + "-" + owner.bot + "-" + owner.process let callerId = bot.devTeam + "-" + bot.codeName + "-" + bot.process - let eventSubscriptionIdStatusReport - let eventSubscriptionIdProcessTerminated + let subscriptionIdStatusReport - eventSubscriptionIdStatusReport = global.SYSTEM_EVENT_HANDLER.listenToEvent(key, 'Status Report Updated', undefined, callerId, responseCallBackStatusReport, eventsCallBackStatusReport) + subscriptionIdStatusReport = global.SYSTEM_EVENT_HANDLER.listenToEvent(key, 'Status Report Updated', undefined, callerId, responseCallBack, eventsCallBack) - function responseCallBackStatusReport(message) { + function responseCallBack(message) { if (message.result !== global.DEFAULT_OK_RESPONSE.result) { logger.write(MODULE_NAME, "[ERROR] initialize -> Could not register event listener for event 'Status Report Updated' -> Message = " + message); - stopListeningAll() let event = { reason: 'This process depends on ' + key + ' which is currently not running.' } @@ -78,44 +76,15 @@ } } - function eventsCallBackStatusReport() { + function eventsCallBack() { /* We continue the normal flow after we learn the dependent process has updated its status report. */ - stopListeningAll() + stopListening() callBackFunction(global.DEFAULT_OK_RESPONSE); } - eventSubscriptionIdProcessTerminated = global.SYSTEM_EVENT_HANDLER.listenToEvent(key, 'Process Terminated', undefined, callerId, responseCallBackProcessTerminated, eventsCallBackProcessTerminated) - - function responseCallBackProcessTerminated(message) { - if (message.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] initialize -> Could not register event listener for event 'Process Terminated' -> Message = " + message); - stopListeningAll() - let event = { - reason: 'This process depends on ' + key + ' which is currently not running.' - } - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated', event) - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function eventsCallBackProcessTerminated() { - /* If the process we depend on to start was terminated, there is no point to continue waiting. We need to cloese everything al terminate this process too. */ - logger.write(MODULE_NAME, "[ERROR] initialize -> Received event 'Process Terminated' from a dependency we are waiting for. -> Message = " + message); - - stopListeningAll() - let event = { - reason: 'This process depends on ' + key + ' which has been terminated.' - } - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated', event) - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - - function stopListeningAll() { - if (eventSubscriptionIdStatusReport) { - stopListening(key, 'Status Report Updated', eventSubscriptionIdStatusReport) - } - if (eventSubscriptionIdProcessTerminated) { - stopListening(key, 'Process Terminated', eventSubscriptionIdStatusReport) + function stopListening() { + if (subscriptionIdStatusReport) { + stopListening(key, 'Status Report Updated', subscriptionIdStatusReport) } } diff --git a/server.js b/server.js index 95a2b25f8..3e7984bc6 100644 --- a/server.js +++ b/server.js @@ -115,6 +115,8 @@ function bootLoader() { heartBeatInterval = setInterval(hearBeat, 1000) function hearBeat() { + + /* The heartbeat event is raised at the event handler of the instance of this task, created at the UI. */ let key = global.USER_DEFINITION.name + '-' + global.USER_DEFINITION.type + '-' + global.USER_DEFINITION.id let event = { seconds: (new Date()).getSeconds() @@ -172,7 +174,10 @@ function startSequence () { /* Delete the event handler for each process. */ let key = code.devTeam + "-" + code.codeName + "-" + code.process - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated') + let event = { + reason: 'Signal Received to Terminate this Process.' + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated', event) global.SYSTEM_EVENT_HANDLER.deleteEventHandler(key) } From 2fc03951aa74141f05719f17e1a854767567315e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 10 Sep 2019 21:01:48 +0200 Subject: [PATCH 23/32] Cleaning and changes to make processes run according to the timing specified at the configuration --- IndicatorBotProcessMainLoop.js | 10 +- Integrations/Definition.js | 51 -- MultiPeriod.js | 2 +- MultiPeriodDaily.js | 2 +- Project.njsproj | 4 - Root.js | 277 ++------- SensorBotProcessMainLoop.js | 14 +- TradingBotProcessMainLoop.js | 1036 ------------------------------- TradingEngineProcessMainLoop.js | 10 +- server.js | 236 +++---- 10 files changed, 150 insertions(+), 1492 deletions(-) delete mode 100644 Integrations/Definition.js delete mode 100644 TradingBotProcessMainLoop.js diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index b1bdfc5d1..6891951ed 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -808,7 +808,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> onStop -> Stopping the Loop Gracefully. See you next time!"); } - if(global.FULL_LOG === 'true'){ + if(global.WRITE_LOGS_TO_FILES === 'true'){ logger.persist(); } @@ -828,7 +828,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.normalWaitTime / 1000) + " seconds."); } checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.normalWaitTime * 5, bot.loopCounter); nextLoopTimeoutHandle = setTimeout(loop, processConfig.normalWaitTime); - if(global.FULL_LOG === 'true'){ + if(global.WRITE_LOGS_TO_FILES === 'true'){ logger.persist(); } } @@ -869,7 +869,7 @@ let now = new Date().valueOf(); - if (now - bot.loopStartTime > processConfig.normalWaitTime) { + if (now - bot.loopStartTime > processConfig.deadWaitTime) { logger.write(MODULE_NAME, "[ERROR] run -> loop -> checkLoopHealth -> Dead loop found -> pLastLoop = " + pLastLoop); console.log((new Date().toISOString() + " [ERROR] run -> loop -> checkLoopHealth -> " + pad(bot.codeName,20) + " " + pad(bot.process,30) + " Loop # " + pad(Number(bot.loopCounter),5) + " found dead. Resurrecting it now.")); @@ -898,7 +898,7 @@ if (process.env.STOP_GRACEFULLY !== undefined) stop = JSON.parse(process.env.STOP_GRACEFULLY) - if (!stop && global.SHALL_BOT_STOP === false) { + if (!stop) { continueCallBack(); } else { stopCallBack(); @@ -925,7 +925,7 @@ } function hearBeat() { - let key = global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].name + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].type + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].id + let key = global.USER_DEFINITION.bot.processes[bot.processIndex].name + '-' + global.USER_DEFINITION.bot.processes[bot.processIndex].type + '-' + global.USER_DEFINITION.bot.processes[bot.processIndex].id let event = { seconds: (new Date()).getSeconds() diff --git a/Integrations/Definition.js b/Integrations/Definition.js deleted file mode 100644 index 3dbbc46eb..000000000 --- a/Integrations/Definition.js +++ /dev/null @@ -1,51 +0,0 @@ -const axios = require('axios') -let fs = require('fs') -const { promisify } = require('util') -const readFileAsync = promisify(fs.readFile) - -exports.getDefinition = async function () { - try { - if (!global.DEFINITION) { - let executionOnCloud = false - if (process.env.ON_CLOUD !== undefined) { - executionOnCloud = JSON.parse(process.env.ON_CLOUD) - } - - if (executionOnCloud) { - const strategizerResponse = await axios({ - url: process.env.GATEWAY_ENDPOINT_K8S, - method: 'post', - data: { - query: ` - query($fbSlug: String!){ - strategizer_TradingSystemByFb(fbSlug: $fbSlug){ - data - } - } - `, - variables: { - fbSlug: process.env.BOT - } - }, - headers: { - authorization: process.env.AUTHORIZATION - } - }) - - if (strategizerResponse.data.errors) { - throw new Error(strategizerResponse.data.errors[0].message) - } - - global.DEFINITION = strategizerResponse.data.data.strategizer_TradingSystemByFb.data - } else { - let fileLocation = process.env.INTER_PROCESS_FILES_PATH + '/definition.json' - let definition = readFile(fileLocation, { encoding: 'utf8' }) - global.DEFINITION = JSON.parse(definition) - } - } - - return global.DEFINITION - } catch (error) { - throw new Error('There has been an error getting the definition: ', JSON.stringify(error)) - } -} diff --git a/MultiPeriod.js b/MultiPeriod.js index 56cd5480e..d464c00dd 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -496,7 +496,7 @@ console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " " + " bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime.toISOString()); - if (global.FULL_LOG === 'true') { + if (global.WRITE_LOGS_TO_FILES === 'true') { logger.newInternalLoop(bot.codeName, bot.process); } diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 09f870f6c..968ad329c 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -300,7 +300,7 @@ console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " " + " bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime.toISOString()); - if (global.FULL_LOG === 'true') { + if (global.WRITE_LOGS_TO_FILES === 'true') { logger.newInternalLoop(bot.codeName, bot.process); } diff --git a/Project.njsproj b/Project.njsproj index 2d9bfbc32..8db0453a3 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -27,7 +27,6 @@ - Code @@ -69,9 +68,6 @@ Code - - Code - diff --git a/Root.js b/Root.js index 1e031b898..d57d59ae2 100644 --- a/Root.js +++ b/Root.js @@ -45,7 +45,7 @@ const FULL_LOG = true; - let logDisplace = "CloneExecutor" + " "; + let logDisplace = "Task Server" + " "; let UI_COMMANDS; return thisObject; @@ -106,7 +106,7 @@ } } - function start(callback) { + function start(processIndex) { /* Now we will run according to what we see at the config file. */ @@ -119,7 +119,6 @@ /* Process Loops Declarations. */ - const TRADING_BOT_MAIN_LOOP_MODULE = require('./TradingBotProcessMainLoop'); const INDICATOR_BOT_MAIN_LOOP_MODULE = require('./IndicatorBotProcessMainLoop'); const EXTRACTION_BOT_MAIN_LOOP_MODULE = require('./SensorBotProcessMainLoop'); const TRADING_ENGINE_MAIN_LOOP_MODULE = require('./TradingEngineProcessMainLoop'); @@ -128,7 +127,7 @@ let cloneToExecute = global.EXECUTION_CONFIG.cloneToExecute; - /* Now we will read the config of the bot from the path we obtained at the CloneExecutor config. */ + /* Now we will read the config of the bot from the path we obtained at the Task Server config. */ let botConfig; @@ -157,7 +156,7 @@ try { botConfig = JSON.parse(text); botConfig.repo = cloneToExecute.repo; - findProcess(); + findProcess(processIndex); } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> err = " + JSON.stringify(err)); return; @@ -170,13 +169,14 @@ } } - function findProcess() { + function findProcess(processIndex) { try { if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> Entering function. "); } botConfig.process = cloneToExecute.process; botConfig.debug = {}; + botConfig.processIndex = processIndex /* Loop Counter */ @@ -395,10 +395,6 @@ } switch (botConfig.type) { - case 'Trading': { - runTradingBot(botConfig, processConfig); - break; - } case 'Trading-Engine': { runTradingEngine(botConfig, processConfig); break; @@ -424,10 +420,6 @@ botConfig.hasTheBotJustStarted = true; switch (botConfig.type) { - case 'Trading': { - runTradingBot(botConfig, processConfig); - break; - } case 'Trading-Engine': { runTradingEngine(botConfig, processConfig); break; @@ -439,33 +431,6 @@ } } - if (processConfig.startMode.competition !== undefined) { - - if (processConfig.startMode.competition.run === "true") { - - botConfig.startMode = "Competition"; - console.log(logDisplace + "Root : [INFO] start -> findProcess -> Process found at the bot configuration file. -> Start Mode = " + botConfig.startMode); - - botConfig.competition = processConfig.startMode.competition; - - if (processConfig.startMode.competition.resumeExecution === false) { - botConfig.hasTheBotJustStarted = true; - } else { - botConfig.hasTheBotJustStarted = false; - } - - switch (botConfig.type) { - case 'Trading': { - runTradingBot(botConfig, processConfig); - break; - } - default: { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); - } - } - } - } - } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected exception. -> err.message = " + err.message); } @@ -474,7 +439,7 @@ if (processFound === false) { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Process listed at the configuration file of CloneExecutor not found at the configuration file of the bot."); + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Process listed at the configuration file of Task Server not found at the configuration file of the bot."); console.log(logDisplace + "Root : [ERROR] start -> findProcess -> cloneToExecute.process = " + cloneToExecute.process); } @@ -520,7 +485,6 @@ console.log(logDisplace + "Root : [INFO] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> botId = " + botId); console.log(logDisplace + "Root : [INFO] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); - logger.persist(); } else { @@ -534,7 +498,7 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot execution was aborted."); logger.persist(); } - callback(err, pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process) + exitProcess() } } else { @@ -543,11 +507,13 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> err = "+ err.stack); logger.persist(); + exitProcess() } } } catch (err) { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> err = "+ err.stack); + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> err = " + err.stack); + exitProcess() } } @@ -591,7 +557,6 @@ console.log(logDisplace + "Root : [INFO] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> botId = " + botId); console.log(logDisplace + "Root : [INFO] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); - logger.persist(); } else { @@ -603,7 +568,7 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); logger.persist(); } - callback(err, pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process) + exitProcess() } } else { @@ -611,191 +576,13 @@ logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> Failed to initialize the bot. "); console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> err = "+ err.stack); logger.persist(); + exitProcess() } } } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> err = "+ err.stack); - - } - } - - function runTradingBot(pBotConfig, pProcessConfig) { - - try { - - if (FULL_LOG === true) { console.log(logDisplace + "[INFO] start -> findProcess -> runTradingBot -> Entering function."); } - - if (pBotConfig.genes !== undefined) { - - /* Here, based on the gene we will create one instance for each combination of genes */ - - let valueMatrix = []; - - for (let i = 0; i < pBotConfig.genes.length; i++) { - - let gene = pBotConfig.genes[i]; - - let possibleValues = []; - - for (let j = gene.lowerLimit; j <= gene.upperLimit; j++) { - - possibleValues.push(j); - - } - - valueMatrix.push(possibleValues); - - } - - /* - Now we have all the possible genes values in a multi-dimensional matrix. - We will go thorugh each of the elements of each dimension and get each combination possible with the other dimmensions. - */ - - let combinations = []; - let combination = []; - let dimensionIndex = 0; - - calculateCombinations(dimensionIndex, combination); - - function calculateCombinations(pDimensionIndex, pCombination) { - - let dimension = valueMatrix[pDimensionIndex]; - - for (let j = 0; j < dimension.length; j++) { - - let copy = JSON.parse(JSON.stringify(pCombination)); - - let value = dimension[j]; - copy.push(value); - - if (pDimensionIndex < valueMatrix.length - 1) { - - calculateCombinations(pDimensionIndex + 1, copy); - - } else { - - combinations.push(copy); - - } - } - } - - /* At this point we have an array with all the possible combinations, we just need to create an instance for each one and set the genes according to that. */ - - for (let i = 0; i < combinations.length; i++) { - - let botConfig = JSON.parse(JSON.stringify(pBotConfig)); - let combination = combinations[i]; - let genes = {}; - let clonKey = ""; - - for (let j = 0; j < botConfig.genes.length; j++) { - - genes[botConfig.genes[j].name] = combination[j]; - clonKey = clonKey + "." + combination[j]; - - } - - let clonName = botConfig.codeName + "-" + "Clon" + clonKey; - clonName += ".1.0"; - clonName += "-" + process.env.CLONE_ID; - - botConfig.filePathRoot = botConfig.devTeam + "/" + clonName + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + botConfig.dataSetVersion; - - botConfig.instance = clonName; - botConfig.instanceIndex = i; - - setTimeout(execute, i * Math.random() * 10 * 1000); - - function execute() { - - createBotInstance(genes, combinations.length, botConfig); - - } - } - } else { - - /* If the bot does not have any genes at all */ - - let genes = {}; - let clonName = botConfig.codeName; - clonName += "-" + process.env.CLONE_ID; - clonName += ".1.0"; - - botConfig.filePathRoot = botConfig.devTeam + "/" + clonName + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + botConfig.dataSetVersion; - - pBotConfig.instance = clonName; - pBotConfig.instanceIndex = 0; - - createBotInstance(genes, 1, pBotConfig); - - } - - - function createBotInstance(pGenes, pTotalInstances, pBotConfig) { - - const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); - let logger; - - logger = DEBUG_MODULE.newDebugLog(); - logger.bot = pBotConfig; - pBotConfig.timePeriodFileStorage = global.EXECUTION_CONFIG.timePeriodFileStorage - pBotConfig.dataSet = global.EXECUTION_CONFIG.dataSet - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runTradingBot -> createBotInstance -> Entering function."); } - - let tradingBotMainLoop = TRADING_BOT_MAIN_LOOP_MODULE.newTradingBotProcessMainLoop(pBotConfig, logger); - tradingBotMainLoop.initialize(UI_COMMANDS, pProcessConfig, onInitializeReady); - - function onInitializeReady(err) { - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - tradingBotMainLoop.run(pGenes, whenRunFinishes); - - function whenRunFinishes(err) { - - pBotConfig.loopCounter = 0; - - let botId; - - botId = pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.instance + "." + pBotConfig.process; - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); - logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); - - console.log(logDisplace + "Root : [INFO] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> whenStartFinishes -> botId = " + botId); - console.log(logDisplace + "Root : [INFO] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); - logger.persist(); - - } else { - - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> whenStartFinishes -> err = "+ err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> whenStartFinishes -> Execution will be stopped. "); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> whenStartFinishes -> Bye."); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); - logger.persist(); - } - callback(err, pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process) - } - - } else { - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> err = "+ err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> Bot will not be started. "); - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingBot -> createBotInstance -> onInitializeReady -> err = "+ err.stack); - - logger.persist(); - } - } - } - } - catch (err) { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingBot -> err = "+ err.stack); + exitProcess() } } @@ -834,7 +621,6 @@ console.log(logDisplace + "Root : [INFO] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> botId = " + botId); console.log(logDisplace + "Root : [INFO] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); - logger.persist(); } else { @@ -846,7 +632,7 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); logger.persist(); } - callback(err, pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process) + exitProcess() } } else { @@ -854,22 +640,49 @@ logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> Failed to initialize the bot. "); console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> err = "+ err.stack); logger.persist(); + exitProcess() } } } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> err = "+ err.stack); - + exitProcess() } } - } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> err.message = " + err.message); - return; + exitProcess() } } + function exitProcess() { + + global.ENDED_PROCESSES_COUNTER++ + console.log("[INFO] Task Server -> exitProcess -> Process #" + global.ENDED_PROCESSES_COUNTER + " exiting."); + + if (global.ENDED_PROCESSES_COUNTER === global.USER_DEFINITION.bot.processes.length) { + /* Cleaning Before Exiting. */ + clearInterval(global.HEARTBEAT_INTERVAL_HANDLER) + + for (let i = 0; i < global.USER_DEFINITION.bot.processes.length; i++) { + let code = global.USER_DEFINITION.bot.processes[i].code + + /* Delete the event handler for each process. */ + + let key = code.devTeam + "-" + code.codeName + "-" + code.process + let event = { + reason: 'Signal Received to Terminate this Process.' + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated', event) + global.SYSTEM_EVENT_HANDLER.deleteEventHandler(key) + } + + global.SYSTEM_EVENT_HANDLER.finalize() + global.SYSTEM_EVENT_HANDLER = undefined + console.log("[INFO] Task Server -> exitProcess -> Task Server Stopped."); + } + } } } diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index a025be6d0..c98cd05d7 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -557,7 +557,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> onStop -> Stopping the Loop Gracefully. See you next time!"); } - if(global.FULL_LOG === 'true'){ + if(global.WRITE_LOGS_TO_FILES === 'true'){ logger.persist(); } @@ -581,15 +581,15 @@ if (bot.runAtFixedInterval === true) { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Fixed Interval Normal exit point reached."); } checkLoopHealthHandle = setTimeout(checkLoopHealth, bot.fixedInterval * 5, bot.loopCounter); - if(global.FULL_LOG === 'true'){ + if(global.WRITE_LOGS_TO_FILES === 'true'){ logger.persist(); } return; } else { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.normalWaitTime / 1000) + " seconds."); } - checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.normalWaitTime * 5, bot.loopCounter); + checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.deadWaitTime * 5, bot.loopCounter); nextLoopTimeoutHandle = setTimeout(loop, processConfig.normalWaitTime); - if(global.FULL_LOG === 'true'){ + if(global.WRITE_LOGS_TO_FILES === 'true'){ logger.persist(); } } @@ -643,7 +643,7 @@ let now = new Date().valueOf(); - if (now - bot.loopStartTime > processConfig.normalWaitTime) { + if (now - bot.loopStartTime > processConfig.deadWaitTime) { logger.write(MODULE_NAME, "[ERROR] run -> loop -> checkLoopHealth -> Dead loop found -> pLastLoop = " + pLastLoop); console.log((new Date().toISOString() + " [ERROR] run -> loop -> checkLoopHealth -> " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " Loop # " + pad(Number(bot.loopCounter), 5) + " found dead. Resurrecting it now.")); @@ -672,7 +672,7 @@ if (process.env.STOP_GRACEFULLY !== undefined) stop = JSON.parse(process.env.STOP_GRACEFULLY) - if (!stop && global.SHALL_BOT_STOP === false) { + if (!stop) { continueCallBack(); } else { stopCallBack(); @@ -700,7 +700,7 @@ } function hearBeat() { - let key = global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].name + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].type + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].id + let key = global.USER_DEFINITION.bot.processes[bot.processIndex].name + '-' + global.USER_DEFINITION.bot.processes[bot.processIndex].type + '-' + global.USER_DEFINITION.bot.processes[bot.processIndex].id let event = { seconds: (new Date()).getSeconds() diff --git a/TradingBotProcessMainLoop.js b/TradingBotProcessMainLoop.js deleted file mode 100644 index 6db2d7c29..000000000 --- a/TradingBotProcessMainLoop.js +++ /dev/null @@ -1,1036 +0,0 @@ -exports.newTradingBotProcessMainLoop = function newTradingBotProcessMainLoop(bot, parentLogger) { - - const ROOT_DIR = './'; - - const MODULE_NAME = "Trading Bot Process Main Loop"; - const FULL_LOG = true; - - let USER_BOT_MODULE; - let COMMONS_MODULE; - - const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); - const FILE_STORAGE = require('./Integrations/FileStorage.js'); - let fileStorage = FILE_STORAGE.newFileStorage(parentLogger); - let logger; // We need this here in order for the loopHealth function to work and be able to rescue the loop when it gets in trouble. - - let nextLoopTimeoutHandle; - let checkLoopHealthHandle; - - let thisObject = { - initialize: initialize, - run: run - }; - - bot["botCache"] = new Map(); - let processConfig; - let UI_COMMANDS; - - return thisObject; - - function initialize(pUI_COMMANDS, pProcessConfig, callBackFunction) { - - /* This function is exactly the same in the 3 modules representing the 2 different bot types loops. */ - - try { - if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } - - UI_COMMANDS = pUI_COMMANDS; - processConfig = pProcessConfig; - - let filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo + "/" + pProcessConfig.name - filePath += "/User.Bot.js" - - fileStorage.getTextFile(global.DEV_TEAM, filePath, onBotDownloaded); - - function onBotDownloaded(err, text) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> err.message = " + err.message); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - USER_BOT_MODULE = {} - USER_BOT_MODULE.newUserBot = eval(text); // TODO This needs to be changed function - - filePath = global.DEV_TEAM + "/" + "bots" + "/" + bot.repo; - filePath += "/Commons.js" - - fileStorage.getTextFile(global.DEV_TEAM, filePath, onCommonsDownloaded); - - function onCommonsDownloaded(err, text) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - parentLogger.write(MODULE_NAME, "[WARN] initialize -> onBotDownloaded -> onCommonsDownloaded -> Commons not found: " + err.code || err.message); - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } - COMMONS_MODULE = {} - COMMONS_MODULE.newCommons = eval(text); // TODO This needs to be changed function - - callBackFunction(global.DEFAULT_OK_RESPONSE); - } - } - } catch (err) { - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> err = "+ err.stack); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } - - function run(pGenes, callBackFunction) { - - let context; - - try { - if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] run -> Entering function."); } - - bot.enableCheckLoopHealth = true; - - let fixedTimeLoopIntervalHandle; - - if (bot.runAtFixedInterval === true) { - - fixedTimeLoopIntervalHandle = setInterval(loop, bot.fixedInterval); - - } else { - - loop(); - - } - - function loop() { - - try { - - function pad(str, max) { - str = str.toString(); - return str.length < max ? pad(" " + str, max) : str; - } - - /* For each loop we want to create a new log file. */ - - logger = DEBUG_MODULE.newDebugLog(); - logger.bot = bot; - logger.initialize(); - - bot.loopCounter++; - bot.loopStartTime = new Date().valueOf(); - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Entering function."); } - - /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ - - const UTILITIES = require(ROOT_DIR + 'CloudUtilities'); - const EXCHANGE_API = require('@superalgos/exchange-gateway'); - const CONTEXT = require(ROOT_DIR + 'Context'); - const ASSISTANT = require(ROOT_DIR + 'Assistant'); - const STATUS_REPORT = require(ROOT_DIR + 'StatusReport'); - const DATA_SET = require(ROOT_DIR + 'DataSet'); - const STATUS_DEPENDENCIES = require(ROOT_DIR + 'StatusDependencies'); - const DATA_DEPENDENCIES = require(ROOT_DIR + 'DataDependencies'); - - /* Waitime Variable */ - - let nextWaitTime; - - /* We define the datetime for the process that we are running now. This will be the official processing time for both the infraestructure and the bot. */ - - switch (bot.startMode) { - case 'Live': { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Live Mode detected."); } - - if(process.env.WAIT_TIME !== undefined) - processConfig.normalWaitTime = process.env.WAIT_TIME - - // This will be considered the process date and time, so as to have it consistenly all over the execution. - - let localDate = new Date(); - - bot.processDatetime = new Date(Date.UTC( - localDate.getUTCFullYear(), - localDate.getUTCMonth(), - localDate.getUTCDate(), - localDate.getUTCHours(), - localDate.getUTCMinutes(), - localDate.getUTCSeconds(), - localDate.getUTCMilliseconds())); - - if (UI_COMMANDS.eventHandler !== undefined) { - - UI_COMMANDS.eventHandler.raiseEvent('Bot Execution Changed Datetime', bot.processDatetime); - - } - - break; - } - case 'Backtest': { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Backtesting Mode detected."); } - - if(process.env.WAIT_TIME !== undefined) { - processConfig.normalWaitTime = JSON.parse(process.env.WAIT_TIME) - } - - if (bot.hasTheBotJustStarted === true) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Setting initial datetime."); } - bot.processDatetime = new Date(bot.backtest.beginDatetime); // Set the starting time as the configured beginDatetime. - } else { - bot.processDatetime = new Date(bot.processDatetime.valueOf() + 60000); // We advance time here. - let endDatetime = new Date(bot.backtest.endDatetime); - - if (bot.processDatetime.valueOf() > endDatetime.valueOf()) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> End of Backtesting Period reached. Exiting Bot Process Loop."); } - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } - } - break; - } - case 'Competition': { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Competition Mode detected."); } - - if(process.env.WAIT_TIME !== undefined) { - processConfig.normalWaitTime = process.env.WAIT_TIME - } - - let localDate = new Date(); - - bot.processDatetime = new Date(Date.UTC( - localDate.getUTCFullYear(), - localDate.getUTCMonth(), - localDate.getUTCDate(), - localDate.getUTCHours(), - localDate.getUTCMinutes(), - localDate.getUTCSeconds(), - localDate.getUTCMilliseconds())); - - if (UI_COMMANDS.eventHandler !== undefined) { - - UI_COMMANDS.eventHandler.raiseEvent('Bot Execution Changed Datetime', bot.processDatetime); - - } - - let beginDatetime = new Date(bot.competition.beginDatetime); - - if (bot.processDatetime.valueOf() < beginDatetime.valueOf()) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Competition not started yet. Wainting for the competition to start."); } - - nextWaitTime = 'Normal'; - loopControl(nextWaitTime); - return; - } - - let endDatetime = new Date(bot.competition.endDatetime); - - if (bot.processDatetime.valueOf() > endDatetime.valueOf()) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> End of Competition Period reached. Exiting Bot Process Loop."); } - - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } - - break; - } - default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> Unexpected bot.startMode."); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> bot.startMode = " + bot.startMode); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> bot.processDatetime = " + bot.processDatetime.toISOString()); } - - - /* High level log entry */ - - console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.devTeam, 20) + " " + pad(bot.process, 30) - + " " + bot.startMode + " Entered into Main Loop # " + pad(Number(bot.loopCounter), 8) + " bot.processDatetime = " + bot.processDatetime.toISOString()); - - /* We will prepare first the infraestructure needed for the bot to run. There are 3 modules we need to sucessfullly initialize first. */ - - let exchangeAPI; - let assistant; - let userBot; - let statusDependencies; - let dataDependencies; - - initializeStatusDependencies(); - - function initializeStatusDependencies() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeStatusDependencies -> Entering function."); } - - statusDependencies = STATUS_DEPENDENCIES.newStatusDependencies(bot, logger, STATUS_REPORT, UTILITIES); - - statusDependencies.initialize(processConfig.statusDependencies, undefined, undefined, onInizialized); - - function onInizialized(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeStatusDependencies -> onInizialized -> Entering function."); } - - switch (err.result) { - case global.DEFAULT_OK_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeStatusDependencies -> onInizialized -> Execution finished well."); } - initializeDataDependencies(); - return; - } - case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); - nextWaitTime = 'Retry'; - loopControl(nextWaitTime); - return; - } - case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Operation Failed. Aborting the process."); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - return; - } - default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Unhandled err.result received. -> err.message = " + err.message); - - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - function initializeDataDependencies() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> Entering function."); } - - dataDependencies = DATA_DEPENDENCIES.newDataDependencies(bot, logger, DATA_SET); - - dataDependencies.initialize(processConfig.dataDependencies, onInizialized); - - function onInizialized(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> onInizialized -> Entering function."); } - - switch (err.result) { - case global.DEFAULT_OK_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> onInizialized -> Execution finished well."); } - initializeContext(); - return; - } - case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); - nextWaitTime = 'Retry'; - loopControl(nextWaitTime); - return; - } - case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Operation Failed. Aborting the process."); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - return; - } - default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Unhandled err.result received. -> err.message = " + err.message); - - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - function initializeContext() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeContext -> Entering function."); } - - context = CONTEXT.newContext(bot, logger, UTILITIES); - context.initialize(statusDependencies, onInizialized); - - function onInizialized(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeContext -> onInizialized -> Entering function."); } - - switch (err.result) { - case global.DEFAULT_OK_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeContext -> onInizialized -> Execution finished well."); } - initializeExchangeAPI(); - return; - } - case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeContext -> onInizialized -> Retry Later. Requesting Execution Retry."); - nextWaitTime = 'Retry'; - loopControl(nextWaitTime); - return; - } - case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeContext -> onInizialized -> Operation Failed. Aborting the process."); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - return; - } - default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeContext -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeContext -> onInizialized -> Unhandled err.result received. -> err.message = " + err.message); - - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeContext -> onInizialized -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeContext -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - function initializeExchangeAPI() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeExchangeAPI -> Entering function."); } - - exchangeAPI = EXCHANGE_API.newExchangeAPI(logger, global.EXCHANGE_NAME); - - exchangeAPI.initialize(onInizialized); - - function onInizialized(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeContext -> onInizialized -> onInizialized -> Entering function."); } - - switch (err.result) { - case global.DEFAULT_OK_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeExchangeAPI -> onInizialized -> Execution finished well."); } - initializeAssistant(); - return; - } - case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeExchangeAPI -> onInizialized -> Retry Later. Requesting Execution Retry."); - nextWaitTime = 'Retry'; - loopControl(nextWaitTime); - return; - } - case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeExchangeAPI -> onInizialized -> Operation Failed. Aborting the process."); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - return; - } - default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeExchangeAPI -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeExchangeAPI -> onInizialized -> Unhandled err.result received. -> err.message = " + err.message); - - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeContext -> onInizialized -> onInizialized -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeExchangeAPI -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - function initializeAssistant() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeAssistant -> Entering function."); } - - assistant = ASSISTANT.newAssistant(bot, logger, UTILITIES); - assistant.initialize(context, exchangeAPI, dataDependencies, onInizialized); - - function onInizialized(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeAssistant -> onInizialized -> Entering function."); } - - switch (err.result) { - case global.DEFAULT_OK_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeAssistant -> onInizialized -> Execution finished well."); } - initializeUserBot(); - return; - } - case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeAssistant -> onInizialized -> Retry Later. Requesting Execution Retry."); - nextWaitTime = 'Retry'; - loopControl(nextWaitTime); - return; - } - case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeAssistant -> onInizialized -> Operation Failed. Aborting the process."); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - return; - } - default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeAssistant -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeAssistant -> onInizialized -> Unhandled err.result received. -> err.message = " + err.message); - - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeAssistant -> onInizialized -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeAssistant -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - function initializeUserBot() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeUserBot -> Entering function."); } - - usertBot = USER_BOT_MODULE.newUserBot(bot, logger, COMMONS_MODULE); - - usertBot.initialize(assistant, pGenes, onInizialized); - - function onInizialized(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeUserBot -> onInizialized -> Entering function."); } - - switch (err.result) { - case global.DEFAULT_OK_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeUserBot -> onInizialized -> Execution finished well."); } - startUserBot(); - return; - } - case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized -> Retry Later. Requesting Execution Retry."); - nextWaitTime = 'Retry'; - loopControl(nextWaitTime); - return; - } - case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized -> Operation Failed. Aborting the process."); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - return; - } - default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized -> Unhandled err.result received. -> err.message = " + err.message); - - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - function startUserBot() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startUserBot -> Entering function."); } - - usertBot.start(onFinished); - - function onFinished(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startUserBot -> onFinished -> Entering function."); } - - switch (err.result) { - case global.DEFAULT_OK_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> startUserBot -> onFinished -> Execution finished well."); } - saveContext(); - return; - } - case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished -> Retry Later. Requesting Execution Retry."); - nextWaitTime = 'Retry'; - loopControl(nextWaitTime); - return; - } - case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished -> Operation Failed. Aborting the process."); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - return; - } - default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished -> Unhandled err.result received. -> err.result = " + err.result); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished -> Unhandled err.result received. -> err.message = " + err.message); - - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - function saveContext() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> saveContext -> Entering function."); } - - context.saveThemAll(onFinished); - - function onFinished(err) { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> saveContext -> onFinished -> Entering function."); } - - switch (err.result) { - case global.DEFAULT_OK_RESPONSE.result: { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> saveContext -> onFinished -> Execution finished well."); } - nextWaitTime = 'Normal'; - updateClonesModule(); - return; - } - case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> saveContext -> onFinished -> Can not retry at this point."); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> saveContext -> onFinished -> Operation Failed. Aborting the process."); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - return; - } - default: { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> saveContext -> onFinished -> Unhandled err.result received. -> err.result = " + err.result); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> saveContext -> onFinished -> Unhandled err.result received. -> err.message = " + err.message); - - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> saveContext -> onFinished -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> saveContext -> err = "+ err.stack); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - async function updateClonesModule() { - - try { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> updateClonesModule -> Entering function."); } - - nextWaitTime = 'Normal'; - loopControl(nextWaitTime); - - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> updateClonesModule -> err = "+ err.stack); - logger.write(MODULE_NAME, "[ERROR] run -> loop -> updateClonesModule -> Execution will continue anyways. "); - nextWaitTime = 'Normal'; - loopControl(nextWaitTime); - } - } - - function loopControl(nextWaitTime) { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Entering function."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> nextWaitTime = " + nextWaitTime); } - - /* Here we check if we must stop the loop gracefully. */ - - shallWeStop(onStop, onContinue); - - function onStop() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> onStop -> Entering function."); } - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> onStop -> Stopping the Loop Gracefully. See you next time!"); } - - if (global.FULL_LOG === 'true' || bot.startMode === 'Competition') { - logger.persist(); - } - - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - - } - - function onContinue() { - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> onContinue -> Entering function."); } - - /* Trading bots are going to be executed after a configured period of time after the last execution ended. This is to avoid overlapping executions. */ - - switch (nextWaitTime) { - case 'Normal': { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Normal -> Restarting Loop in " + (processConfig.normalWaitTime / 1000) + " seconds."); } - checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.normalWaitTime * 5, bot.loopCounter); - nextLoopTimeoutHandle = setTimeout(loop, processConfig.normalWaitTime); - if (global.FULL_LOG === 'true' || bot.startMode === 'Competition') { - logger.persist(); - } - } - break; - case 'Retry': { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Retry -> Restarting Loop in " + (processConfig.retryWaitTime / 1000) + " seconds."); } - checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.retryWaitTime * 5, bot.loopCounter); - nextLoopTimeoutHandle = setTimeout(loop, processConfig.retryWaitTime); - logger.persist(); - } - break; - case 'Sleep': { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Sleep -> Restarting Loop in " + (processConfig.sleepWaitTime / 60000) + " minutes."); } - nextLoopTimeoutHandle = setTimeout(loop, processConfig.sleepWaitTime); - logger.persist(); - } - break; - case 'Coma': { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Coma -> Restarting Loop in " + (processConfig.comaWaitTime / 3600000) + " hours."); } - nextLoopTimeoutHandle = setTimeout(loop, processConfig.comaWaitTime); - logger.persist(); - } - break; - } - } - } - - function checkLoopHealth(pLastLoop) { - - if (bot.startMode === 'Backtest') { return; } - - if (bot.enableCheckLoopHealth === false) { - - logger.write(MODULE_NAME, "[WARN] run -> loop -> checkLoopHealth -> bot.enableCheckLoopHealth = " + bot.enableCheckLoopHealth); - - return; - } // This gets disabled anytime the Main Loop is shut down by any condition. - - if (bot.loopCounter <= pLastLoop + 1) { // This means that the next loop started but also stopped executing abruptally. - - let now = new Date().valueOf(); - - if (now - bot.loopStartTime > processConfig.normalWaitTime) { - - logger.write(MODULE_NAME, "[ERROR] run -> loop -> checkLoopHealth -> Dead loop found -> pLastLoop = " + pLastLoop); - console.log((new Date().toISOString() + " [ERROR] run -> loop -> checkLoopHealth -> " + pad(bot.codeName, 20) + " " + pad(bot.instance, 20) + " " + pad(bot.process, 30) + " Loop # " + pad(Number(bot.loopCounter), 5) + " found dead. Resurrecting it now.")); - - logger.persist(); // We persist the logs of the failed execution. - clearTimeout(nextLoopTimeoutHandle); // We cancel the timeout in case the original loop was still running and schedulled to reexecute. - loop(); // We restart the loop so that the processing can continue. - - } else { - - console.log((new Date().toISOString() + " [WARN] run -> loop -> checkLoopHealth -> " + pad(bot.codeName, 20) + " " + pad(bot.instance, 20) + " " + pad(bot.process, 30) + " Loop # " + pad(Number(bot.loopCounter), 5) + " found delayed but still alive. No action taken.")); - - } - } - } - - function shallWeStop(stopCallBack, continueCallBack) { - - try { - - /* IMPORTANT: This function is exactly the same on the 3 modules. */ - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> shallWeStop -> Entering function. "); } - - let stop = false - if (process.env.STOP_GRACEFULLY !== undefined) - stop = JSON.parse(process.env.STOP_GRACEFULLY) - - if (!stop && global.SHALL_BOT_STOP === false) { - continueCallBack(); - } else { - stopCallBack(); - } - } catch (err) { - logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> err.message = " + err.message); - logger.persist(); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - } - - } catch (err) { - parentLogger.write(MODULE_NAME, "[ERROR] run -> loop -> err = " + err.stack); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(err); - } - } - - } - - catch (err) { - parentLogger.write(MODULE_NAME, "[ERROR] run -> err = "+ err.stack); - clearInterval(fixedTimeLoopIntervalHandle); - clearTimeout(nextLoopTimeoutHandle); - clearTimeout(checkLoopHealthHandle); - bot.enableCheckLoopHealth = false; - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - } - } -}; diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index a8df4efef..50021a60c 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -887,7 +887,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> onStop -> Stopping the Loop Gracefully. See you next time!"); } - if (global.FULL_LOG === 'true') { + if (global.WRITE_LOGS_TO_FILES === 'true') { logger.persist(); } @@ -907,7 +907,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.normalWaitTime / 1000) + " seconds."); } checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.normalWaitTime * 5, bot.loopCounter); nextLoopTimeoutHandle = setTimeout(loop, processConfig.normalWaitTime); - if (global.FULL_LOG === 'true') { + if (global.WRITE_LOGS_TO_FILES === 'true') { logger.persist(); } } @@ -948,7 +948,7 @@ let now = new Date().valueOf(); - if (now - bot.loopStartTime > processConfig.normalWaitTime) { + if (now - bot.loopStartTime > processConfig.deadWaitTime) { logger.write(MODULE_NAME, "[ERROR] run -> loop -> checkLoopHealth -> Dead loop found -> pLastLoop = " + pLastLoop); console.log((new Date().toISOString() + " [ERROR] run -> loop -> checkLoopHealth -> " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " Loop # " + pad(Number(bot.loopCounter), 5) + " found dead. Resurrecting it now.")); @@ -977,7 +977,7 @@ if (process.env.STOP_GRACEFULLY !== undefined) stop = JSON.parse(process.env.STOP_GRACEFULLY) - if (!stop && global.SHALL_BOT_STOP === false) { + if (!stop) { continueCallBack(); } else { stopCallBack(); @@ -1005,7 +1005,7 @@ } function hearBeat() { - let key = global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].name + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].type + '-' + global.USER_DEFINITION.bot.processes[global.sequenceStep - 1].id + let key = global.USER_DEFINITION.bot.processes[bot.processIndex].name + '-' + global.USER_DEFINITION.bot.processes[bot.processIndex].type + '-' + global.USER_DEFINITION.bot.processes[bot.processIndex].id let event = { seconds: (new Date()).getSeconds() diff --git a/server.js b/server.js index 3e7984bc6..65deed103 100644 --- a/server.js +++ b/server.js @@ -66,7 +66,7 @@ if (global.USER_DEFINITION !== undefined) { } else { // I use this section to debug in standalone mode. - let argument = '{"type":"Task","name":"Extract Trades from Poloniex","bot":{"type":"Sensor","name":"Charly","processes":[{"type":"Process","name":"Live-Trades","code":{"devTeam":"AAMasters","bot":"AACharly","mode":"noTime","resumeExecution":true,"type":"Sensor","exchangeName":"Poloniex","process":"Live-Trades"}},{"type":"Process","name":"Hole-Fixing","code":{"devTeam":"AAMasters","bot":"AACharly","mode":"oneMonth","resumeExecution":true,"type":"Sensor","exchangeName":"Poloniex","process":"Hole-Fixing","startYear":2019,"endYear":2019,"month":9}}]}}' + let argument = '{"type":"Task","name":"Brings Trades Records from the Exchange","bot":{"type":"Sensor","name":"Charly","processes":[{"type":"Process","name":"Live Trades","code":{"devTeam":"AAMasters","bot":"AACharly","mode":"noTime","resumeExecution":true,"type":"Sensor","exchangeName":"Poloniex","process":"Live-Trades"},"id":"3aae75f5-67ae-4ae5-9586-ec5640f79b18"}]},"id":"54444b07-eee2-4fa8-9183-8fea3b5ff9de"}' try { global.USER_DEFINITION = JSON.parse(argument) } catch (err) { @@ -75,11 +75,35 @@ else { // I use this section to debug in standalone mode. } +require('dotenv').config(); + +global.DEFINITION = require(process.env.INTER_PROCESS_FILES_PATH + '/Definition'); +global.WRITE_LOGS_TO_FILES = process.env.WRITE_LOGS_TO_FILES + +/* Default parameters can be changed by the execution configuration */ +global.EXCHANGE_NAME = process.env.EXCHANGE_NAME +global.MARKET = { assetA: 'USDT', assetB: 'BTC' } +global.CLONE_EXECUTOR = { codeName: 'AACloud', version: '1.1' } +global.ENDED_PROCESSES_COUNTER = 0 + +/* Here we listen for the message to stop this Task / Process comming from the Task Manager, which is the paret of this node js process. */ +process.on('message', message => { + if (message === 'Stop this Task') { + + process.env.STOP_GRACEFULLY = true; + + } +}); + +let notFirstSequence = false; + +/* Setting up the global Event Handler */ + const EVENT_HANDLER_MODULE = require('../Libraries/SystemEventsClient/SystemEventHandler.js'); global.SYSTEM_EVENT_HANDLER = EVENT_HANDLER_MODULE.newSystemEventHandler() - global.SYSTEM_EVENT_HANDLER.initialize('Task Server', bootLoader) + function bootLoader() { //global.SYSTEM_EVENT_HANDLER.createEventHandler(global.USER_DEFINITION.name) @@ -89,7 +113,7 @@ function bootLoader() { function tryToListenToCockpit() { - global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Simulation', 'Simulation Started', undefined, 'Clone Executor Boot Loader', onResponse, startSequence) + global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Simulation', 'Simulation Started', undefined, 'Task Server Boot Loader', onResponse, startSequence) function onResponse(message) { if (message.result !== global.DEFAULT_OK_RESPONSE.result) { @@ -112,7 +136,7 @@ function bootLoader() { } /* Heartbeat sent to the UI */ - heartBeatInterval = setInterval(hearBeat, 1000) + global.HEARTBEAT_INTERVAL_HANDLER = setInterval(hearBeat, 1000) function hearBeat() { @@ -127,146 +151,58 @@ function bootLoader() { startSequence() } - /* Old Run.js code follows... */ -require('dotenv').config(); +function startSequence() { -global.DEFINITION = require(process.env.INTER_PROCESS_FILES_PATH + '/Definition'); -const definition = global.DEFINITION -const path = require('path'); + for (let processIndex = 0; processIndex < global.USER_DEFINITION.bot.processes.length; processIndex++) { + let execution = sequenceList[processIndex]; -global.SHALL_BOT_STOP = false + process.env.STOP_GRACEFULLY = false; + execution.devTeam ? process.env.DEV_TEAM = execution.devTeam : undefined; + execution.bot ? process.env.BOT = execution.bot : undefined; + execution.mode ? process.env.START_MODE = execution.mode : undefined; + execution.resumeExecution = true; + execution.type ? process.env.TYPE = execution.type : undefined; + execution.process ? process.env.PROCESS = execution.process : undefined; + execution.startYear ? process.env.MIN_YEAR = execution.startYear : undefined; + execution.endYear ? process.env.MAX_YEAR = execution.endYear : undefined; + execution.month ? process.env.MONTH = execution.month : undefined; + execution.beginDatetime ? process.env.BEGIN_DATE_TIME = execution.beginDatetime : undefined; + execution.endDatetime ? process.env.END_DATE_TIME = execution.endDatetime : undefined; + execution.dataSet ? process.env.DATA_SET = execution.dataSet : undefined; + execution.timePeriod ? process.env.TIME_PERIOD = execution.timePeriod : undefined; + execution.baseAsset ? process.env.BASE_ASSET = execution.baseAsset : undefined; + execution.balanceAssetA ? process.env.INITIAL_BALANCE_ASSET_A = execution.balanceAssetA : undefined; + execution.balanceAssetB ? process.env.INITIAL_BALANCE_ASSET_B = execution.balanceAssetB : undefined; + execution.type === 'Trading' ? process.env.CLONE_ID = 1 : undefined; -global.FULL_LOG = process.env.FULL_LOG + execution.exchangeName ? global.EXCHANGE_NAME = execution.exchangeName : undefined; -/* Default parameters can be changed by the execution configuration */ -global.EXCHANGE_NAME = process.env.EXCHANGE_NAME -global.MARKET = { assetA: 'USDT', assetB: 'BTC' } -global.CLONE_EXECUTOR = { codeName: 'AACloud', version: '1.1' } + if (global.DEFINITION) { + if (global.DEFINITION.personalData) { + if (global.DEFINITION.personalData.exchangeAccounts) { + if (global.DEFINITION.personalData.exchangeAccounts.length > 0) { + let exchangeAccount = global.DEFINITION.personalData.exchangeAccounts[0] + if (exchangeAccount.keys) { + if (exchangeAccount.keys.length > 0) { + let key = exchangeAccount.keys[0] + process.env.KEY = key.name + process.env.SECRET = key.code - - - -let isRunSequence = false; -global.sequenceStep = 0; -let processedSteps = new Map(); -let notFirstSequence = false; -let runClonExecutor = true; - -if (process.env.RUN_SEQUENCE !== undefined) { - isRunSequence = JSON.parse(process.env.RUN_SEQUENCE) -} - -function startSequence () { - if (isRunSequence) { - process.on('message', message => { - if (message === 'Stop this Task') { - runClonExecutor = false; - - /* Cleaning Before Exiting. */ - clearInterval(heartBeatInterval) - - for (let i = 0; i < global.USER_DEFINITION.bot.processes.length; i++) { - let code = global.USER_DEFINITION.bot.processes[i].code - - /* Delete the event handler for each process. */ - - let key = code.devTeam + "-" + code.codeName + "-" + code.process - let event = { - reason: 'Signal Received to Terminate this Process.' - } - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated', event) - global.SYSTEM_EVENT_HANDLER.deleteEventHandler(key) - } - - global.SYSTEM_EVENT_HANDLER.finalize() - global.SYSTEM_EVENT_HANDLER = undefined - console.log("[INFO] CloneExecutor -> Clone Executor Stopped."); - } - }); - sequenceExecution(sequenceStep) - } else { - readExecutionConfiguration() - } -} - - -async function sequenceExecution(currentStep) { - - let execution = sequenceList[currentStep]; - - process.env.STOP_GRACEFULLY = true; - execution.devTeam ? process.env.DEV_TEAM = execution.devTeam : undefined; - execution.bot ? process.env.BOT = execution.bot : undefined; - execution.mode ? process.env.START_MODE = execution.mode : undefined; - execution.resumeExecution = true; - execution.type ? process.env.TYPE = execution.type : undefined; - execution.process ? process.env.PROCESS = execution.process : undefined; - execution.startYear ? process.env.MIN_YEAR = execution.startYear : undefined; - execution.endYear ? process.env.MAX_YEAR = execution.endYear : undefined; - execution.month ? process.env.MONTH = execution.month : undefined; - execution.beginDatetime ? process.env.BEGIN_DATE_TIME = execution.beginDatetime : undefined; - execution.endDatetime ? process.env.END_DATE_TIME = execution.endDatetime : undefined; - execution.dataSet ? process.env.DATA_SET = execution.dataSet : undefined; - execution.timePeriod ? process.env.TIME_PERIOD = execution.timePeriod : undefined; - execution.baseAsset ? process.env.BASE_ASSET = execution.baseAsset : undefined; - execution.balanceAssetA ? process.env.INITIAL_BALANCE_ASSET_A = execution.balanceAssetA : undefined; - execution.balanceAssetB ? process.env.INITIAL_BALANCE_ASSET_B = execution.balanceAssetB : undefined; - execution.type === 'Trading' ? process.env.CLONE_ID = 1 : undefined; - - execution.exchangeName ? global.EXCHANGE_NAME = execution.exchangeName : undefined; - - if (definition) { - if (definition.personalData) { - if (definition.personalData.exchangeAccounts) { - if (definition.personalData.exchangeAccounts.length > 0) { - let exchangeAccount = definition.personalData.exchangeAccounts[0] - if (exchangeAccount.keys) { - if (exchangeAccount.keys.length > 0) { - let key = exchangeAccount.keys[0] - - process.env.KEY = key.name - process.env.SECRET = key.code - + } } } } } } - } - let stepKey = execution.devTeam + '.' + execution.bot + '.' + execution.process; - if (processedSteps.has(stepKey)) { - processedSteps.set(stepKey, processedSteps.get(stepKey) + 1); - } else { - processedSteps.set(stepKey, 0); + readExecutionConfiguration(execution, processIndex); } - - readExecutionConfiguration(execution); - sequenceStep++; } -function onExecutionFinish(result, finishStepKey) { - if (global.sequenceStep < sequenceList.length && runClonExecutor) { - sequenceExecution(sequenceStep); - } else { - setTimeout(function () { - if (runClonExecutor) { - console.log("[INFO] onExecutionFinish -> New round for sequence execution started."); - - global.sequenceStep = 0; - processedSteps = new Map(); - notFirstSequence = true; - sequenceExecution(sequenceStep); - } - }, process.env.EXECUTION_LOOP_DELAY); - } - -} - -async function readExecutionConfiguration(execution) { +async function readExecutionConfiguration(execution, processIndex) { try { console.log("[INFO] Task Server -> server -> readExecutionConfiguration -> Entering function. "); @@ -291,31 +227,31 @@ async function readExecutionConfiguration(execution) { process.env.START_MODE = "live" // If we have keys, then we are in live mode. } - if (definition !== undefined) { - if (definition.simulationParams !== undefined) { - if (definition.simulationParams.beginDatetime !== undefined) { - initialDatetime = new Date(definition.simulationParams.beginDatetime) /* The first override occurs here, with the simulation parameters */ + if (global.DEFINITION !== undefined) { + if (global.DEFINITION.simulationParams !== undefined) { + if (global.DEFINITION.simulationParams.beginDatetime !== undefined) { + initialDatetime = new Date(global.DEFINITION.simulationParams.beginDatetime) /* The first override occurs here, with the simulation parameters */ } /* Here we only look for one timePeriod, in the future we will be able to process the whole array, but not for now. */ - if (definition.simulationParams.timePeriodDailyArray !== undefined) { - if (definition.simulationParams.timePeriodDailyArray.length === 1) { - timePeriodFilter = definition.simulationParams.timePeriodDailyArray[0] + if (global.DEFINITION.simulationParams.timePeriodDailyArray !== undefined) { + if (global.DEFINITION.simulationParams.timePeriodDailyArray.length === 1) { + timePeriodFilter = global.DEFINITION.simulationParams.timePeriodDailyArray[0] botProcess = "Multi-Period" } } - if (definition.simulationParams.timePeriodMarketArray !== undefined) { - if (definition.simulationParams.timePeriodMarketArray.length === 1) { - timePeriodFilter = definition.simulationParams.timePeriodMarketArray[0] + if (global.DEFINITION.simulationParams.timePeriodMarketArray !== undefined) { + if (global.DEFINITION.simulationParams.timePeriodMarketArray.length === 1) { + timePeriodFilter = global.DEFINITION.simulationParams.timePeriodMarketArray[0] botProcess = "Multi-Period" } } } - if (definition.tradingSystem !== undefined) { - if (definition.tradingSystem.parameters !== undefined) { - if (definition.tradingSystem.parameters.timeRange !== undefined) { - if (definition.tradingSystem.parameters.timeRange.code !== undefined) { + if (global.DEFINITION.tradingSystem !== undefined) { + if (global.DEFINITION.tradingSystem.parameters !== undefined) { + if (global.DEFINITION.tradingSystem.parameters.timeRange !== undefined) { + if (global.DEFINITION.tradingSystem.parameters.timeRange.code !== undefined) { try { - let code = JSON.parse(definition.tradingSystem.parameters.timeRange.code) + let code = JSON.parse(global.DEFINITION.tradingSystem.parameters.timeRange.code) if (code.initialDatetime !== undefined) { initialDatetime = code.initialDatetime /* The second override occurs here, with the date explicitelly defined by the user */ } @@ -323,14 +259,14 @@ async function readExecutionConfiguration(execution) { finalDatetime = code.finalDatetime } } catch (err) { - definition.tradingSystem.parameters.timeRange.error = err.message + global.DEFINITION.tradingSystem.parameters.timeRange.error = err.message } } } } } - /* Get the initial balance from the definition */ - let tradingSystem = definition.tradingSystem + /* Get the initial balance from the global.DEFINITION */ + let tradingSystem = global.DEFINITION.tradingSystem if (tradingSystem) { if (tradingSystem.parameters !== undefined) { @@ -359,7 +295,7 @@ async function readExecutionConfiguration(execution) { } } } catch (err) { - definition.tradingSystem.parameters.baseAsset.error = err.message + global.DEFINITION.tradingSystem.parameters.baseAsset.error = err.message process.env.INITIAL_BALANCE_ASSET_A = 0 // default process.env.INITIAL_BALANCE_ASSET_B = 0.001 // default @@ -381,7 +317,7 @@ async function readExecutionConfiguration(execution) { let live = { run: 'false', resumeExecution: execution.resumeExecution, - beginDatetime: new Date(definition.simulationParams.timestamp).toISOString(), + beginDatetime: new Date(global.DEFINITION.simulationParams.timestamp).toISOString(), endDatetime: finalDatetime } @@ -471,7 +407,7 @@ async function readExecutionConfiguration(execution) { version: '1.1' } - startRoot(); + startRoot(processIndex); } catch (err) { @@ -513,7 +449,7 @@ function getTimePeriod(timePeriod) { } } -function startRoot() { +function startRoot(processIndex) { console.log('[INFO] Task Server -> server -> startRoot -> Entering function. ') const ROOT_DIR = './' @@ -533,7 +469,7 @@ function startRoot() { function onInitialized() { console.log('[INFO] Task Server -> server -> startRoot -> onInitialized -> Entering function. ') - root.start(onExecutionFinish) + root.start(processIndex) } } From 5128358bacf3dfed04891c7f575c85b1f16b09e6 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 11 Sep 2019 09:34:56 +0200 Subject: [PATCH 24/32] Processes Running from Task Manager at the UI working well --- IndicatorBotProcessMainLoop.js | 8 +- SensorBotProcessMainLoop.js | 6 +- TradingEngineProcessMainLoop.js | 8 +- sequence - all processes.json | 145 -------------------------------- sequence.json | 142 ------------------------------- 5 files changed, 17 insertions(+), 292 deletions(-) delete mode 100644 sequence - all processes.json delete mode 100644 sequence.json diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 6891951ed..38111dd02 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -826,7 +826,7 @@ switch (nextWaitTime) { case 'Normal': { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.normalWaitTime / 1000) + " seconds."); } - checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.normalWaitTime * 5, bot.loopCounter); + checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.deadWaitTime * 5, bot.loopCounter); nextLoopTimeoutHandle = setTimeout(loop, processConfig.normalWaitTime); if(global.WRITE_LOGS_TO_FILES === 'true'){ logger.persist(); @@ -858,7 +858,11 @@ function checkLoopHealth(pLastLoop) { - if (bot.enableCheckLoopHealth === false) { + let stop = false + if (process.env.STOP_GRACEFULLY !== undefined) + stop = JSON.parse(process.env.STOP_GRACEFULLY) + + if (bot.enableCheckLoopHealth === false || stop === true) { logger.write(MODULE_NAME, "[WARN] run -> loop -> checkLoopHealth -> bot.enableCheckLoopHealth = " + bot.enableCheckLoopHealth); diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index c98cd05d7..d2e992e46 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -632,7 +632,11 @@ function checkLoopHealth(pLastLoop) { - if (bot.enableCheckLoopHealth === false) { + let stop = false + if (process.env.STOP_GRACEFULLY !== undefined) + stop = JSON.parse(process.env.STOP_GRACEFULLY) + + if (bot.enableCheckLoopHealth === false || stop === true) { logger.write(MODULE_NAME, "[WARN] run -> loop -> checkLoopHealth -> bot.enableCheckLoopHealth = " + bot.enableCheckLoopHealth); diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 50021a60c..de45d16b1 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -905,7 +905,7 @@ switch (nextWaitTime) { case 'Normal': { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.normalWaitTime / 1000) + " seconds."); } - checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.normalWaitTime * 5, bot.loopCounter); + checkLoopHealthHandle = setTimeout(checkLoopHealth, processConfig.deadWaitTime * 5, bot.loopCounter); nextLoopTimeoutHandle = setTimeout(loop, processConfig.normalWaitTime); if (global.WRITE_LOGS_TO_FILES === 'true') { logger.persist(); @@ -937,7 +937,11 @@ function checkLoopHealth(pLastLoop) { - if (bot.enableCheckLoopHealth === false) { + let stop = false + if (process.env.STOP_GRACEFULLY !== undefined) + stop = JSON.parse(process.env.STOP_GRACEFULLY) + + if (bot.enableCheckLoopHealth === false || stop === true) { logger.write(MODULE_NAME, "[WARN] run -> loop -> checkLoopHealth -> bot.enableCheckLoopHealth = " + bot.enableCheckLoopHealth); diff --git a/sequence - all processes.json b/sequence - all processes.json deleted file mode 100644 index 0ae6028e3..000000000 --- a/sequence - all processes.json +++ /dev/null @@ -1,145 +0,0 @@ -[ - { - "devTeam": "AAMasters", - "bot": "AACharly", - "mode": "noTime", - "resumeExecution": true, - "type": "Sensor", - "exchangeName": "Poloniex", - "process": "Live-Trades", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AACharly", - "mode": "oneMonth", - "resumeExecution": true, - "type": "Sensor", - "exchangeName": "Poloniex", - "process": "Hole-Fixing", - "startYear": 2019, - "endYear": 2019, - "month": 9 - }, - { - "devTeam": "AAMasters", - "bot": "AABruce", - "mode": "oneMonth", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Single-Period-Daily", - "startYear": 2019, - "endYear": 2021, - "month": 9 - }, - { - "devTeam": "AAMasters", - "bot": "AAOlivia", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAOlivia", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Market", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AATom", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AATom", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Market", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAChris", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAChris", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Market", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAPaula", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAPaula", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Market", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAJason", - "mode": "live", - "resumeExecution": true, - "type": "Trading-Engine", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "beginDatetime": "2019-08-01T08:00:00.000Z", - "timePeriod": "05-min" - } -] diff --git a/sequence.json b/sequence.json deleted file mode 100644 index c0db3b13f..000000000 --- a/sequence.json +++ /dev/null @@ -1,142 +0,0 @@ -[ - { - "devTeam": "AAMasters", - "bot": "AACharly", - "mode": "noTime", - "resumeExecution": true, - "type": "Sensor", - "exchangeName": "Poloniex", - "process": "Live-Trades" - }, - { - "devTeam": "AAMasters", - "bot": "AACharly", - "mode": "oneMonth", - "resumeExecution": true, - "type": "Sensor", - "exchangeName": "Poloniex", - "process": "Hole-Fixing", - "startYear": 2019, - "endYear": 2019, - "month": 9 - }, - { - "devTeam": "AAMasters", - "bot": "AABruce", - "mode": "oneMonth", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Single-Period-Daily", - "startYear": 2019, - "endYear": 2021, - "month": 9 - }, - { - "devTeam": "AAMasters", - "bot": "AAOlivia", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAOlivia", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Market", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AATom", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AATom", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Market", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAChris", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAChris", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Market", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAPaula", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAPaula", - "mode": "noTime", - "resumeExecution": true, - "type": "Indicator", - "exchangeName": "Poloniex", - "process": "Multi-Period-Market", - "startYear": 2019, - "endYear": 2021, - "month": 1 - }, - { - "devTeam": "AAMasters", - "bot": "AAJason", - "mode": "live", - "resumeExecution": true, - "type": "Trading-Engine", - "exchangeName": "Poloniex", - "process": "Multi-Period-Daily", - "beginDatetime": "2019-08-01T08:00:00.000Z", - "timePeriod": "05-min" - } -] From 0e606c971bcd90d338c48f3b1941412d8e7f5dfc Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 11 Sep 2019 17:04:43 +0200 Subject: [PATCH 25/32] Trading Engine Loop Improved --- TradingEngineProcessMainLoop.js | 31 ++++++++++++++++++++++++++++--- server.js | 22 ++++------------------ 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index de45d16b1..22bc0b3f2 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -94,6 +94,7 @@ bot.enableCheckLoopHealth = true; let fixedTimeLoopIntervalHandle; + let skipProcessing = true; if (bot.runAtFixedInterval === true) { @@ -101,10 +102,25 @@ } else { - loop(); + backtestingEventListenerSubscriptionId = global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Backstesting Started', undefined, undefined, undefined, startBackTesting) + liveTradingEventListenerSubscriptionId = global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Live-Trading Started', undefined, undefined, undefined, startLiveTrading) + stopRequestedEventListenerSubscriptionId = global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Stop Requested', undefined, undefined, undefined, stopRequested) + function startBackTesting() { + skipProcessing = false + } + + function startLiveTrading() { + skipProcessing = false + } + + function stopRequested() { + skipProcessing = true + } } + loop(); + function loop() { try { @@ -125,6 +141,17 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Entering function."); } + let nextWaitTime; + + /* Checking if we should process this loop or not.*/ + if (skipProcessing === true) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> We are going to skip this Loop bacause we were requested to stop or never asked to start."); } + console.log("[INFO] run -> loop -> We are going to skip this Loop bacause we were requested to stop or never asked to start.") + nextWaitTime = 'Normal'; + loopControl(nextWaitTime); + return + } + /* Loop Heartbeat sent to the UI */ hearBeat() @@ -155,8 +182,6 @@ let statusDependencies; let dataDependencies; - let nextWaitTime; - initializeStatusDependencies(); function initializeStatusDependencies() { diff --git a/server.js b/server.js index 65deed103..35e80281a 100644 --- a/server.js +++ b/server.js @@ -105,23 +105,6 @@ global.SYSTEM_EVENT_HANDLER.initialize('Task Server', bootLoader) function bootLoader() { - - //global.SYSTEM_EVENT_HANDLER.createEventHandler(global.USER_DEFINITION.name) - - /* - tryToListenToCockpit() - - function tryToListenToCockpit() { - - global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Simulation', 'Simulation Started', undefined, 'Task Server Boot Loader', onResponse, startSequence) - - function onResponse(message) { - if (message.result !== global.DEFAULT_OK_RESPONSE.result) { - setTimeout(tryToListenToCockpit, 3000) - } - } - } - */ for (let i = 0; i < global.USER_DEFINITION.bot.processes.length; i++) { let code = global.USER_DEFINITION.bot.processes[i].code @@ -136,12 +119,15 @@ function bootLoader() { } /* Heartbeat sent to the UI */ + + let key = global.USER_DEFINITION.name + '-' + global.USER_DEFINITION.type + '-' + global.USER_DEFINITION.id + + global.SYSTEM_EVENT_HANDLER.createEventHandler(key) global.HEARTBEAT_INTERVAL_HANDLER = setInterval(hearBeat, 1000) function hearBeat() { /* The heartbeat event is raised at the event handler of the instance of this task, created at the UI. */ - let key = global.USER_DEFINITION.name + '-' + global.USER_DEFINITION.type + '-' + global.USER_DEFINITION.id let event = { seconds: (new Date()).getSeconds() } From f18f86fa17a8f54491933fb152b6412c7e6d9cc5 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 11 Sep 2019 19:07:51 +0200 Subject: [PATCH 26/32] Re-enabling multiple process instances executing in parallel and integrating them with the system event handler --- Integrations/FileStorage.js | 2 +- Project.njsproj | 2 - Root.js | 52 +++++++----------- StatusReport.js | 6 ++- server.js | 103 ++++++++++++++---------------------- 5 files changed, 64 insertions(+), 101 deletions(-) diff --git a/Integrations/FileStorage.js b/Integrations/FileStorage.js index e9c7011cd..699797ddd 100644 --- a/Integrations/FileStorage.js +++ b/Integrations/FileStorage.js @@ -40,7 +40,7 @@ exports.newFileStorage = function newFileStorage(logger) { if (logger) { logger.write(MODULE_NAME, message); } else { - console.log("['" + new Date().toISOString() + "', 0,'" + MODULE_NAME + "','" + message + "']") + //console.log("['" + new Date().toISOString() + "', 0,'" + MODULE_NAME + "','" + message + "']") } } diff --git a/Project.njsproj b/Project.njsproj index 8db0453a3..50156dc5f 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -73,8 +73,6 @@ - - diff --git a/Root.js b/Root.js index d57d59ae2..c16978526 100644 --- a/Root.js +++ b/Root.js @@ -201,7 +201,6 @@ let processConfig = botConfig.processes[i]; - processConfig.startMode = global.EXECUTION_CONFIG.startMode // Override file storage configuration if(global.EXECUTION_CONFIG.timePeriod){ processConfig.timePeriod = global.EXECUTION_CONFIG.timePeriod } @@ -447,6 +446,7 @@ function runSensorBot(pBotConfig, pProcessConfig, pMonth, pYear) { try { + global.TOTAL_PROCESS_INSTANCES_CREATED++ const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); let logger; @@ -498,7 +498,7 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot execution was aborted."); logger.persist(); } - exitProcess() + exitProcessInstance() } } else { @@ -507,19 +507,21 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> err = "+ err.stack); logger.persist(); - exitProcess() + exitProcessInstance() } } } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> err = " + err.stack); - exitProcess() + exitProcessInstance() } } function runIndicatorBot(pBotConfig, pProcessConfig, pMonth, pYear) { try { + global.TOTAL_PROCESS_INSTANCES_CREATED++ + const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); let logger; @@ -568,7 +570,7 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); logger.persist(); } - exitProcess() + exitProcessInstance() } } else { @@ -576,18 +578,20 @@ logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> Failed to initialize the bot. "); console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> err = "+ err.stack); logger.persist(); - exitProcess() + exitProcessInstance() } } } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> err = "+ err.stack); - exitProcess() + exitProcessInstance() } } function runTradingEngine(pBotConfig, pProcessConfig) { + global.TOTAL_PROCESS_INSTANCES_CREATED++ + try { const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); let logger; @@ -632,7 +636,7 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); logger.persist(); } - exitProcess() + exitProcessInstance() } } else { @@ -640,47 +644,29 @@ logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> Failed to initialize the bot. "); console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> err = "+ err.stack); logger.persist(); - exitProcess() + exitProcessInstance() } } } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> err = "+ err.stack); - exitProcess() + exitProcessInstance() } } } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> err.message = " + err.message); - exitProcess() + exitProcessInstance() } } - function exitProcess() { + function exitProcessInstance() { global.ENDED_PROCESSES_COUNTER++ - console.log("[INFO] Task Server -> exitProcess -> Process #" + global.ENDED_PROCESSES_COUNTER + " exiting."); + console.log("[INFO] Task Server -> exitProcessInstance -> Process #" + global.ENDED_PROCESSES_COUNTER + " from " + global.TOTAL_PROCESS_INSTANCES_CREATED + ' exiting.'); - if (global.ENDED_PROCESSES_COUNTER === global.USER_DEFINITION.bot.processes.length) { - /* Cleaning Before Exiting. */ - clearInterval(global.HEARTBEAT_INTERVAL_HANDLER) - - for (let i = 0; i < global.USER_DEFINITION.bot.processes.length; i++) { - let code = global.USER_DEFINITION.bot.processes[i].code - - /* Delete the event handler for each process. */ - - let key = code.devTeam + "-" + code.codeName + "-" + code.process - let event = { - reason: 'Signal Received to Terminate this Process.' - } - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated', event) - global.SYSTEM_EVENT_HANDLER.deleteEventHandler(key) - } - - global.SYSTEM_EVENT_HANDLER.finalize() - global.SYSTEM_EVENT_HANDLER = undefined - console.log("[INFO] Task Server -> exitProcess -> Task Server Stopped."); + if (global.ENDED_PROCESSES_COUNTER === global.TOTAL_PROCESS_INSTANCES_CREATED) { + global.EXIT_NODE_PROCESS() } } } diff --git a/StatusReport.js b/StatusReport.js index 46c4b8e05..51d4778ff 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -58,8 +58,12 @@ if (owner.waitUntilNextUpdate === true) { /* This forces this process to wait until the process that this one depends on, updates its status report. */ + let extraCallerId = '' + if (pMonth) { extraCallerId = extraCallerId + '-' + pMonth } + if (pYear) { extraCallerId = extraCallerId + '-' + pYear } + let key = owner.devTeam + "-" + owner.bot + "-" + owner.process - let callerId = bot.devTeam + "-" + bot.codeName + "-" + bot.process + let callerId = bot.devTeam + "-" + bot.codeName + "-" + bot.process + extraCallerId let subscriptionIdStatusReport diff --git a/server.js b/server.js index 35e80281a..0f9438085 100644 --- a/server.js +++ b/server.js @@ -84,7 +84,15 @@ global.WRITE_LOGS_TO_FILES = process.env.WRITE_LOGS_TO_FILES global.EXCHANGE_NAME = process.env.EXCHANGE_NAME global.MARKET = { assetA: 'USDT', assetB: 'BTC' } global.CLONE_EXECUTOR = { codeName: 'AACloud', version: '1.1' } + +/* +We need to count how many process instances we deployd and how many of them have already finished their job, either +because they just finished or because there was a request to stop the proceses. In this way, once we reach the +amount of instances started, we can safelly destroy the rest of the objects running and let this nodejs process die. +*/ + global.ENDED_PROCESSES_COUNTER = 0 +global.TOTAL_PROCESS_INSTANCES_CREATED = 0 /* Here we listen for the message to stop this Task / Process comming from the Task Manager, which is the paret of this node js process. */ process.on('message', message => { @@ -92,9 +100,40 @@ process.on('message', message => { process.env.STOP_GRACEFULLY = true; + /* + There are some process that might no be able to end grafully, for example the ones schedulle to process information in a future day or month. + In order to be sure that the process will be terminated, we schedulle one forced exit in 2 minutes from now. + */ + console.log('[INFO] Task Server -> server -> process.on -> Executing order received from Task Manager to Stop this Task. Nodejs process will be exited in less than 2 minutes.') + setTimeout(global.EXIT_NODE_PROCESS, 120000); } }); +global.EXIT_NODE_PROCESS = function exitProcess() { + + /* Cleaning Before Exiting. */ + clearInterval(global.HEARTBEAT_INTERVAL_HANDLER) + + for (let i = 0; i < global.USER_DEFINITION.bot.processes.length; i++) { + let code = global.USER_DEFINITION.bot.processes[i].code + + /* Delete the event handler for each process. */ + + let key = code.devTeam + "-" + code.codeName + "-" + code.process + let event = { + reason: 'Signal Received to Terminate this Process.' + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Process Terminated', event) + global.SYSTEM_EVENT_HANDLER.deleteEventHandler(key) + } + + global.SYSTEM_EVENT_HANDLER.finalize() + global.SYSTEM_EVENT_HANDLER = undefined + console.log("[INFO] Task Server -> exitProcessInstance -> Task Server Stopped."); + + process.exit() +} + let notFirstSequence = false; /* Setting up the global Event Handler */ @@ -293,73 +332,10 @@ async function readExecutionConfiguration(execution, processIndex) { } } - let startMode - // General Financial Being Configuration global.DEV_TEAM = process.env.DEV_TEAM global.CURRENT_BOT_REPO = process.env.BOT + "-" + process.env.TYPE + "-Bot" - if (process.env.TYPE === 'Trading' || process.env.TYPE === 'Trading-Engine') { - let live = { - run: 'false', - resumeExecution: execution.resumeExecution, - beginDatetime: new Date(global.DEFINITION.simulationParams.timestamp).toISOString(), - endDatetime: finalDatetime - } - - let backtest = { - run: 'false', - resumeExecution: execution.resumeExecution, - beginDatetime: initialDatetime, - endDatetime: finalDatetime - } - - let competition = { - run: 'false', - resumeExecution: execution.resumeExecution, - beginDatetime: initialDatetime, - endDatetime: finalDatetime - } - - startMode = { - live: live, - backtest: backtest, - competition: competition - } - } else if (process.env.TYPE === 'Indicator' || process.env.TYPE === 'Sensor') { - let allMonths = { - run: "false", - minYear: process.env.MIN_YEAR, - maxYear: process.env.MAX_YEAR - } - let oneMonth = { - run: "false", - year: process.env.MIN_YEAR, - month: process.env.MONTH - } - let noTime = { - run: "false", - beginDatetime: process.env.BEGIN_DATE_TIME, - resumeExecution: execution.resumeExecution - } - let fixedInterval = { - run: "false", - interval: process.env.INTERVAL - } - - startMode = { - allMonths: allMonths, - oneMonth: oneMonth, - noTime: noTime, - fixedInterval: fixedInterval - } - } else { - console.log("[ERROR] readExecutionConfiguration -> Bot Type is invalid."); - throw new Error("readExecutionConfiguration -> Bot Type is invalid.") - } - - startMode[process.env.START_MODE].run = "true" - if (botProcess === undefined) { botProcess = process.env.PROCESS } // Only use the .env when nothing comes at Definition.json let cloneToExecute = { enabled: "true", @@ -378,7 +354,6 @@ async function readExecutionConfiguration(execution, processIndex) { global.EXECUTION_CONFIG = { cloneToExecute: cloneToExecute, - startMode: startMode, timePeriod: getTimePeriod(timePeriod), timePeriodFileStorage: timePeriod, timePeriodFilter: timePeriodFilter, From a2136f90f13f05b7b3aba8705def8b1446f591dd Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 11 Sep 2019 19:30:12 +0200 Subject: [PATCH 27/32] Better Logging --- DebugLog.js | 6 +++++- Root.js | 2 +- server.js | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/DebugLog.js b/DebugLog.js index c5ae362da..5112b0882 100644 --- a/DebugLog.js +++ b/DebugLog.js @@ -130,7 +130,11 @@ exports.newDebugLog = function newDebugLog() { let logLine = '\r\n' + message; if (process.env.CONSOLE_LOG === "true" || message.indexOf("ERROR") > 0) { - console.log(message) + let key = '' + if (thisObject.bot) { + key = thisObject.bot.devTeam + '-' + thisObject.bot.codeName + '-' + thisObject.bot.process + } + console.log(message + ' @ ' + key) } accumulatedLog = accumulatedLog + logLine; diff --git a/Root.js b/Root.js index c16978526..ec9720690 100644 --- a/Root.js +++ b/Root.js @@ -663,7 +663,7 @@ function exitProcessInstance() { global.ENDED_PROCESSES_COUNTER++ - console.log("[INFO] Task Server -> exitProcessInstance -> Process #" + global.ENDED_PROCESSES_COUNTER + " from " + global.TOTAL_PROCESS_INSTANCES_CREATED + ' exiting.'); + console.log("[INFO] Task Server -> " + global.USER_DEFINITION.name + " -> exitProcessInstance -> Process #" + global.ENDED_PROCESSES_COUNTER + " from " + global.TOTAL_PROCESS_INSTANCES_CREATED + " exiting."); if (global.ENDED_PROCESSES_COUNTER === global.TOTAL_PROCESS_INSTANCES_CREATED) { global.EXIT_NODE_PROCESS() diff --git a/server.js b/server.js index 0f9438085..83640a8ac 100644 --- a/server.js +++ b/server.js @@ -129,7 +129,7 @@ global.EXIT_NODE_PROCESS = function exitProcess() { global.SYSTEM_EVENT_HANDLER.finalize() global.SYSTEM_EVENT_HANDLER = undefined - console.log("[INFO] Task Server -> exitProcessInstance -> Task Server Stopped."); + console.log("[INFO] Task Server -> " + global.USER_DEFINITION.name + " -> EXIT_NODE_PROCESS -> Task Server Stopped."); process.exit() } From 3dd9e954c58835ddf6d8be605e40869bc1f86d66 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 11 Sep 2019 20:51:41 +0200 Subject: [PATCH 28/32] All Sensors & Indicators Running with no errors with system events --- Root.js | 4 ++-- TradingEngineProcessMainLoop.js | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Root.js b/Root.js index ec9720690..83e31a185 100644 --- a/Root.js +++ b/Root.js @@ -208,12 +208,12 @@ if (processConfig.framework !== undefined) { if (processConfig.framework.name === "Multi-Period-Daily" || processConfig.framework.name === "Multi-Period-Market" || processConfig.framework.name === "Multi-Period") { - processConfig.framework.startDate.resumeExecution = false; + processConfig.framework.startDate.resumeExecution = true; if (processConfig.startMode.noTime !== undefined) { if (processConfig.startMode.noTime.run === "true") { if (processConfig.startMode.noTime.beginDatetime !== undefined) { processConfig.framework.startDate.fixedDate = processConfig.startMode.noTime.beginDatetime; - processConfig.framework.startDate.resumeExecution = false; + processConfig.framework.startDate.resumeExecution = true; } } } diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 22bc0b3f2..32cfb5938 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -107,10 +107,13 @@ stopRequestedEventListenerSubscriptionId = global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Stop Requested', undefined, undefined, undefined, stopRequested) function startBackTesting() { + processConfig.framework.startDate.resumeExecution = false; skipProcessing = false } function startLiveTrading() { + processConfig.framework.startDate.fixedDate = new Date() + processConfig.framework.startDate.resumeExecution = false; skipProcessing = false } From e8165368fd0137a0a5412a7e95147535e187dbe5 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 11 Sep 2019 22:15:50 +0200 Subject: [PATCH 29/32] Dates at Trading Engine Loop --- MultiPeriodDaily.js | 10 ------- MultiPeriodMarket.js | 8 ------ Root.js | 51 ++++++--------------------------- TradingEngineProcessMainLoop.js | 51 +++++++++++++++++++++++++++------ server.js | 38 ++---------------------- 5 files changed, 54 insertions(+), 104 deletions(-) diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 968ad329c..b3ce60d62 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -372,16 +372,6 @@ } } - let timePeriodFilter = global.EXECUTION_CONFIG.timePeriodFilter - if (timePeriodFilter !== undefined) { - if (timePeriodFilter.indexOf(outputPeriodLabel) === -1) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Discarding period for being filtered out. -> outputPeriodLabel = " + outputPeriodLabel); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> Discarding period for being filtered out. -> timePeriodFilter = " + timePeriodFilter); } - periodsControlLoop(); - return; - } - } - let dependencyIndex = 0; dataFiles = []; diff --git a/MultiPeriodMarket.js b/MultiPeriodMarket.js index 619c4a097..c03f1420a 100644 --- a/MultiPeriodMarket.js +++ b/MultiPeriodMarket.js @@ -143,14 +143,6 @@ let dependencyIndex = 0; dataFiles = []; - let timePeriodFilter = global.EXECUTION_CONFIG.timePeriodFilter - if (timePeriodFilter !== undefined) { - if (timePeriodFilter.indexOf(outputPeriodLabel) === -1) { - periodsControlLoop(); - return; - } - } - dependencyLoopBody(); function dependencyLoopBody() { diff --git a/Root.js b/Root.js index 83e31a185..511b904f8 100644 --- a/Root.js +++ b/Root.js @@ -217,19 +217,11 @@ } } } - if (processConfig.startMode.live !== undefined) { - if (processConfig.startMode.live.run === "true") { - if (processConfig.startMode.live.beginDatetime !== undefined) { - processConfig.framework.startDate.fixedDate = processConfig.startMode.live.beginDatetime; - processConfig.framework.startDate.resumeExecution = processConfig.startMode.live.resumeExecution; - } - } - } - if (processConfig.startMode.backtest !== undefined) { - if (processConfig.startMode.backtest.run === "true") { - if (processConfig.startMode.backtest.beginDatetime !== undefined) { - processConfig.framework.startDate.fixedDate = processConfig.startMode.backtest.beginDatetime; - processConfig.framework.startDate.resumeExecution = processConfig.startMode.backtest.resumeExecution; + if (processConfig.startMode.userDefined !== undefined) { + if (processConfig.startMode.userDefined.run === "true") { + if (processConfig.startMode.userDefined.beginDatetime !== undefined) { + processConfig.framework.startDate.fixedDate = processConfig.startMode.userDefined.beginDatetime; + processConfig.framework.startDate.resumeExecution = processConfig.startMode.userDefined.resumeExecution; } } } @@ -377,17 +369,17 @@ } } - if (processConfig.startMode.live !== undefined) { + if (processConfig.startMode.userDefined !== undefined) { - if (processConfig.startMode.live.run === "true") { + if (processConfig.startMode.userDefined.run === "true") { - botConfig.startMode = "Live"; + botConfig.startMode = "User Defined"; console.log(logDisplace + "Root : [INFO] start -> findProcess -> Process found at the bot configuration file. -> Start Mode = " + botConfig.startMode); let month = pad((new Date()).getUTCMonth() + 1, 2); let year = (new Date()).getUTCFullYear(); - if (processConfig.startMode.live.resumeExecution === true) { + if (processConfig.startMode.userDefined.resumeExecution === true) { botConfig.hasTheBotJustStarted = false; } else { botConfig.hasTheBotJustStarted = true; @@ -405,31 +397,6 @@ } } - if (processConfig.startMode.backtest !== undefined) { - - if (processConfig.startMode.backtest.run === "true") { - - botConfig.startMode = "Backtest"; - console.log(logDisplace + "Root : [INFO] start -> findProcess -> Process found at the bot configuration file. -> Start Mode = " + botConfig.startMode); - - botConfig.backtest = processConfig.startMode.backtest - - /* Backtest Mode does not support Resume Execution, so this is the only way. */ - - botConfig.hasTheBotJustStarted = true; - - switch (botConfig.type) { - case 'Trading-Engine': { - runTradingEngine(botConfig, processConfig); - break; - } - default: { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); - } - } - } - } - } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected exception. -> err.message = " + err.message); } diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 32cfb5938..0a9b7fa62 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -107,14 +107,47 @@ stopRequestedEventListenerSubscriptionId = global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Stop Requested', undefined, undefined, undefined, stopRequested) function startBackTesting() { + /* Reload the definition in case something changed. */ + global.DEFINITION = require(process.env.INTER_PROCESS_FILES_PATH + '/Definition'); + + bot.startMode = "Backtest" processConfig.framework.startDate.resumeExecution = false; skipProcessing = false + bot.hasTheBotJustStarted = true + + /* We exctact the initial and final date for the backtest from the Definition */ + processConfig.framework.startDate.fixedDate = new Date() // If we can not get for any reason the initial date of the backtest, we will start at present time. + + if (global.DEFINITION.tradingSystem !== undefined) { + if (global.DEFINITION.tradingSystem.parameters !== undefined) { + if (global.DEFINITION.tradingSystem.parameters.timeRange !== undefined) { + if (global.DEFINITION.tradingSystem.parameters.timeRange.code !== undefined) { + try { + let code = JSON.parse(global.DEFINITION.tradingSystem.parameters.timeRange.code) + if (code.initialDatetime !== undefined) { + processConfig.framework.startDate.fixedDate = code.initialDatetime /* The second override occurs here, with the date explicitelly defined by the user */ + } + if (code.finalDatetime !== undefined) { + processConfig.framework.endDate.fixedDate = code.finalDatetime + } + } catch (err) { + global.DEFINITION.tradingSystem.parameters.timeRange.error = err.message + } + } + } + } + } } function startLiveTrading() { + /* Reload the definition in case something changed. */ + global.DEFINITION = require(process.env.INTER_PROCESS_FILES_PATH + '/Definition'); + + bot.startMode = "Live" processConfig.framework.startDate.fixedDate = new Date() processConfig.framework.startDate.resumeExecution = false; skipProcessing = false + bot.hasTheBotJustStarted = true } function stopRequested() { @@ -146,15 +179,6 @@ let nextWaitTime; - /* Checking if we should process this loop or not.*/ - if (skipProcessing === true) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> We are going to skip this Loop bacause we were requested to stop or never asked to start."); } - console.log("[INFO] run -> loop -> We are going to skip this Loop bacause we were requested to stop or never asked to start.") - nextWaitTime = 'Normal'; - loopControl(nextWaitTime); - return - } - /* Loop Heartbeat sent to the UI */ hearBeat() @@ -178,6 +202,15 @@ console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " Entered into Main Loop # " + pad(Number(bot.loopCounter), 8)); + /* Checking if we should process this loop or not.*/ + if (skipProcessing === true) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> We are going to skip this Loop bacause we were requested to stop or never asked to start."); } + console.log("[INFO] run -> loop -> We are going to skip this Loop bacause we were requested to stop or never asked to start.") + nextWaitTime = 'Normal'; + loopControl(nextWaitTime); + return + } + /* We will prepare first the infraestructure needed for the bot to run. There are 3 modules we need to sucessfullly initialize first. */ let userBot; diff --git a/server.js b/server.js index 83640a8ac..5f55aea66 100644 --- a/server.js +++ b/server.js @@ -66,7 +66,7 @@ if (global.USER_DEFINITION !== undefined) { } else { // I use this section to debug in standalone mode. - let argument = '{"type":"Task","name":"Brings Trades Records from the Exchange","bot":{"type":"Sensor","name":"Charly","processes":[{"type":"Process","name":"Live Trades","code":{"devTeam":"AAMasters","bot":"AACharly","mode":"noTime","resumeExecution":true,"type":"Sensor","exchangeName":"Poloniex","process":"Live-Trades"},"id":"3aae75f5-67ae-4ae5-9586-ec5640f79b18"}]},"id":"54444b07-eee2-4fa8-9183-8fea3b5ff9de"}' + let argument = ' {"type":"Task","name":"Runs Backtests, Fordwardtests & Live Trades ","bot":{"type":"Trading Engine","processes":[{"type":"Process","name":"Multi Period","code":{"devTeam":"AAMasters","bot":"AAJason","mode":"live","resumeExecution":true,"type":"Trading-Engine","exchangeName":"Poloniex","process":"Multi-Period-Daily","beginDatetime":"2019-08-01T08:00:00.000Z","timePeriod":"05-min"},"id":"1bfce24d-8c05-4be9-bd25-328f07c85265"}]},"id":"cbb13086-608d-4bb4-960a-17a81038877b"}' try { global.USER_DEFINITION = JSON.parse(argument) } catch (err) { @@ -186,7 +186,6 @@ function startSequence() { process.env.STOP_GRACEFULLY = false; execution.devTeam ? process.env.DEV_TEAM = execution.devTeam : undefined; execution.bot ? process.env.BOT = execution.bot : undefined; - execution.mode ? process.env.START_MODE = execution.mode : undefined; execution.resumeExecution = true; execution.type ? process.env.TYPE = execution.type : undefined; execution.process ? process.env.PROCESS = execution.process : undefined; @@ -234,10 +233,6 @@ async function readExecutionConfiguration(execution, processIndex) { let timePeriodFilter let botProcess - /* Dates are taken initially from .env, but can be overwritten if they are defined by the user */ - let initialDatetime = process.env.BEGIN_DATE_TIME - let finalDatetime = process.env.END_DATE_TIME - if (execution.type === 'Trading-Engine') { /* The Trading Engine only resumes its execution after the first sequence was completed. */ @@ -245,18 +240,9 @@ async function readExecutionConfiguration(execution, processIndex) { execution.resumeExecution = false } - /* We set the START MODE of the Trading Engine */ - if (process.env.KEY === undefined || process.env.SECRET === undefined) { - process.env.START_MODE = "backtest" // if we dont have keys, we swtich to backtest mode to avoid exchange errors. - } else { - process.env.START_MODE = "live" // If we have keys, then we are in live mode. - } - if (global.DEFINITION !== undefined) { if (global.DEFINITION.simulationParams !== undefined) { - if (global.DEFINITION.simulationParams.beginDatetime !== undefined) { - initialDatetime = new Date(global.DEFINITION.simulationParams.beginDatetime) /* The first override occurs here, with the simulation parameters */ - } + /* Here we only look for one timePeriod, in the future we will be able to process the whole array, but not for now. */ if (global.DEFINITION.simulationParams.timePeriodDailyArray !== undefined) { if (global.DEFINITION.simulationParams.timePeriodDailyArray.length === 1) { @@ -271,25 +257,7 @@ async function readExecutionConfiguration(execution, processIndex) { } } } - if (global.DEFINITION.tradingSystem !== undefined) { - if (global.DEFINITION.tradingSystem.parameters !== undefined) { - if (global.DEFINITION.tradingSystem.parameters.timeRange !== undefined) { - if (global.DEFINITION.tradingSystem.parameters.timeRange.code !== undefined) { - try { - let code = JSON.parse(global.DEFINITION.tradingSystem.parameters.timeRange.code) - if (code.initialDatetime !== undefined) { - initialDatetime = code.initialDatetime /* The second override occurs here, with the date explicitelly defined by the user */ - } - if (code.finalDatetime !== undefined) { - finalDatetime = code.finalDatetime - } - } catch (err) { - global.DEFINITION.tradingSystem.parameters.timeRange.error = err.message - } - } - } - } - } + /* Get the initial balance from the global.DEFINITION */ let tradingSystem = global.DEFINITION.tradingSystem From 2a88a6da8819f4901c652aa2b307a09e292a7c7c Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 12 Sep 2019 10:07:30 +0200 Subject: [PATCH 30/32] Backtests Tested and Running with events --- IndicatorBotProcessMainLoop.js | 12 +- MultiPeriod.js | 68 ++- Root.js | 829 ++++++++++++++++---------------- SensorBotProcessMainLoop.js | 12 +- TradingEngineProcessMainLoop.js | 35 +- server.js | 51 +- 6 files changed, 501 insertions(+), 506 deletions(-) diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 38111dd02..a1bfe1ef8 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -858,11 +858,7 @@ function checkLoopHealth(pLastLoop) { - let stop = false - if (process.env.STOP_GRACEFULLY !== undefined) - stop = JSON.parse(process.env.STOP_GRACEFULLY) - - if (bot.enableCheckLoopHealth === false || stop === true) { + if (bot.enableCheckLoopHealth === false || global.STOP_TASK_GRACEFULLY === true) { logger.write(MODULE_NAME, "[WARN] run -> loop -> checkLoopHealth -> bot.enableCheckLoopHealth = " + bot.enableCheckLoopHealth); @@ -898,11 +894,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> shallWeStop -> Entering function. "); } - let stop = false - if (process.env.STOP_GRACEFULLY !== undefined) - stop = JSON.parse(process.env.STOP_GRACEFULLY) - - if (!stop) { + if (!global.STOP_TASK_GRACEFULLY) { continueCallBack(); } else { stopCallBack(); diff --git a/MultiPeriod.js b/MultiPeriod.js index d464c00dd..c1faa087f 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -326,14 +326,10 @@ let dependencyIndex = 0; dataFiles = []; - let timePeriodFilter = global.EXECUTION_CONFIG.timePeriodFilter - if (timePeriodFilter !== undefined) { - if (timePeriodFilter.indexOf(outputPeriodLabel) > -1) { - currentTimePeriod = global.marketFilesPeriods[n][0]; - currentOutputPeriodLabel = global.marketFilesPeriods[n][1]; - } + if (global.DEFINITION.simulationParams.timePeriod === outputPeriodLabel) { + currentTimePeriod = global.marketFilesPeriods[n][0]; + currentOutputPeriodLabel = global.marketFilesPeriods[n][1]; } - dependencyLoopBody(); function dependencyLoopBody() { @@ -515,7 +511,7 @@ } - periodsLoop(); + checkStopTaskGracefully(); } catch (err) { logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> advanceTime -> err = " + err.stack); @@ -523,6 +519,52 @@ } } + function checkStopTaskGracefully() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> checkStopTaskGracefully -> Entering function."); } + + /* Validation that we dont need to stop. */ + + if (global.STOP_TASK_GRACEFULLY === true) { + + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + + } + + checkStopProcessing(); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> checkStopTaskGracefully -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function checkStopProcessing() { + + try { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> checkStopProcessing -> Entering function."); } + + /* Validation that we dont need to stop. */ + + if (global.STOP_PROCESSING === true) { + + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + + } + + periodsLoop(); + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriodsDailyFiles -> checkStopProcessing -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + function periodsLoop() { try { @@ -568,17 +610,13 @@ } } - let timePeriodFilter = global.EXECUTION_CONFIG.timePeriodFilter - if (timePeriodFilter !== undefined) { - if (timePeriodFilter.indexOf(outputPeriodLabel) > -1) { - currentTimePeriod = global.dailyFilePeriods[n][0]; - currentOutputPeriodLabel = global.dailyFilePeriods[n][1]; - } + if (global.DEFINITION.simulationParams.timePeriod === outputPeriodLabel) { + currentTimePeriod = global.dailyFilePeriods[n][0]; + currentOutputPeriodLabel = global.dailyFilePeriods[n][1]; } let dependencyIndex = 0; dataFiles = []; - dependencyLoopBody(); function dependencyLoopBody() { diff --git a/Root.js b/Root.js index 511b904f8..b26b69501 100644 --- a/Root.js +++ b/Root.js @@ -108,194 +108,226 @@ function start(processIndex) { - /* Now we will run according to what we see at the config file. */ + try { - /* Small Function to fix numbers into strings in a cool way. */ + /* Now we will run according to what we see at the config file. */ - function pad(str, max) { - str = str.toString(); - return str.length < max ? pad("0" + str, max) : str; - } + /* Small Function to fix numbers into strings in a cool way. */ - /* Process Loops Declarations. */ - - const INDICATOR_BOT_MAIN_LOOP_MODULE = require('./IndicatorBotProcessMainLoop'); - const EXTRACTION_BOT_MAIN_LOOP_MODULE = require('./SensorBotProcessMainLoop'); - const TRADING_ENGINE_MAIN_LOOP_MODULE = require('./TradingEngineProcessMainLoop'); - - /* Loop through all the processes configured to be run by this Node.js Instance. */ - - let cloneToExecute = global.EXECUTION_CONFIG.cloneToExecute; - - /* Now we will read the config of the bot from the path we obtained at the Task Server config. */ - - let botConfig; - - getBotConfig(); - - function getBotConfig() { - - try { - console.log(logDisplace + "Root : [INFO] start -> getBotConfig -> Entering function. "); - - const FILE_STORAGE = require('./Integrations/FileStorage.js'); - let fileStorage = FILE_STORAGE.newFileStorage(); - - let filePath = global.DEV_TEAM + '/bots/' + global.CURRENT_BOT_REPO + '/this.bot.config.json'; - - fileStorage.getTextFile(global.DEV_TEAM, filePath, onFileReceived); - - function onFileReceived(err, text) { - - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> err = " + JSON.stringify(err)); - - return; - } - - try { - botConfig = JSON.parse(text); - botConfig.repo = cloneToExecute.repo; - findProcess(processIndex); - } catch (err) { - console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> err = " + JSON.stringify(err)); - return; - } - } + function pad(str, max) { + str = str.toString(); + return str.length < max ? pad("0" + str, max) : str; } - catch (err) { - console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> err.message = " + err.message); - return; - } - } - function findProcess(processIndex) { + /* Process Loops Declarations. */ - try { - if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> Entering function. "); } + const INDICATOR_BOT_MAIN_LOOP_MODULE = require('./IndicatorBotProcessMainLoop'); + const SENSOR_BOT_MAIN_LOOP_MODULE = require('./SensorBotProcessMainLoop'); + const TRADING_ENGINE_MAIN_LOOP_MODULE = require('./TradingEngineProcessMainLoop'); - botConfig.process = cloneToExecute.process; - botConfig.debug = {}; - botConfig.processIndex = processIndex + /* Loop through all the processes configured to be run by this Node.js Instance. */ - /* Loop Counter */ + let cloneToExecute = global.EXECUTION_CONFIG.cloneToExecute; - botConfig.loopCounter = 0; + /* Now we will read the config of the bot from the path we obtained at the Task Server config. */ - /* File Path Root */ + let botConfig; - botConfig.filePathRoot = botConfig.devTeam + "/" + botConfig.codeName + "." + botConfig.version.major + "." + botConfig.version.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + botConfig.dataSetVersion; + getBotConfig(); - if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> filePathRoot = " + botConfig.filePathRoot); } + function getBotConfig() { - /* Now we loop throug all the configured processes at each bots configuration until we find the one we are supposed to run at this Node.js process. */ + try { + console.log(logDisplace + "Root : [INFO] start -> getBotConfig -> Entering function. "); - let processFound = false; + const FILE_STORAGE = require('./Integrations/FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(); - for (let i = 0; i < botConfig.processes.length; i++) { + let filePath = global.DEV_TEAM + '/bots/' + global.CURRENT_BOT_REPO + '/this.bot.config.json'; - if (botConfig.processes[i].name === cloneToExecute.process) { + fileStorage.getTextFile(global.DEV_TEAM, filePath, onFileReceived); - processFound = true; - if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> Process found at the bot configuration file. -> cloneToExecute.process = " + cloneToExecute.process); } + function onFileReceived(err, text) { - let processConfig = botConfig.processes[i]; + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> err = " + JSON.stringify(err)); - if(global.EXECUTION_CONFIG.timePeriod){ - processConfig.timePeriod = global.EXECUTION_CONFIG.timePeriod - } - - - if (processConfig.framework !== undefined) { - if (processConfig.framework.name === "Multi-Period-Daily" || processConfig.framework.name === "Multi-Period-Market" || processConfig.framework.name === "Multi-Period") { - processConfig.framework.startDate.resumeExecution = true; - if (processConfig.startMode.noTime !== undefined) { - if (processConfig.startMode.noTime.run === "true") { - if (processConfig.startMode.noTime.beginDatetime !== undefined) { - processConfig.framework.startDate.fixedDate = processConfig.startMode.noTime.beginDatetime; - processConfig.framework.startDate.resumeExecution = true; - } - } - } - if (processConfig.startMode.userDefined !== undefined) { - if (processConfig.startMode.userDefined.run === "true") { - if (processConfig.startMode.userDefined.beginDatetime !== undefined) { - processConfig.framework.startDate.fixedDate = processConfig.startMode.userDefined.beginDatetime; - processConfig.framework.startDate.resumeExecution = processConfig.startMode.userDefined.resumeExecution; - } - } - } - } + return; } try { + botConfig = JSON.parse(text); + botConfig.repo = cloneToExecute.repo; + findProcess(processIndex); + } catch (err) { + console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> onInizialized -> onFileReceived -> err = " + JSON.stringify(err)); + return; + } + } + } + catch (err) { + console.log(logDisplace + "Root : [ERROR] start -> getBotConfig -> err.message = " + err.message); + return; + } + } - /* We test each type of start Mode to get what to run and how. */ + function findProcess(processIndex) { - if (processConfig.startMode.allMonths !== undefined) { + try { + if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> Entering function. "); } - if (processConfig.startMode.allMonths.run === "true") { + botConfig.process = cloneToExecute.process; + botConfig.debug = {}; + botConfig.processIndex = processIndex - if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> allMonths start mode detected. "); } + /* Loop Counter */ - for (let year = processConfig.startMode.allMonths.maxYear; year >= processConfig.startMode.allMonths.minYear; year--) { + botConfig.loopCounter = 0; - for (let month = 12; month > 0; month--) { + /* File Path Root */ - let padMonth = pad(month, 2) + botConfig.filePathRoot = botConfig.devTeam + "/" + botConfig.codeName + "." + botConfig.version.major + "." + botConfig.version.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + botConfig.dataSetVersion; - let newInstanceBotConfig = JSON.parse(JSON.stringify(botConfig)); + if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> filePathRoot = " + botConfig.filePathRoot); } - newInstanceBotConfig.debug = { - month: pad(month, 2), - year: pad(year, 4) - }; + /* Now we loop throug all the configured processes at each bots configuration until we find the one we are supposed to run at this Node.js process. */ - let timeDelay = Math.random() * 10 * 1000; // We introduce a short delay so as to not overload the machine. - setTimeout(startProcess, timeDelay); + let processFound = false; - function startProcess() { + for (let i = 0; i < botConfig.processes.length; i++) { - if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> startProcess -> Ready to start process."); } + if (botConfig.processes[i].name === cloneToExecute.process) { - switch (botConfig.type) { - case 'Sensor': { - runSensorBot(newInstanceBotConfig, processConfig, padMonth, year); - break; - } - case 'Indicator': { - runIndicatorBot(newInstanceBotConfig, processConfig, padMonth, year); - break; - } - default: { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> startProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); - } - } + processFound = true; + if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> Process found at the bot configuration file. -> cloneToExecute.process = " + cloneToExecute.process); } + + let processConfig = botConfig.processes[i]; + + if (global.EXECUTION_CONFIG.timePeriod) { + processConfig.timePeriod = global.EXECUTION_CONFIG.timePeriod + } + + + if (processConfig.framework !== undefined) { + if (processConfig.framework.name === "Multi-Period-Daily" || processConfig.framework.name === "Multi-Period-Market" || processConfig.framework.name === "Multi-Period") { + processConfig.framework.startDate.resumeExecution = true; + if (processConfig.startMode.noTime !== undefined) { + if (processConfig.startMode.noTime.run === "true") { + if (processConfig.startMode.noTime.beginDatetime !== undefined) { + processConfig.framework.startDate.fixedDate = processConfig.startMode.noTime.beginDatetime; + processConfig.framework.startDate.resumeExecution = true; + } + } + } + if (processConfig.startMode.userDefined !== undefined) { + if (processConfig.startMode.userDefined.run === "true") { + if (processConfig.startMode.userDefined.beginDatetime !== undefined) { + processConfig.framework.startDate.fixedDate = processConfig.startMode.userDefined.beginDatetime; + processConfig.framework.startDate.resumeExecution = processConfig.startMode.userDefined.resumeExecution; } } } } } - if (processConfig.startMode.oneMonth !== undefined) { + try { - if (processConfig.startMode.oneMonth.run === "true") { + /* We test each type of start Mode to get what to run and how. */ - if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> oneMonth start mode detected. "); } + if (processConfig.startMode.allMonths !== undefined) { - startProcess(); + if (processConfig.startMode.allMonths.run === "true") { - function startProcess() { + if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> allMonths start mode detected. "); } - if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> startProcess -> Ready to start process."); } + for (let year = processConfig.startMode.allMonths.maxYear; year >= processConfig.startMode.allMonths.minYear; year--) { - let month = pad(processConfig.startMode.oneMonth.month, 2); - let year = processConfig.startMode.oneMonth.year; + for (let month = 12; month > 0; month--) { - botConfig.debug = { - month: pad(month, 2), - year: pad(year, 4) - }; + let padMonth = pad(month, 2) + + let newInstanceBotConfig = JSON.parse(JSON.stringify(botConfig)); + + newInstanceBotConfig.debug = { + month: pad(month, 2), + year: pad(year, 4) + }; + + let timeDelay = Math.random() * 10 * 1000; // We introduce a short delay so as to not overload the machine. + setTimeout(startProcess, timeDelay); + + function startProcess() { + + if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> startProcess -> Ready to start process."); } + + switch (botConfig.type) { + case 'Sensor': { + runSensorBot(newInstanceBotConfig, processConfig, padMonth, year); + break; + } + case 'Indicator': { + runIndicatorBot(newInstanceBotConfig, processConfig, padMonth, year); + break; + } + default: { + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> startProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); + } + } + } + } + } + } + } + + if (processConfig.startMode.oneMonth !== undefined) { + + if (processConfig.startMode.oneMonth.run === "true") { + + if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> oneMonth start mode detected. "); } + + startProcess(); + + function startProcess() { + + if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> startProcess -> Ready to start process."); } + + let month = pad(processConfig.startMode.oneMonth.month, 2); + let year = processConfig.startMode.oneMonth.year; + + botConfig.debug = { + month: pad(month, 2), + year: pad(year, 4) + }; + + switch (botConfig.type) { + case 'Sensor': { + runSensorBot(botConfig, processConfig, month, year); + break; + } + case 'Indicator': { + runIndicatorBot(botConfig, processConfig, month, year); + break; + } + default: { + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> startProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); + } + } + } + } + } + + if (processConfig.startMode.noTime !== undefined) { + + if (processConfig.startMode.noTime.run === "true") { + + let month = pad((new Date()).getUTCMonth() + 1, 2); + let year = (new Date()).getUTCFullYear(); + + if (processConfig.startMode.noTime.resumeExecution === true) { + botConfig.hasTheBotJustStarted = false; + } else { + botConfig.hasTheBotJustStarted = true; + } switch (botConfig.type) { case 'Sensor': { @@ -306,335 +338,308 @@ runIndicatorBot(botConfig, processConfig, month, year); break; } + case 'Trading-Engine': { + runTradingEngine(botConfig, processConfig); + break; + } default: { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> startProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); } } } } - } - if (processConfig.startMode.noTime !== undefined) { + if (processConfig.startMode.fixedInterval !== undefined) { - if (processConfig.startMode.noTime.run === "true") { + if (processConfig.startMode.fixedInterval.run === "true") { - let month = pad((new Date()).getUTCMonth() + 1, 2); - let year = (new Date()).getUTCFullYear(); + botConfig.runAtFixedInterval = true; + botConfig.fixedInterval = processConfig.startMode.fixedInterval.interval; - if (processConfig.startMode.noTime.resumeExecution === true) { - botConfig.hasTheBotJustStarted = false; - } else { - botConfig.hasTheBotJustStarted = true; - } + let month = pad((new Date()).getUTCMonth() + 1, 2); + let year = (new Date()).getUTCFullYear(); - switch (botConfig.type) { - case 'Sensor': { - runSensorBot(botConfig, processConfig, month, year); - break; - } - case 'Indicator': { - runIndicatorBot(botConfig, processConfig, month, year); - break; - } - case 'Trading-Engine': { - runTradingEngine(botConfig, processConfig); - break; - } - default: { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); + switch (botConfig.type) { + case 'Sensor': { + runSensorBot(botConfig, processConfig, month, year); + break; + } + default: { + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); + } } } } - } - if (processConfig.startMode.fixedInterval !== undefined) { + if (processConfig.startMode.userDefined !== undefined) { - if (processConfig.startMode.fixedInterval.run === "true") { + if (processConfig.startMode.userDefined.run === "true") { - botConfig.runAtFixedInterval = true; - botConfig.fixedInterval = processConfig.startMode.fixedInterval.interval; + botConfig.startMode = "User Defined"; + console.log(logDisplace + "Root : [INFO] start -> findProcess -> Process found at the bot configuration file. -> Start Mode = " + botConfig.startMode); - let month = pad((new Date()).getUTCMonth() + 1, 2); - let year = (new Date()).getUTCFullYear(); + let month = pad((new Date()).getUTCMonth() + 1, 2); + let year = (new Date()).getUTCFullYear(); - switch (botConfig.type) { - case 'Sensor': { - runSensorBot(botConfig, processConfig, month, year); - break; + if (processConfig.startMode.userDefined.resumeExecution === true) { + botConfig.hasTheBotJustStarted = false; + } else { + botConfig.hasTheBotJustStarted = true; } - default: { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); + + switch (botConfig.type) { + case 'Trading-Engine': { + runTradingEngine(botConfig, processConfig); + break; + } + default: { + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); + } } } } + + } catch (err) { + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected exception. -> err.message = " + err.message); } - - if (processConfig.startMode.userDefined !== undefined) { - - if (processConfig.startMode.userDefined.run === "true") { - - botConfig.startMode = "User Defined"; - console.log(logDisplace + "Root : [INFO] start -> findProcess -> Process found at the bot configuration file. -> Start Mode = " + botConfig.startMode); - - let month = pad((new Date()).getUTCMonth() + 1, 2); - let year = (new Date()).getUTCFullYear(); - - if (processConfig.startMode.userDefined.resumeExecution === true) { - botConfig.hasTheBotJustStarted = false; - } else { - botConfig.hasTheBotJustStarted = true; - } - - switch (botConfig.type) { - case 'Trading-Engine': { - runTradingEngine(botConfig, processConfig); - break; - } - default: { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected bot type. -> botConfig.type = " + botConfig.type); - } - } - } - } - - } catch (err) { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Unexpected exception. -> err.message = " + err.message); } } - } - if (processFound === false) { + if (processFound === false) { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Process listed at the configuration file of Task Server not found at the configuration file of the bot."); - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> cloneToExecute.process = " + cloneToExecute.process); + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> Process listed at the configuration file of Task Server not found at the configuration file of the bot."); + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> cloneToExecute.process = " + cloneToExecute.process); - } + } - function runSensorBot(pBotConfig, pProcessConfig, pMonth, pYear) { + function runSensorBot(pBotConfig, pProcessConfig, pMonth, pYear) { + + try { + global.TOTAL_PROCESS_INSTANCES_CREATED++ + + const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); + let logger; + + logger = DEBUG_MODULE.newDebugLog(); + logger.bot = pBotConfig; + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runSensorBot -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runSensorBot -> pMonth = " + pMonth); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runSensorBot -> pYear = " + pYear); } + + let extractionBotMainLoop = SENSOR_BOT_MAIN_LOOP_MODULE.newSensorBotProcessMainLoop(pBotConfig, logger); + extractionBotMainLoop.initialize(UI_COMMANDS, pProcessConfig, onInitializeReady); + + function onInitializeReady(err) { + + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + + extractionBotMainLoop.run(pMonth, pYear, whenRunFinishes); + + function whenRunFinishes(err) { + + pBotConfig.loopCounter = 0; + + let botId; + if (pYear !== undefined) { + botId = pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process + "." + pYear + "." + pMonth; + } else { + botId = pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process; + } + + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); + logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); + + console.log(logDisplace + "Root : [INFO] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> botId = " + botId); + console.log(logDisplace + "Root : [INFO] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); + logger.persist(); + + } else { + + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Execution will be stopped. "); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bye."); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); + + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> botId = " + botId); + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot execution was aborted."); + logger.persist(); + } + exitProcessInstance() + } + + } else { + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> Bot will not be started. "); + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> err = " + err.stack); + + logger.persist(); + exitProcessInstance() + } + } + } + catch (err) { + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> err = " + err.stack); + exitProcessInstance() + } + } + + function runIndicatorBot(pBotConfig, pProcessConfig, pMonth, pYear) { + + try { + global.TOTAL_PROCESS_INSTANCES_CREATED++ + + const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); + let logger; + + logger = DEBUG_MODULE.newDebugLog(); + logger.bot = pBotConfig; + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runIndicatorBot -> Entering function."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runIndicatorBot -> pMonth = " + pMonth); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runIndicatorBot -> pYear = " + pYear); } + + let indicatorBotMainLoop = INDICATOR_BOT_MAIN_LOOP_MODULE.newIndicatorBotProcessMainLoop(pBotConfig, logger); + indicatorBotMainLoop.initialize(UI_COMMANDS, pProcessConfig, onInitializeReady); + + function onInitializeReady(err) { + + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + + indicatorBotMainLoop.run(pMonth, pYear, whenRunFinishes); + + function whenRunFinishes(err) { + + pBotConfig.loopCounter = 0; + + let botId; + if (pYear !== undefined) { + botId = pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process + "." + pYear + "." + pMonth; + } else { + botId = pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process; + } + + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); + logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); + + console.log(logDisplace + "Root : [INFO] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> botId = " + botId); + console.log(logDisplace + "Root : [INFO] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); + logger.persist(); + + } else { + + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Execution will be stopped. "); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bye."); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); + logger.persist(); + } + exitProcessInstance() + } + + } else { + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> Failed to initialize the bot. "); + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> err = " + err.stack); + logger.persist(); + exitProcessInstance() + } + } + } + catch (err) { + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> err = " + err.stack); + exitProcessInstance() + } + } + + function runTradingEngine(pBotConfig, pProcessConfig) { - try { global.TOTAL_PROCESS_INSTANCES_CREATED++ - const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); - let logger; + try { + const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); + let logger; - logger = DEBUG_MODULE.newDebugLog(); - logger.bot = pBotConfig; + logger = DEBUG_MODULE.newDebugLog(); + logger.bot = pBotConfig; - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runSensorBot -> Entering function."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runSensorBot -> pMonth = " + pMonth); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runSensorBot -> pYear = " + pYear); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runTradingEngine -> Entering function."); } - let extractionBotMainLoop = EXTRACTION_BOT_MAIN_LOOP_MODULE.newSensorBotProcessMainLoop(pBotConfig, logger); - extractionBotMainLoop.initialize(UI_COMMANDS, pProcessConfig, onInitializeReady); + let tradingEngineMainLoop = TRADING_ENGINE_MAIN_LOOP_MODULE.newTradingEngineProcessMainLoop(pBotConfig, logger); + tradingEngineMainLoop.initialize(UI_COMMANDS, pProcessConfig, onInitializeReady); - function onInitializeReady(err) { + function onInitializeReady(err) { - if (err.result === global.DEFAULT_OK_RESPONSE.result) { + if (err.result === global.DEFAULT_OK_RESPONSE.result) { - extractionBotMainLoop.run(pMonth, pYear, whenRunFinishes); + tradingEngineMainLoop.run(whenRunFinishes); - function whenRunFinishes(err) { + function whenRunFinishes(err) { - pBotConfig.loopCounter = 0; + pBotConfig.loopCounter = 0; + + let botId; - let botId; - if (pYear !== undefined) { - botId = pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process + "." + pYear + "." + pMonth; - } else { botId = pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process; + + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); + logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); + + console.log(logDisplace + "Root : [INFO] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> botId = " + botId); + console.log(logDisplace + "Root : [INFO] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); + logger.persist(); + + } else { + + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Execution will be stopped. "); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bye."); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); + logger.persist(); + } + exitProcessInstance() } - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); - logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); - - console.log(logDisplace + "Root : [INFO] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> botId = " + botId); - console.log(logDisplace + "Root : [INFO] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); - logger.persist(); - - } else { - - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> err = "+ err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Execution will be stopped. "); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bye."); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); - - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> botId = " + botId); - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot execution was aborted."); - logger.persist(); - } + } else { + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> Failed to initialize the bot. "); + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> err = " + err.stack); + logger.persist(); exitProcessInstance() } - - } else { - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> err = "+ err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> Bot will not be started. "); - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> err = "+ err.stack); - - logger.persist(); - exitProcessInstance() } } - } - catch (err) { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> err = " + err.stack); - exitProcessInstance() - } - } - - function runIndicatorBot(pBotConfig, pProcessConfig, pMonth, pYear) { - - try { - global.TOTAL_PROCESS_INSTANCES_CREATED++ - - const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); - let logger; - - logger = DEBUG_MODULE.newDebugLog(); - logger.bot = pBotConfig; - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runIndicatorBot -> Entering function."); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runIndicatorBot -> pMonth = " + pMonth); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runIndicatorBot -> pYear = " + pYear); } - - let indicatorBotMainLoop = INDICATOR_BOT_MAIN_LOOP_MODULE.newIndicatorBotProcessMainLoop(pBotConfig, logger); - indicatorBotMainLoop.initialize(UI_COMMANDS, pProcessConfig, onInitializeReady); - - function onInitializeReady(err) { - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - indicatorBotMainLoop.run(pMonth, pYear, whenRunFinishes); - - function whenRunFinishes(err) { - - pBotConfig.loopCounter = 0; - - let botId; - if (pYear !== undefined) { - botId = pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process + "." + pYear + "." + pMonth; - } else { - botId = pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process; - } - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); - logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); - - console.log(logDisplace + "Root : [INFO] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> botId = " + botId); - console.log(logDisplace + "Root : [INFO] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); - logger.persist(); - - } else { - - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> err = "+ err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Execution will be stopped. "); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bye."); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); - logger.persist(); - } - exitProcessInstance() - } - - } else { - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> err = "+ err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> Failed to initialize the bot. "); - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> err = "+ err.stack); - logger.persist(); - exitProcessInstance() - } + catch (err) { + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> err = " + err.stack); + exitProcessInstance() } } - catch (err) { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> err = "+ err.stack); - exitProcessInstance() - } } - - function runTradingEngine(pBotConfig, pProcessConfig) { - - global.TOTAL_PROCESS_INSTANCES_CREATED++ - - try { - const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); - let logger; - - logger = DEBUG_MODULE.newDebugLog(); - logger.bot = pBotConfig; - - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runTradingEngine -> Entering function."); } - - let tradingEngineMainLoop = TRADING_ENGINE_MAIN_LOOP_MODULE.newTradingEngineProcessMainLoop(pBotConfig, logger); - tradingEngineMainLoop.initialize(UI_COMMANDS, pProcessConfig, onInitializeReady); - - function onInitializeReady(err) { - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - tradingEngineMainLoop.run(whenRunFinishes); - - function whenRunFinishes(err) { - - pBotConfig.loopCounter = 0; - - let botId; - - botId = pBotConfig.devTeam + "." + pBotConfig.codeName + "." + pBotConfig.process; - - if (err.result === global.DEFAULT_OK_RESPONSE.result) { - - logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); - logger.write(MODULE_NAME, "[INFO] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); - - console.log(logDisplace + "Root : [INFO] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> botId = " + botId); - console.log(logDisplace + "Root : [INFO] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); - logger.persist(); - - } else { - - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> err = "+ err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Execution will be stopped. "); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bye."); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); - logger.persist(); - } - exitProcessInstance() - } - - } else { - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> err = "+ err.stack); - logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> Failed to initialize the bot. "); - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> err = "+ err.stack); - logger.persist(); - exitProcessInstance() - } - } - } - catch (err) { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> err = "+ err.stack); - exitProcessInstance() - } + catch (err) { + console.log(logDisplace + "Root : [ERROR] start -> findProcess -> err.message = " + err.message); + exitProcessInstance() } } - catch (err) { - console.log(logDisplace + "Root : [ERROR] start -> findProcess -> err.message = " + err.message); - exitProcessInstance() - } - } - function exitProcessInstance() { + function exitProcessInstance() { - global.ENDED_PROCESSES_COUNTER++ - console.log("[INFO] Task Server -> " + global.USER_DEFINITION.name + " -> exitProcessInstance -> Process #" + global.ENDED_PROCESSES_COUNTER + " from " + global.TOTAL_PROCESS_INSTANCES_CREATED + " exiting."); + global.ENDED_PROCESSES_COUNTER++ + console.log("[INFO] Task Server -> " + global.USER_DEFINITION.name + " -> exitProcessInstance -> Process #" + global.ENDED_PROCESSES_COUNTER + " from " + global.TOTAL_PROCESS_INSTANCES_CREATED + " exiting."); - if (global.ENDED_PROCESSES_COUNTER === global.TOTAL_PROCESS_INSTANCES_CREATED) { - global.EXIT_NODE_PROCESS() + if (global.ENDED_PROCESSES_COUNTER === global.TOTAL_PROCESS_INSTANCES_CREATED) { + global.EXIT_NODE_PROCESS() + } } + } catch (err) { + console.log("[ERROR] Task Server -> " + global.USER_DEFINITION.name + " -> Root -> Start -> Err = " + err.stack); } } } diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index d2e992e46..dda3a4a8f 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -632,11 +632,7 @@ function checkLoopHealth(pLastLoop) { - let stop = false - if (process.env.STOP_GRACEFULLY !== undefined) - stop = JSON.parse(process.env.STOP_GRACEFULLY) - - if (bot.enableCheckLoopHealth === false || stop === true) { + if (bot.enableCheckLoopHealth === false || global.STOP_TASK_GRACEFULLY === true) { logger.write(MODULE_NAME, "[WARN] run -> loop -> checkLoopHealth -> bot.enableCheckLoopHealth = " + bot.enableCheckLoopHealth); @@ -672,11 +668,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> shallWeStop -> Entering function. "); } - let stop = false - if (process.env.STOP_GRACEFULLY !== undefined) - stop = JSON.parse(process.env.STOP_GRACEFULLY) - - if (!stop) { + if (!global.STOP_TASK_GRACEFULLY) { continueCallBack(); } else { stopCallBack(); diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 0a9b7fa62..47b3b8c9d 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -94,7 +94,7 @@ bot.enableCheckLoopHealth = true; let fixedTimeLoopIntervalHandle; - let skipProcessing = true; + global.STOP_PROCESSING = true; if (bot.runAtFixedInterval === true) { @@ -112,12 +112,20 @@ bot.startMode = "Backtest" processConfig.framework.startDate.resumeExecution = false; - skipProcessing = false + global.STOP_PROCESSING = false bot.hasTheBotJustStarted = true - /* We exctact the initial and final date for the backtest from the Definition */ - processConfig.framework.startDate.fixedDate = new Date() // If we can not get for any reason the initial date of the backtest, we will start at present time. + /* If we can not get for any reason the initial date of the backtest, we will start at present time. */ + processConfig.framework.startDate.fixedDate = new Date() + /* If we received simulation params we use them instead. */ + if (global.DEFINITION.simulationParams) { + if (global.DEFINITION.simulationParams.beginDatetime) { + processConfig.framework.startDate.fixedDate = global.DEFINITION.simulationParams.beginDatetime + } + } + + /* We finally will try to exctact the initial and final date for the backtest from the Definition */ if (global.DEFINITION.tradingSystem !== undefined) { if (global.DEFINITION.tradingSystem.parameters !== undefined) { if (global.DEFINITION.tradingSystem.parameters.timeRange !== undefined) { @@ -146,13 +154,14 @@ bot.startMode = "Live" processConfig.framework.startDate.fixedDate = new Date() processConfig.framework.startDate.resumeExecution = false; - skipProcessing = false + global.STOP_PROCESSING = false bot.hasTheBotJustStarted = true } function stopRequested() { - skipProcessing = true + global.STOP_PROCESSING = true } + } loop(); @@ -203,7 +212,7 @@ + " Entered into Main Loop # " + pad(Number(bot.loopCounter), 8)); /* Checking if we should process this loop or not.*/ - if (skipProcessing === true) { + if (global.STOP_PROCESSING === true) { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> We are going to skip this Loop bacause we were requested to stop or never asked to start."); } console.log("[INFO] run -> loop -> We are going to skip this Loop bacause we were requested to stop or never asked to start.") nextWaitTime = 'Normal'; @@ -998,11 +1007,7 @@ function checkLoopHealth(pLastLoop) { - let stop = false - if (process.env.STOP_GRACEFULLY !== undefined) - stop = JSON.parse(process.env.STOP_GRACEFULLY) - - if (bot.enableCheckLoopHealth === false || stop === true) { + if (bot.enableCheckLoopHealth === false || global.STOP_TASK_GRACEFULLY === true) { logger.write(MODULE_NAME, "[WARN] run -> loop -> checkLoopHealth -> bot.enableCheckLoopHealth = " + bot.enableCheckLoopHealth); @@ -1038,11 +1043,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> shallWeStop -> Entering function. "); } - let stop = false - if (process.env.STOP_GRACEFULLY !== undefined) - stop = JSON.parse(process.env.STOP_GRACEFULLY) - - if (!stop) { + if (!global.STOP_TASK_GRACEFULLY) { continueCallBack(); } else { stopCallBack(); diff --git a/server.js b/server.js index 5f55aea66..803d3e70d 100644 --- a/server.js +++ b/server.js @@ -98,7 +98,8 @@ global.TOTAL_PROCESS_INSTANCES_CREATED = 0 process.on('message', message => { if (message === 'Stop this Task') { - process.env.STOP_GRACEFULLY = true; + global.STOP_TASK_GRACEFULLY = true; + global.STOP_PROCESSING = true /* There are some process that might no be able to end grafully, for example the ones schedulle to process information in a future day or month. @@ -183,19 +184,13 @@ function startSequence() { for (let processIndex = 0; processIndex < global.USER_DEFINITION.bot.processes.length; processIndex++) { let execution = sequenceList[processIndex]; - process.env.STOP_GRACEFULLY = false; + global.STOP_TASK_GRACEFULLY = false; execution.devTeam ? process.env.DEV_TEAM = execution.devTeam : undefined; execution.bot ? process.env.BOT = execution.bot : undefined; execution.resumeExecution = true; execution.type ? process.env.TYPE = execution.type : undefined; execution.process ? process.env.PROCESS = execution.process : undefined; - execution.startYear ? process.env.MIN_YEAR = execution.startYear : undefined; - execution.endYear ? process.env.MAX_YEAR = execution.endYear : undefined; - execution.month ? process.env.MONTH = execution.month : undefined; - execution.beginDatetime ? process.env.BEGIN_DATE_TIME = execution.beginDatetime : undefined; - execution.endDatetime ? process.env.END_DATE_TIME = execution.endDatetime : undefined; - execution.dataSet ? process.env.DATA_SET = execution.dataSet : undefined; - execution.timePeriod ? process.env.TIME_PERIOD = execution.timePeriod : undefined; + execution.baseAsset ? process.env.BASE_ASSET = execution.baseAsset : undefined; execution.balanceAssetA ? process.env.INITIAL_BALANCE_ASSET_A = execution.balanceAssetA : undefined; execution.balanceAssetB ? process.env.INITIAL_BALANCE_ASSET_B = execution.balanceAssetB : undefined; @@ -230,7 +225,6 @@ async function readExecutionConfiguration(execution, processIndex) { try { console.log("[INFO] Task Server -> server -> readExecutionConfiguration -> Entering function. "); - let timePeriodFilter let botProcess if (execution.type === 'Trading-Engine') { @@ -241,23 +235,9 @@ async function readExecutionConfiguration(execution, processIndex) { } if (global.DEFINITION !== undefined) { - if (global.DEFINITION.simulationParams !== undefined) { - - /* Here we only look for one timePeriod, in the future we will be able to process the whole array, but not for now. */ - if (global.DEFINITION.simulationParams.timePeriodDailyArray !== undefined) { - if (global.DEFINITION.simulationParams.timePeriodDailyArray.length === 1) { - timePeriodFilter = global.DEFINITION.simulationParams.timePeriodDailyArray[0] - botProcess = "Multi-Period" - } - } - if (global.DEFINITION.simulationParams.timePeriodMarketArray !== undefined) { - if (global.DEFINITION.simulationParams.timePeriodMarketArray.length === 1) { - timePeriodFilter = global.DEFINITION.simulationParams.timePeriodMarketArray[0] - botProcess = "Multi-Period" - } - } - } - + + botProcess = "Multi-Period" + /* Get the initial balance from the global.DEFINITION */ let tradingSystem = global.DEFINITION.tradingSystem @@ -312,24 +292,11 @@ async function readExecutionConfiguration(execution, processIndex) { process: botProcess, repo: global.CURRENT_BOT_REPO } - - let timePeriod - if (timePeriodFilter === undefined) { - timePeriod = process.env.TIME_PERIOD - } else { - timePeriod = timePeriodFilter - } - + global.EXECUTION_CONFIG = { - cloneToExecute: cloneToExecute, - timePeriod: getTimePeriod(timePeriod), - timePeriodFileStorage: timePeriod, - timePeriodFilter: timePeriodFilter, - dataSet: process.env.DATA_SET + cloneToExecute: cloneToExecute }; - timePeriodFilter = undefined - timePeriod = undefined global.CLONE_EXECUTOR = { codeName: 'AACloud', From 89fda8207455d69f4ad3b96f4fa191a62991402f Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 12 Sep 2019 13:02:20 +0200 Subject: [PATCH 31/32] Receiving the definition via events --- TradingEngineProcessMainLoop.js | 92 +++++++++++++++------------------ 1 file changed, 43 insertions(+), 49 deletions(-) diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 47b3b8c9d..69fc465e5 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -96,72 +96,66 @@ let fixedTimeLoopIntervalHandle; global.STOP_PROCESSING = true; - if (bot.runAtFixedInterval === true) { + global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Backstesting Started', undefined, undefined, undefined, startBackTesting) + global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Live-Trading Started', undefined, undefined, undefined, startLiveTrading) + global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Stop Requested', undefined, undefined, undefined, stopRequested) - fixedTimeLoopIntervalHandle = setInterval(loop, bot.fixedInterval); + function startBackTesting(message) { - } else { + /* We are going to run the Definition comming at the event. */ + global.DEFINITION = JSON.parse(message.event.definition) - backtestingEventListenerSubscriptionId = global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Backstesting Started', undefined, undefined, undefined, startBackTesting) - liveTradingEventListenerSubscriptionId = global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Live-Trading Started', undefined, undefined, undefined, startLiveTrading) - stopRequestedEventListenerSubscriptionId = global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Stop Requested', undefined, undefined, undefined, stopRequested) + bot.startMode = "Backtest" + processConfig.framework.startDate.resumeExecution = false; + global.STOP_PROCESSING = false + bot.hasTheBotJustStarted = true - function startBackTesting() { - /* Reload the definition in case something changed. */ - global.DEFINITION = require(process.env.INTER_PROCESS_FILES_PATH + '/Definition'); + /* If we can not get for any reason the initial date of the backtest, we will start at present time. */ + processConfig.framework.startDate.fixedDate = new Date() - bot.startMode = "Backtest" - processConfig.framework.startDate.resumeExecution = false; - global.STOP_PROCESSING = false - bot.hasTheBotJustStarted = true - - /* If we can not get for any reason the initial date of the backtest, we will start at present time. */ - processConfig.framework.startDate.fixedDate = new Date() - - /* If we received simulation params we use them instead. */ - if (global.DEFINITION.simulationParams) { - if (global.DEFINITION.simulationParams.beginDatetime) { - processConfig.framework.startDate.fixedDate = global.DEFINITION.simulationParams.beginDatetime - } + /* If we received simulation params we use them instead. */ + if (global.DEFINITION.simulationParams) { + if (global.DEFINITION.simulationParams.beginDatetime) { + processConfig.framework.startDate.fixedDate = global.DEFINITION.simulationParams.beginDatetime } + } - /* We finally will try to exctact the initial and final date for the backtest from the Definition */ - if (global.DEFINITION.tradingSystem !== undefined) { - if (global.DEFINITION.tradingSystem.parameters !== undefined) { - if (global.DEFINITION.tradingSystem.parameters.timeRange !== undefined) { - if (global.DEFINITION.tradingSystem.parameters.timeRange.code !== undefined) { - try { - let code = JSON.parse(global.DEFINITION.tradingSystem.parameters.timeRange.code) - if (code.initialDatetime !== undefined) { - processConfig.framework.startDate.fixedDate = code.initialDatetime /* The second override occurs here, with the date explicitelly defined by the user */ - } - if (code.finalDatetime !== undefined) { - processConfig.framework.endDate.fixedDate = code.finalDatetime - } - } catch (err) { - global.DEFINITION.tradingSystem.parameters.timeRange.error = err.message + /* We finally will try to exctact the initial and final date for the backtest from the Definition */ + if (global.DEFINITION.tradingSystem !== undefined) { + if (global.DEFINITION.tradingSystem.parameters !== undefined) { + if (global.DEFINITION.tradingSystem.parameters.timeRange !== undefined) { + if (global.DEFINITION.tradingSystem.parameters.timeRange.code !== undefined) { + try { + let code = JSON.parse(global.DEFINITION.tradingSystem.parameters.timeRange.code) + if (code.initialDatetime !== undefined) { + processConfig.framework.startDate.fixedDate = code.initialDatetime /* The second override occurs here, with the date explicitelly defined by the user */ } + if (code.finalDatetime !== undefined) { + processConfig.framework.endDate.fixedDate = code.finalDatetime + } + } catch (err) { + global.DEFINITION.tradingSystem.parameters.timeRange.error = err.message } } } } } + } - function startLiveTrading() { - /* Reload the definition in case something changed. */ - global.DEFINITION = require(process.env.INTER_PROCESS_FILES_PATH + '/Definition'); + function startLiveTrading(message) { - bot.startMode = "Live" - processConfig.framework.startDate.fixedDate = new Date() - processConfig.framework.startDate.resumeExecution = false; - global.STOP_PROCESSING = false - bot.hasTheBotJustStarted = true - } + /* We are going to run the Definition comming at the event. */ + global.DEFINITION = JSON.parse(message.event.definition) - function stopRequested() { - global.STOP_PROCESSING = true - } + bot.startMode = "Live" + processConfig.framework.startDate.fixedDate = new Date() + processConfig.framework.startDate.resumeExecution = false; + global.STOP_PROCESSING = false + bot.hasTheBotJustStarted = true + } + function stopRequested() { + global.STOP_PROCESSING = true } loop(); From 3c32f9baf3d70abdbcfef37762b5ae7fd304c21e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 12 Sep 2019 20:02:16 +0200 Subject: [PATCH 32/32] Having the Client of Event Handler here --- SystemEventHandler.js | 178 ++++++++++++++++++++++++++++++++++++++++++ server.js | 5 +- 2 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 SystemEventHandler.js diff --git a/SystemEventHandler.js b/SystemEventHandler.js new file mode 100644 index 000000000..7620ca72e --- /dev/null +++ b/SystemEventHandler.js @@ -0,0 +1,178 @@ +exports.newSystemEventHandler = function newSystemEventHandler(ipc) { + let thisObject = { + initialize: initialize, + finalize: finalize, + createEventHandler: createEventHandler, + deleteEventHandler: deleteEventHandler, + listenToEvent: listenToEvent, + stopListening: stopListening, + raiseEvent: raiseEvent + } + + let eventListeners = new Map() + let responseWaiters = new Map() + + ipc.config.id = 'Task Server'; + ipc.config.retry = 1500; + ipc.config.silent = true; + + return thisObject + + function initialize(canllerName, callBackFunction) { + + ipc.connectTo( + 'world', + function () { + ipc.of.world.on( + 'connect', + function () { + ipc.log('## Connected to Superalgos Event Server ##'.rainbow, ipc.config.delay); + + let eventCommand = { + action: 'greetings', + from: canllerName + } + + sendCommand(eventCommand) + + callBackFunction() + } + ); + ipc.of.world.on( + 'disconnect', + function () { + ipc.log('Disconnected from Superalgos Event Server '.notice); + } + ); + ipc.of.world.on( + 'message', // here is where all incomming messages are processed. + function (data) { + ipc.log('Got a message from Superalgos Event Server: '.debug, data); + + let message = JSON.parse(data) + + if (message.action === 'Event Raised') { + let key + if (message.callerId) { + key = message.eventHandlerName + '-' + message.eventType + '-' + message.callerId + } else { + key = message.eventHandlerName + '-' + message.eventType + } + let handler = eventListeners.get(key) + if (handler) { + handler.callBack(message) + } else { + console.log(key + ' not found so could not deliver event raised.') + console.log(' Message = ' + data) + } + return + } + + if (message.action === 'Event Server Response') { + let handler = responseWaiters.get(message.callerId) + if (handler) { + handler(message) + } + return + } + } + ); + } + ); + } + + function finalize() { + /* Before disconnecting we will forze all eventListeners to stop listening. */ + const eventListenersArray = [...eventListeners.values()] + for (let i = 0; i < eventListenersArray.length; i++) { + let handler = eventListenersArray[i] + let eventCommand = { + action: 'stopListening', + eventHandlerName: handler.eventHandlerName, + eventType: handler.eventType, + callerId: handler.callerId + } + sendCommand(eventCommand) + } + ipc.disconnect('world'); + } + + function sendCommand(command, responseCallBack, eventsCallBack) { + + if (command.action === 'listenToEvent') { + let key + if (command.callerId) { + key = command.eventHandlerName + '-' + command.eventType + '-' + command.callerId + } else { + key = command.eventHandlerName + '-' + command.eventType + } + let handler = { + eventHandlerName: command.eventHandlerName, + eventType: command.eventType, + callerId: command.callerId, + callBack: eventsCallBack + } + eventListeners.set(key, handler) + } + if (command.callerId && responseCallBack) { + responseWaiters.set(command.callerId, responseCallBack) + } + + ipc.of.world.emit( + 'message', + JSON.stringify(command) + ) + } + + function createEventHandler(eventHandlerName, callerId, responseCallBack) { + let eventCommand = { + action: 'createEventHandler', + eventHandlerName: eventHandlerName, + callerId: callerId + } + sendCommand(eventCommand, responseCallBack) + } + + function deleteEventHandler(eventHandlerName, callerId, responseCallBack) { + let eventCommand = { + action: 'deleteEventHandler', + eventHandlerName: eventHandlerName, + callerId: callerId + } + sendCommand(eventCommand, responseCallBack) + } + + function listenToEvent(eventHandlerName, eventType, extraData, callerId, responseCallBack, eventsCallBack) { + let eventCommand = { + action: 'listenToEvent', + eventHandlerName: eventHandlerName, + eventType: eventType, + extraData: extraData, + callerId: callerId + } + sendCommand(eventCommand, responseCallBack, eventsCallBack) + } + + function stopListening(eventHandlerName, eventType, eventSubscriptionId, callerId, responseCallBack) { + /* User either needs to specify a valid eventSubscriptionId OR the 3 params: eventHandlerName, eventType, callerId which were used when start listening to events. */ + let eventCommand = { + action: 'stopListening', + eventHandlerName: eventHandlerName, + eventType: eventType, + eventSubscriptionId: eventSubscriptionId, + callerId: callerId + } + sendCommand(eventCommand, responseCallBack) + } + + function raiseEvent(eventHandlerName, eventType, event, callerId, responseCallBack) { + let eventCommand = { + action: 'raiseEvent', + eventHandlerName: eventHandlerName, + eventType: eventType, + event: event, + callerId: callerId + } + sendCommand(eventCommand, responseCallBack) + } +} \ No newline at end of file diff --git a/server.js b/server.js index 803d3e70d..ac33d8bd6 100644 --- a/server.js +++ b/server.js @@ -139,8 +139,9 @@ let notFirstSequence = false; /* Setting up the global Event Handler */ -const EVENT_HANDLER_MODULE = require('../Libraries/SystemEventsClient/SystemEventHandler.js'); -global.SYSTEM_EVENT_HANDLER = EVENT_HANDLER_MODULE.newSystemEventHandler() +const EVENT_HANDLER_MODULE = require('./SystemEventHandler.js'); +const IPC = require('node-ipc'); +global.SYSTEM_EVENT_HANDLER = EVENT_HANDLER_MODULE.newSystemEventHandler(IPC) global.SYSTEM_EVENT_HANDLER.initialize('Task Server', bootLoader)