diff --git a/Commons.js b/Commons.js index 050791f24..7e32f50cb 100644 --- a/Commons.js +++ b/Commons.js @@ -99,6 +99,9 @@ if (botNode === undefined) { botNode = global.FIND_NODE_IN_NODE_MESH(outputDatasetNode, 'Trading Bot') } + if (botNode === undefined) { + botNode = global.FIND_NODE_IN_NODE_MESH(outputDatasetNode, 'Learning Bot') + } if (botNode === undefined) { logger.write(MODULE_NAME, "[ERROR] start -> Product Definition not attached to a Bot. Product Definition = "); callBackFunction(global.DEFAULT_FAIL_RESPONSE); @@ -115,9 +118,18 @@ if (dataMineNode === undefined) { let tradingMineNode = global.FIND_NODE_IN_NODE_MESH(outputDatasetNode, 'Trading Mine') if (tradingMineNode === undefined) { - logger.write(MODULE_NAME, "[ERROR] start -> Bot not attached to a Data Mine or a Trading Mine."); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return + let learningMineNode = global.FIND_NODE_IN_NODE_MESH(outputDatasetNode, 'Learning Mine') + if (learningMineNode === undefined) { + logger.write(MODULE_NAME, "[ERROR] start -> Bot not attached to a Data Mine, Trading Mine or Learning Mine."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return + } else { + if (tradingMineNode.config.codeName === undefined) { + logger.write(MODULE_NAME, "[ERROR] start -> Learning Mine witn no codeName defined."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return + } + } } else { if (tradingMineNode.config.codeName === undefined) { logger.write(MODULE_NAME, "[ERROR] start -> Trading Mine witn no codeName defined."); @@ -380,12 +392,6 @@ let product = {} product[variableName] = {} - /* Here is how we know if we are processing Yesterday. */ - let positionedAtYesterday = false - if (processingDailyFiles) { - positionedAtYesterday = (record.current.end < currentDay.valueOf()) - } - /* This is Loop Code */ try { eval(dataBuildingProcedure.loop.javascriptCode.code) @@ -423,9 +429,9 @@ results.push(product[variableName]); } - /* While we are positioned at Yesterday, we keep updating this data structure. */ + /* At the last instant of yesterday, we update the yesterday object. */ if (processingDailyFiles) { - if (positionedAtYesterday) { + if (record.current.end + 1 === currentDay.valueOf()) { yesterday.variable = JSON.parse(JSON.stringify(variable)) } } diff --git a/DataDependencies.js b/DataDependencies.js index 54eb5916d..bc6cbf79e 100644 --- a/DataDependencies.js +++ b/DataDependencies.js @@ -36,7 +36,7 @@ thisObject.nodeArray = global.FILTER_OUT_NODES_WITHOUT_REFERENCE_PARENT_FROM_NODE_ARRAY(thisObject.nodeArray) } else { - logger.write(MODULE_NAME, "[ERROR] initialize -> onInitilized -> It is not possible to not have process dependencies, which means not data dependencies."); + logger.write(MODULE_NAME, "[ERROR] initialize -> onInitilized -> It is not possible not to have process dependencies, which means not data dependencies."); callBackFunction(global.DEFAULT_OK_RESPONSE) return } @@ -65,7 +65,7 @@ let skipCount = 0 /* - The current nodeArray that we have includes all the dependencies daclared in the Data Mine / Trading Mine + The current nodeArray that we have includes all the dependencies daclared in the Data Mine | Trading Mine | Learning Mine process dependencies, minus the ones without reference parent, meaning that references Data Mines that are not present at the workspace. From all the remaining dependencies there will be others that we need to filter out, and they are the ones which the user does not have a data product anywhere on the network where diff --git a/DataSet.js b/DataSet.js index 5b43b7bbf..4637e72b9 100644 --- a/DataSet.js +++ b/DataSet.js @@ -24,8 +24,8 @@ exports.newDataSet = function newDataSet(BOT, logger) { try { thisObject.node = dataDependency.referenceParent; - logger.fileName = MODULE_NAME + "." + thisObject.node.type + "." + thisObject.node.name + "." + thisObject.node.id; - + logger.fileName = MODULE_NAME + /* Some very basic validations that we have all the information needed. */ if (thisObject.node === undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> Data Dependency without Reference Parent -> dataDependency = " + JSON.stringify(dataDependency)); @@ -33,6 +33,8 @@ exports.newDataSet = function newDataSet(BOT, logger) { return } + logger.fileName = MODULE_NAME + "." + thisObject.node.type + "." + thisObject.node.name + "." + thisObject.node.id; + if (thisObject.node.config.codeName === undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> Dataset witn no codeName defined -> Product Dataset = " + JSON.stringify(thisObject.node)); callBackFunction(global.DEFAULT_FAIL_RESPONSE); diff --git a/DebugLog.js b/DebugLog.js index a1ef69a37..91b318fe7 100644 --- a/DebugLog.js +++ b/DebugLog.js @@ -80,10 +80,14 @@ exports.newDebugLog = function newDebugLog() { let filePath = thisObject.bot.filePathRoot + "/Logs/" + thisObject.bot.process + "/" - if (thisObject.bot.SESSION !== undefined) { - filePath = filePath + thisObject.bot.SESSION.folderName + "/" + executionDatetime; + if (thisObject.bot.TRADING_SESSION !== undefined) { + filePath = filePath + thisObject.bot.TRADING_SESSION.folderName + "/" + executionDatetime; } else { - filePath = filePath + executionDatetime; + if (thisObject.bot.LEARNING_SESSION !== undefined) { + filePath = filePath + thisObject.bot.LEARNING_SESSION.folderName + "/" + executionDatetime; + } else { + filePath = filePath + executionDatetime; + } } if (thisObject.bot.debug.year !== undefined) { diff --git a/EventServerClient.js b/EventServerClient.js index a5de99f5a..09bf453cb 100644 --- a/EventServerClient.js +++ b/EventServerClient.js @@ -1,7 +1,7 @@ exports.newEventsServerClient = function newEventsServerClient(host, port) { const MODULE = "Task Server " - const ERROR_LOG = true + const INFO_LOG = false let thisObject = { @@ -27,6 +27,8 @@ port = process.env.WEB_SOCKETS_SERVER_PORT } + let messageCounter = 0 + let commandCounter = 0 return thisObject @@ -39,25 +41,32 @@ function setuptWebSockets(callBackFunction) { try { + if (INFO_LOG === true) { + console.log('setuptWebSockets at ' + host + ':' + port) + } WEB_SOCKETS_CLIENT = new WEB_SOCKET('ws://' + host + ':' + port ) - WEB_SOCKETS_CLIENT.onerror = error => { - console.log('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> On connection error -> error = ' + JSON.stringify(error)) + WEB_SOCKETS_CLIENT.onerror = err => { + console.log('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> On connection error -> error = ' + err.stack) + console.log('[ERROR] This could mean that the port '+ port +' is taken by some other app running at your system. To resolve this issue, please pick another port number and change it at the .ENV file inside the Superalgos folder AND at the .ENV file inside the TaskServer folder. After that run the app again. ') } WEB_SOCKETS_CLIENT.onopen = () => { - if (INFO_LOG === true) { - console.log('Websocket connection opened.') - } - - if (callBackFunction !== undefined) { - callBackFunction() + try { + if (INFO_LOG === true) { + console.log('Websocket connection opened.') + } + + if (callBackFunction !== undefined) { + callBackFunction() + } + } catch(err) { + console.log('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> onopen -> err = ' + err.stack) } } WEB_SOCKETS_CLIENT.onmessage = e => { - try { if (INFO_LOG === true) { - console.log('Websocket Message Received: ' + e.data) + console.log('Websocket Message Received: ' + e.data.substring(0, 1000)) } let message = JSON.parse(e.data) @@ -84,11 +93,11 @@ return } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> onmessage -> err = ' + err.stack) } + console.log('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> onmessage -> err = ' + err.stack) } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> err = ' + err.stack) } + console.log('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> err = ' + err.stack) } } @@ -109,30 +118,35 @@ } function sendCommand(command, responseCallBack, eventsCallBack) { - - if (command.action === 'listenToEvent') { - let key - if (command.callerId) { - key = command.eventHandlerName + '-' + command.eventType + '-' + command.callerId + try { + if (command.action === 'listenToEvent') { + let key + if (command.callerId) { + key = command.eventHandlerName + '-' + command.eventType + '-' + command.callerId + } else { + key = command.eventHandlerName + '-' + command.eventType + } + let handler = { + eventHandlerName: command.eventHandlerName, + eventType: command.eventType, + callerId: command.callerId, + callBack: eventsCallBack + } + eventListeners.set(key, handler) + } + if (command.callerId && responseCallBack) { + responseWaiters.set(command.callerId, responseCallBack) + } + + if (WEB_SOCKETS_CLIENT.readyState === 1) { // 1 means connected and ready. + + WEB_SOCKETS_CLIENT.send("Task Server" + "|*|" + "|*|" + JSON.stringify(command)) + } else { - key = command.eventHandlerName + '-' + command.eventType + console.log('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> sendCommand -> WebSocket message could not be sent because the connection was not ready. Message = ' + JSON.stringify(command)) } - let handler = { - eventHandlerName: command.eventHandlerName, - eventType: command.eventType, - callerId: command.callerId, - callBack: eventsCallBack - } - eventListeners.set(key, handler) - } - if (command.callerId && responseCallBack) { - responseWaiters.set(command.callerId, responseCallBack) - } - - if (WEB_SOCKETS_CLIENT.readyState === 1) { // 1 means connected and ready. - WEB_SOCKETS_CLIENT.send("Task Server" + "|*|" + "|*|" + JSON.stringify(command)) - } else { - console.log('[ERROR] Task Server -> Event Server Client -> setuptWebSockets -> sendCommand -> WebSocket message could not be sent because the connection was not ready. Message = ' + JSON.stringify(command)) + } catch(err) { + console.log('[ERROR] Task Server -> Event Server Client -> sendCommand -> err = ' + err.stack) } } diff --git a/IndicatorBot.js b/IndicatorBot.js index cd513dee4..2f55a1daa 100644 --- a/IndicatorBot.js +++ b/IndicatorBot.js @@ -61,7 +61,7 @@ function onCommonsDownloaded(err, text) { if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - parentLogger.write(MODULE_NAME, "[WARN] initialize -> onBotDownloaded -> onCommonsDownloaded -> Commons not found: " + err.code || err.message); + parentLogger.write(MODULE_NAME, "[WARN] initialize -> onBotDownloaded -> onCommonsDownloaded -> Commons not found: " + err.message); callBackFunction(global.DEFAULT_OK_RESPONSE); return; } diff --git a/IndicatorOutput.js b/IndicatorOutput.js index d7f258b6b..cc585a1db 100644 --- a/IndicatorOutput.js +++ b/IndicatorOutput.js @@ -40,8 +40,6 @@ try { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> Entering function."); } - let products = {} let mainDependency = {} let processingDailyFiles diff --git a/Session.js b/LearningSession.js similarity index 65% rename from Session.js rename to LearningSession.js index 9b8c82b67..6a7953b65 100644 --- a/Session.js +++ b/LearningSession.js @@ -1,6 +1,6 @@ -exports.newSession = function newSession(bot, parentLogger) { +exports.newLearningSession = function newLearningSession(bot, parentLogger) { - const MODULE_NAME = "Session" + const MODULE_NAME = "Learning Session" const FULL_LOG = true; let thisObject = { @@ -15,16 +15,16 @@ function initialize(pProcessConfig, callBackFunction) { try { /* Initialize this info so that everything is logged propeerly */ - bot.SESSION = { + bot.LEARNING_SESSION = { name: bot.processNode.session.name, id: bot.processNode.session.id } /* Set the folderName for early logging */ if (bot.processNode.session.config.folderName === undefined) { - bot.SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.SESSION.id + bot.LEARNING_SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.LEARNING_SESSION.id } else { - bot.SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.processNode.session.config.folderName + bot.LEARNING_SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.processNode.session.config.folderName } /* Check if there is a session */ @@ -38,10 +38,10 @@ bot.sessionKey = bot.processNode.session.name + '-' + bot.processNode.session.type + '-' + bot.processNode.session.id global.SESSION_MAP.set(bot.sessionKey, bot.sessionKey) - global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Session Status', undefined, bot.sessionKey, undefined, onSessionStatus) - global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Run Session', undefined, bot.sessionKey, undefined, onSessionRun) - global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Stop Session', undefined, bot.sessionKey, undefined, onSessionStop) - global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Resume Session', undefined, bot.sessionKey, undefined, onSessionResume) + global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Learning Session Status', undefined, bot.sessionKey, undefined, onSessionStatus) + global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Run Learning Session', undefined, bot.sessionKey, undefined, onSessionRun) + global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Stop Learning Session', undefined, bot.sessionKey, undefined, onSessionStop) + global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Resume Learning Session', undefined, bot.sessionKey, undefined, onSessionResume) /* Connect this here so that it is accesible from other places */ bot.sessionError = sessionError @@ -57,12 +57,12 @@ function onSessionStatus() { if (bot.SESSION_STATUS === 'Running') { let event = { - status: 'Session Runnning' + status: 'Learning Session Runnning' } global.EVENT_SERVER_CLIENT.raiseEvent(bot.sessionKey, 'Status Response', event) } else { let event = { - status: 'Not Session Runnning' + status: 'Learning Session Not Runnning' } global.EVENT_SERVER_CLIENT.raiseEvent(bot.sessionKey, 'Status Response', event) } @@ -79,11 +79,11 @@ /* We are going to run the Definition comming at the event. */ bot.TRADING_SYSTEM = JSON.parse(message.event.tradingSystem) bot.TRADING_ENGINE = JSON.parse(message.event.tradingEngine) - bot.SESSION = JSON.parse(message.event.session) + bot.LEARNING_SESSION = JSON.parse(message.event.session) bot.DEPENDENCY_FILTER = JSON.parse(message.event.dependencyFilter) bot.RESUME = false bot.FIRST_EXECUTION = true - bot.SESSION.stop = stopSession // stop function + bot.LEARNING_SESSION.stop = stopSession // stop function setUpSessionFolderName() @@ -93,7 +93,7 @@ socialBotsModule.initialize() let allGood - switch (bot.SESSION.type) { + switch (bot.LEARNING_SESSION.type) { case 'Backtesting Session': { allGood = startBackTesting(message) break @@ -119,7 +119,7 @@ if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, '[IMPORTANT] onSessionRun -> Stopping the Session now. ') } } - socialBotsModule.sendMessage(bot.SESSION.type + " '" + bot.SESSION.name + "' is starting.") + socialBotsModule.sendMessage(bot.LEARNING_SESSION.type + " '" + bot.LEARNING_SESSION.name + "' is starting.") } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onSessionRun -> err = " + err.stack); } @@ -131,7 +131,7 @@ function onSessionResume(message) { try { - if (bot.SESSION.stop === undefined) { + if (bot.LEARNING_SESSION.stop === undefined) { parentLogger.write(MODULE_NAME, "[WARN] onSessionResume -> Event received to resume the Session that have never be ran before. ") return } @@ -145,7 +145,7 @@ bot.RESUME = true bot.STOP_SESSION = false - socialBotsModule.sendMessage(bot.SESSION.type + " '" + bot.SESSION.name + "' is resuming.") + socialBotsModule.sendMessage(bot.LEARNING_SESSION.type + " '" + bot.LEARNING_SESSION.name + "' is resuming.") } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onSessionResume -> err = " + err.stack); } @@ -153,11 +153,11 @@ function stopSession(commandOrigin) { - socialBotsModule.sendMessage(bot.SESSION.type + " '" + bot.SESSION.name + "' is stopping " + commandOrigin) + socialBotsModule.sendMessage(bot.LEARNING_SESSION.type + " '" + bot.LEARNING_SESSION.name + "' is stopping " + commandOrigin) socialBotsModule.finalize() bot.STOP_SESSION = true if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, '[IMPORTANT] stopSession -> Stopping the Session now. ') } - sessionInfo(bot.SESSION, commandOrigin) + sessionInfo(bot.LEARNING_SESSION, commandOrigin) } function setUpSessionFolderName() { @@ -167,12 +167,12 @@ Set the folderName for logging, reports, context and data output */ let config - if (bot.SESSION.config !== undefined) { - config = bot.SESSION.config + if (bot.LEARNING_SESSION.config !== undefined) { + config = bot.LEARNING_SESSION.config if (config.folderName === undefined) { - bot.SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.SESSION.id + bot.LEARNING_SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.LEARNING_SESSION.id } else { - bot.SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.processNode.session.config.folderName + bot.LEARNING_SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.processNode.session.config.folderName } } } @@ -183,16 +183,16 @@ non critical is missing, we complete it with a default value. */ - if (bot.SESSION.parameters === undefined) { + if (bot.LEARNING_SESSION.learningParameters === undefined) { let errorMessage = "Session Node with no Parameters." parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) - bot.sessionError(bot.SESSION, errorMessage) + bot.sessionError(bot.LEARNING_SESSION, errorMessage) return false } /* Time Range */ - if (bot.SESSION.parameters.timeRange === undefined) { // if the Time Range is missing we create a default one. - bot.SESSION.parameters.timeRange = { + if (bot.LEARNING_SESSION.learningParameters.timeRange === undefined) { // if the Time Range is missing we create a default one. + bot.LEARNING_SESSION.learningParameters.timeRange = { name: 'Missing Time Range', type: 'Time Range', config: { @@ -202,18 +202,18 @@ } } else { /* Check that we received valid dates */ - if (bot.SESSION.type === 'Backtesting Session') { - if (isNaN(new Date(bot.SESSION.parameters.timeRange.config.initialDatetime)).valueOf()) { + if (bot.LEARNING_SESSION.type === 'Backtesting Session') { + if (isNaN(new Date(bot.LEARNING_SESSION.learningParameters.timeRange.config.initialDatetime)).valueOf()) { let errorMessage = "sessionParameters.timeRange.config.initialDatetime is not a valid date." parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) - bot.sessionError(bot.SESSION.parameters, errorMessage) + bot.sessionError(bot.LEARNING_SESSION.learningParameters, errorMessage) return false } } - if (isNaN(new Date(bot.SESSION.parameters.timeRange.config.finalDatetime)).valueOf()) { + if (isNaN(new Date(bot.LEARNING_SESSION.learningParameters.timeRange.config.finalDatetime)).valueOf()) { let errorMessage = "sessionParameters.timeRange.config.initialDatetime is not a valid date." parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) - bot.sessionError(bot.SESSION.parameters, errorMessage) + bot.sessionError(bot.LEARNING_SESSION.learningParameters, errorMessage) return false } } @@ -222,7 +222,7 @@ let today = (new Date()).valueOf() let aYearAgo = today - global.ONE_YEAR_IN_MILISECONDS let aYearFromNow = today + global.ONE_YEAR_IN_MILISECONDS - switch (bot.SESSION.type) { + switch (bot.LEARNING_SESSION.type) { case 'Backtesting Session': { useDefaultDatetimes(aYearAgo, today) break @@ -248,103 +248,103 @@ */ /* Initial Datetime */ - if (bot.SESSION.type === 'Backtesting Session') { - if (bot.SESSION.parameters.timeRange.config.initialDatetime === undefined) { - bot.SESSION.parameters.timeRange.config.initialDatetime = initialDefault + if (bot.LEARNING_SESSION.type === 'Backtesting Session') { + if (bot.LEARNING_SESSION.learningParameters.timeRange.config.initialDatetime === undefined) { + bot.LEARNING_SESSION.learningParameters.timeRange.config.initialDatetime = initialDefault } else { - bot.SESSION.parameters.timeRange.config.initialDatetime = (new Date(bot.SESSION.parameters.timeRange.config.initialDatetime)).valueOf() + bot.LEARNING_SESSION.learningParameters.timeRange.config.initialDatetime = (new Date(bot.LEARNING_SESSION.learningParameters.timeRange.config.initialDatetime)).valueOf() } } else { /* Non backtest session can start from the past only if explicitly configured that way */ - if (bot.SESSION.parameters.timeRange.config.initialDatetime === undefined || bot.SESSION.parameters.timeRange.config.allowStartingFromThePast !== true) { - bot.SESSION.parameters.timeRange.config.initialDatetime = initialDefault + if (bot.LEARNING_SESSION.learningParameters.timeRange.config.initialDatetime === undefined || bot.LEARNING_SESSION.learningParameters.timeRange.config.allowStartingFromThePast !== true) { + bot.LEARNING_SESSION.learningParameters.timeRange.config.initialDatetime = initialDefault } else { - bot.SESSION.parameters.timeRange.config.initialDatetime = (new Date(bot.SESSION.parameters.timeRange.config.initialDatetime)).valueOf() + bot.LEARNING_SESSION.learningParameters.timeRange.config.initialDatetime = (new Date(bot.LEARNING_SESSION.learningParameters.timeRange.config.initialDatetime)).valueOf() } } /* Final Datetime */ - if (bot.SESSION.parameters.timeRange.config.finalDatetime === undefined) { - bot.SESSION.parameters.timeRange.config.finalDatetime = finalDefault + if (bot.LEARNING_SESSION.learningParameters.timeRange.config.finalDatetime === undefined) { + bot.LEARNING_SESSION.learningParameters.timeRange.config.finalDatetime = finalDefault } else { - bot.SESSION.parameters.timeRange.config.finalDatetime = (new Date(bot.SESSION.parameters.timeRange.config.finalDatetime)).valueOf() + bot.LEARNING_SESSION.learningParameters.timeRange.config.finalDatetime = (new Date(bot.LEARNING_SESSION.learningParameters.timeRange.config.finalDatetime)).valueOf() } } /* Time Frame */ - if (bot.SESSION.parameters.timeFrame === undefined) { + if (bot.LEARNING_SESSION.learningParameters.timeFrame === undefined) { let errorMessage = "Session Parameters Node with no Time Frame." parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) - bot.sessionError(bot.SESSION.parameters, errorMessage) + bot.sessionError(bot.LEARNING_SESSION.learningParameters, errorMessage) return false } - if (bot.SESSION.parameters.timeFrame.config.label === undefined) { + if (bot.LEARNING_SESSION.learningParameters.timeFrame.config.label === undefined) { let errorMessage = "Session Parameters Node with no Time Frame Label configuration." parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) - bot.sessionError(bot.SESSION.parameters.timeFrame, errorMessage) + bot.sessionError(bot.LEARNING_SESSION.learningParameters.timeFrame, errorMessage) return false } - bot.SESSION.parameters.timeFrame.config.value = getTimeFrameFromLabel(bot.SESSION.parameters.timeFrame.config.label) - if (bot.SESSION.parameters.timeFrame.config.value === undefined) { + bot.LEARNING_SESSION.learningParameters.timeFrame.config.value = getTimeFrameFromLabel(bot.LEARNING_SESSION.learningParameters.timeFrame.config.label) + if (bot.LEARNING_SESSION.learningParameters.timeFrame.config.value === undefined) { let errorMessage = "Config error: label value not recognized. Try 01-min or 01-hs for example." parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) - bot.sessionError(bot.SESSION.parameters.timeFrame, errorMessage) + bot.sessionError(bot.LEARNING_SESSION.learningParameters.timeFrame, errorMessage) return false } /* Session Base Asset */ - if (bot.SESSION.parameters.sessionBaseAsset === undefined) { + if (bot.LEARNING_SESSION.learningParameters.sessionBaseAsset === undefined) { let errorMessage = "Session Parameters Node with no Session Base Asset." parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) - bot.sessionError(bot.SESSION.parameters, errorMessage) + bot.sessionError(bot.LEARNING_SESSION.learningParameters, errorMessage) return false } - if (bot.SESSION.parameters.sessionBaseAsset.config.initialBalance === undefined) { + if (bot.LEARNING_SESSION.learningParameters.sessionBaseAsset.config.initialBalance === undefined) { let errorMessage = "Session Parameters Session Base Asset with no initialBalance configuration." parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) - bot.sessionError(bot.SESSION.parameters.sessionBaseAsset, errorMessage) + bot.sessionError(bot.LEARNING_SESSION.learningParameters.sessionBaseAsset, errorMessage) return false } /* Session Quoted Asset */ - if (bot.SESSION.parameters.sessionQuotedAsset === undefined) { + if (bot.LEARNING_SESSION.learningParameters.sessionQuotedAsset === undefined) { let errorMessage = "Session Parameters Node with no Session Quoted Asset." parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) - bot.sessionError(bot.SESSION.parameters, errorMessage) + bot.sessionError(bot.LEARNING_SESSION.learningParameters, errorMessage) return false } - if (bot.SESSION.parameters.sessionQuotedAsset.config.initialBalance === undefined) { + if (bot.LEARNING_SESSION.learningParameters.sessionQuotedAsset.config.initialBalance === undefined) { let errorMessage = "Session Parameters Session Quoted Asset with no initialBalance configuration." parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) - bot.sessionError(bot.SESSION.parameters.sessionQuotedAsset, errorMessage) + bot.sessionError(bot.LEARNING_SESSION.learningParameters.sessionQuotedAsset, errorMessage) return false } /* Slippage */ - if (bot.SESSION.parameters.slippage === undefined) { // if the Slippage is missing we create a default one. - bot.SESSION.parameters.timeRange = { + if (bot.LEARNING_SESSION.learningParameters.slippage === undefined) { // if the Slippage is missing we create a default one. + bot.LEARNING_SESSION.learningParameters.slippage = { name: 'Missing Slippage', type: 'Slippage', config: { - positionRate: 0, + marketOrderRate: 0, stopLoss: 0, takeProfit: 0 } } } - if (bot.SESSION.parameters.slippage.config.positionRate === undefined) { // if the positionRate is missing we create a default one. - bot.SESSION.parameters.slippage.config.positionRate = 0 + if (bot.LEARNING_SESSION.learningParameters.slippage.config.marketOrderRate === undefined) { // if the marketOrderRate is missing we create a default one. + bot.LEARNING_SESSION.learningParameters.slippage.config.marketOrderRate = 0 } - if (bot.SESSION.parameters.slippage.config.stopLoss === undefined) { // if the stopLoss is missing we create a default one. - bot.SESSION.parameters.slippage.config.stopLoss = 0 + if (bot.LEARNING_SESSION.learningParameters.slippage.config.stopLoss === undefined) { // if the stopLoss is missing we create a default one. + bot.LEARNING_SESSION.learningParameters.slippage.config.stopLoss = 0 } - if (bot.SESSION.parameters.slippage.config.takeProfit === undefined) { // if the takeProfit is missing we create a default one. - bot.SESSION.parameters.slippage.config.takeProfit = 0 + if (bot.LEARNING_SESSION.learningParameters.slippage.config.takeProfit === undefined) { // if the takeProfit is missing we create a default one. + bot.LEARNING_SESSION.learningParameters.slippage.config.takeProfit = 0 } /* Fee Structure */ - if (bot.SESSION.parameters.feeStructure === undefined) { // if the Fee Structure is missing we create a default one. - bot.SESSION.parameters.feeStructure = { + if (bot.LEARNING_SESSION.learningParameters.feeStructure === undefined) { // if the Fee Structure is missing we create a default one. + bot.LEARNING_SESSION.learningParameters.feeStructure = { name: 'Missing Fee Structure', type: 'Fee Structure', config: { @@ -353,11 +353,11 @@ } } } - if (bot.SESSION.parameters.feeStructure.config.maker === undefined) { // if the maker is missing we create a default one. - bot.SESSION.parameters.feeStructure.config.maker = 0 + if (bot.LEARNING_SESSION.learningParameters.feeStructure.config.maker === undefined) { // if the maker is missing we create a default one. + bot.LEARNING_SESSION.learningParameters.feeStructure.config.maker = 0 } - if (bot.SESSION.parameters.feeStructure.config.taker === undefined) { // if the taker is missing we create a default one. - bot.SESSION.parameters.feeStructure.config.taker = 0 + if (bot.LEARNING_SESSION.learningParameters.feeStructure.config.taker === undefined) { // if the taker is missing we create a default one. + bot.LEARNING_SESSION.learningParameters.feeStructure.config.taker = 0 } return true @@ -369,7 +369,9 @@ function startLiveTrading(message) { if (bot.KEY === undefined || bot.SECRET === undefined) { - if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[ERROR] initialize -> startLiveTrading -> Key 'codeName' or 'secret' not provided. Plese check that and try again."); } + let errorMessage = "Key 'codeName' or 'secret' not provided. Plese check that and try again." + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[ERROR] initialize -> startLiveTrading -> " + errorMessage); } + bot.sessionError(bot.LEARNING_SESSION, errorMessage) return false } return true @@ -385,20 +387,20 @@ /* Reduce the balance */ let balancePercentage = 1 // This is the default value - if (bot.SESSION.config.balancePercentage !== undefined) { - balancePercentage = bot.SESSION.config.balancePercentage + if (bot.LEARNING_SESSION.config.balancePercentage !== undefined) { + balancePercentage = bot.LEARNING_SESSION.config.balancePercentage } - bot.SESSION.parameters.sessionBaseAsset.config.initialBalance = bot.SESSION.parameters.sessionBaseAsset.config.initialBalance * balancePercentage / 100 - bot.SESSION.parameters.sessionQuotedAsset.config.initialBalance = bot.SESSION.parameters.sessionQuotedAsset.config.initialBalance * balancePercentage / 100 + bot.LEARNING_SESSION.learningParameters.sessionBaseAsset.config.initialBalance = bot.LEARNING_SESSION.learningParameters.sessionBaseAsset.config.initialBalance * balancePercentage / 100 + bot.LEARNING_SESSION.learningParameters.sessionQuotedAsset.config.initialBalance = bot.LEARNING_SESSION.learningParameters.sessionQuotedAsset.config.initialBalance * balancePercentage / 100 - bot.SESSION.parameters.sessionBaseAsset.config.minimumBalance = bot.SESSION.parameters.sessionBaseAsset.config.minimumBalance * balancePercentage / 100 - bot.SESSION.parameters.sessionQuotedAsset.config.minimumBalance = bot.SESSION.parameters.sessionQuotedAsset.config.minimumBalance * balancePercentage / 100 + bot.LEARNING_SESSION.learningParameters.sessionBaseAsset.config.minimumBalance = bot.LEARNING_SESSION.learningParameters.sessionBaseAsset.config.minimumBalance * balancePercentage / 100 + bot.LEARNING_SESSION.learningParameters.sessionQuotedAsset.config.minimumBalance = bot.LEARNING_SESSION.learningParameters.sessionQuotedAsset.config.minimumBalance * balancePercentage / 100 - bot.SESSION.parameters.sessionBaseAsset.config.maximumBalance = bot.SESSION.parameters.sessionBaseAsset.config.maximumBalance * balancePercentage / 100 - bot.SESSION.parameters.sessionQuotedAsset.config.maximumBalance = bot.SESSION.parameters.sessionQuotedAsset.config.maximumBalance * balancePercentage / 100 + bot.LEARNING_SESSION.learningParameters.sessionBaseAsset.config.maximumBalance = bot.LEARNING_SESSION.learningParameters.sessionBaseAsset.config.maximumBalance * balancePercentage / 100 + bot.LEARNING_SESSION.learningParameters.sessionQuotedAsset.config.maximumBalance = bot.LEARNING_SESSION.learningParameters.sessionQuotedAsset.config.maximumBalance * balancePercentage / 100 - pProcessConfig.normalWaitTime = bot.SESSION.parameters.timeFrame.config.value + pProcessConfig.normalWaitTime = bot.LEARNING_SESSION.learningParameters.timeFrame.config.value return true } diff --git a/LowFrequencyTrading/Announcements.js b/LowFrequencyTrading/Announcements.js index 2c63c88d1..a5bff8a8f 100644 --- a/LowFrequencyTrading/Announcements.js +++ b/LowFrequencyTrading/Announcements.js @@ -21,7 +21,7 @@ exports.newAnnouncements = function newAnnouncements(bot, logger) { function finalize() { tradingSystem = undefined tradingEngine = undefined - bot.SESSION.messagesSent = undefined + bot.TRADING_SESSION.messagesSent = undefined } function makeAnnoucements(node) { @@ -44,8 +44,8 @@ exports.newAnnouncements = function newAnnouncements(bot, logger) { text = tradingSystem.formulas.get(announcement.formula.id) } - if (bot.SESSION.socialBots !== undefined) { - bot.SESSION.socialBots.announce(text) + if (bot.TRADING_SESSION.socialBots !== undefined) { + bot.TRADING_SESSION.socialBots.announce(text) tradingSystem.announcements.push([announcement.id, text]) } else { tradingSystem.errors.push([announcement.id, 'Could not announce because session does not have Social Bots.']) diff --git a/LowFrequencyTrading/ExchangeAPI.js b/LowFrequencyTrading/ExchangeAPI.js index a6d314c43..6b9ff2e92 100644 --- a/LowFrequencyTrading/ExchangeAPI.js +++ b/LowFrequencyTrading/ExchangeAPI.js @@ -27,7 +27,7 @@ let key = bot.KEY let secret = bot.SECRET - + const exchangeClass = ccxt[exchangeId] const exchangeConstructorParams = { 'apiKey': key, @@ -129,7 +129,23 @@ return order.id } catch (err) { tradingSystem.errors.push([tradingSystemOrder.id, "createOrder -> Error = " + err.message]) - logError("createOrder -> Error = " + err.message); + logError("createOrder -> Error = " + err.message) + if (err.message.indexOf('API-key format invalid') >= 0) { + logError ('The exchange says the API key provided is not good or it is not in the correct format. This is what you are using:') + errorFooter() + } + if (err.message.indexOf('Signature for this request is not valid.') >= 0) { + logError ('The exchange says the secret provided is not good, incorrect or not in the correct format. This is what you are using:') + errorFooter() + } + function errorFooter(){ + logError ('key: ->' + bot.KEY + '<-') + logError ('secret: ->' + bot.SECRET + '<-') + logError ('key.length:' + bot.KEY.length ) + logError ('secret.length:' + bot.SECRET.length ) + logError ('Double check that you copied the key at the codeName property and the secret at secret property without bringing any invisible caracter from the exchange web site, or leaving any character from the sample text. Also check that the keys are not disabled at the exchange ot that they are not restricted by IP.') + logError ('As a remainder. Binance key and secret are 64 bytes in lenght each one. ' ) + } } } diff --git a/LowFrequencyTrading/OrdersCalculations.js b/LowFrequencyTrading/OrdersCalculations.js new file mode 100644 index 000000000..94ffe42d3 --- /dev/null +++ b/LowFrequencyTrading/OrdersCalculations.js @@ -0,0 +1,316 @@ +exports.newOrdersCalculations = function newOrdersCalculations(bot, logger) { + /* + When we are live trading, we need to syncronize with the exchange. + */ + const MODULE_NAME = 'Orders Calculations' + + let thisObject = { + actualSizeCalculation: actualSizeCalculation, + actualRateCalculation: actualRateCalculation, + feesToBePaidCalculation: feesToBePaidCalculation, + amountReceivedCalculation: amountReceivedCalculation, + finapercentageFilledCalculationlize: percentageFilledCalculation, + feesPaidCalculation: feesPaidCalculation, + sizeFilledCalculation: sizeFilledCalculation, + initialize: initialize, + finalize: finalize + } + + let tradingEngine + let tradingSystem + let sessionParameters + + return thisObject + + function initialize() { + sessionParameters = bot.TRADING_SESSION.tradingParameters + tradingEngine = bot.simulationState.tradingEngine + tradingSystem = bot.simulationState.tradingSystem + } + + function finalize() { + sessionParameters = undefined + tradingEngine = undefined + tradingSystem = undefined + } + + async function actualSizeCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) { + /* + When we submit the order to the exchange, we do it specifying the order size + in Base Asset. It is mandatory to be in Base Asset as per CCXT Library API and + probably as per the Exchange API too. + + The exchange might take that size or might change it a little + bit. For example, if it does not accept as many decimals as we are sending. + + We have also seen that for Market Orders, the exchange usually does not take + the size requested exactly. Sometimes is a little bit less or even a little bit + more. For Limit orders it is usually the same. + + If it happens that the size accepted (Actual Size) is different than the + size requested (Size), we need to make several adjustments so that the + accounting syncronizes with reality. + */ + + /* This calculation needs to happen only once, the first time the order is checked. */ + if (tradingEngineOrder.orderBaseAsset.actualSize.value !== tradingEngineOrder.orderBaseAsset.actualSize.config.initialValue) { return } + + /* We receive the actual size from the exchange at the order.amount field. */ + tradingEngineOrder.orderBaseAsset.actualSize.value = order.amount + + recalculateActualSize() + recalculateSizePlaced() + + function recalculateActualSize() { + /* + We will recalculate the Quoted Asset Actual Size. This will also give to it an initial value. + In the next formula, we are unsing the rate of the order because we dont know yet the Actual Rate. + This rate might be replaced afteerwards for the Actual Rate when this is calculated again once + the Actual Rate is known. + */ + tradingEngineOrder.orderQuotedAsset.actualSize.value = tradingEngineOrder.orderBaseAsset.actualSize.value * tradingEngineOrder.rate.value + + tradingEngineOrder.orderBaseAsset.actualSize.value = global.PRECISE(tradingEngineOrder.orderBaseAsset.actualSize.value, 10) + tradingEngineOrder.orderQuotedAsset.actualSize.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.actualSize.value, 10) + } + + function recalculateSizePlaced() { + /* + We will also recalculate the Size Placed that was previously calculated with the order Size, + and now we will do it with the order Actual Size. + + We need to unaccount the previous size placed and correctly account + for the the new actual sizes we now know. + */ + tradingEngineStage.stageBaseAsset.sizePlaced.value = + tradingEngineStage.stageBaseAsset.sizePlaced.value - + tradingEngineOrder.orderBaseAsset.size.value + + tradingEngineStage.stageBaseAsset.sizePlaced.value = + tradingEngineStage.stageBaseAsset.sizePlaced.value + + tradingEngineOrder.orderBaseAsset.actualSize.value + + tradingEngineStage.stageQuotedAsset.sizePlaced.value = + tradingEngineStage.stageQuotedAsset.sizePlaced.value - + tradingEngineOrder.orderQuotedAsset.size.value + + tradingEngineStage.stageQuotedAsset.sizePlaced.value = + tradingEngineStage.stageQuotedAsset.sizePlaced.value + + tradingEngineOrder.orderQuotedAsset.actualSize.value + + tradingEngineStage.stageBaseAsset.sizePlaced.value = global.PRECISE(tradingEngineStage.stageBaseAsset.sizePlaced.value, 10) + tradingEngineStage.stageQuotedAsset.sizePlaced.value = global.PRECISE(tradingEngineStage.stageQuotedAsset.sizePlaced.value, 10) + } + } + + async function actualRateCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) { + /* + Actual Rate Calculation: Let's remember that the exchange may not take the rate + we send it with the order (in Limit orders) and for any reason change it a little bit. + For example we might have sent a rate with too many decimals and the exchange truncate + them. Another thing that might happen is that we send a rate that is matching orders already + at the order book. In this case we would get an actual rate not only different than the one + we sent, but also, better since the trading engine inside the exchange would match orders + with the first orders best by price. + + In the case of Market Orders we don't even send a rate. For all those reason + we get and store the Actual Rate at which the order was filled. + */ + if (order.average !== undefined) { + /* + We will use the average whenever is available. As of today it is not 100% clear when this is + available, but it seems sometimes it is not. In those cases we use the price. + */ + tradingEngineOrder.orderStatistics.actualRate.value = order.average + } else { + /* + We use the order.price when the average is not available. + */ + tradingEngineOrder.orderStatistics.actualRate.value = order.price + } + + tradingEngineOrder.orderStatistics.actualRate.value = global.PRECISE(tradingEngineOrder.orderStatistics.actualRate.value, 10) + + /* + If the Actual Rate happens to be exctly the same than the order rate, then there is + nothing else to do here. Otherwise there are adjustments to be made. + */ + if (tradingEngineOrder.orderStatistics.actualRate.value === tradingEngineOrder.rate.value) { return } + + let previousQuotedAssetActualSize = tradingEngineOrder.orderQuotedAsset.actualSize.value + + recalculateActualSize() + recalculateSizePlaced() + + function recalculateActualSize() { + if (tradingEngineOrder.orderStatistics.actualRate.value === tradingEngineOrder.rate.value) { return } + /* + Now we know the Actual Rate at which the order was filled. Since the actual rate + is not the same as the Rate we defined for the order, we need to syncronize + the Actual Order Size for Quoted Asset since it was calculated with the Order Size that we + now know it is not the one really used at the exchange. We will recalculate the + Actual Size in Quoted Asset and not in Base Asset, since the Actual Size in Base Asset + is the one we accepted by the exchange, so that is fixed, regardless of how we initially + got the Order Size in Base Asset, either from direct imput from the user or calculated + from the Order Size in Quoted Asset input it by the user. So here we go. + */ + previousQuotedAssetActualSize = tradingEngineOrder.orderQuotedAsset.actualSize.value + + tradingEngineOrder.orderQuotedAsset.actualSize.value = + tradingEngineOrder.orderBaseAsset.actualSize.value * + tradingEngineOrder.orderStatistics.actualRate.value + + tradingEngineOrder.orderQuotedAsset.actualSize.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.actualSize.value, 10) + + tradingSystem.warnings.push( + [ + [tradingEngineOrder.orderQuotedAsset.actualSize.id, tradingEngineOrder.orderStatistics.actualRate.id], + 'Actual Size (' + previousQuotedAssetActualSize + ') recalculated (' + tradingEngineOrder.orderQuotedAsset.actualSize.value + ') because the Actual Rate (' + tradingEngineOrder.orderStatistics.actualRate.value + ') is different than the Order Rate (' + tradingEngineOrder.rate.value + '))' + ] + ) + } + + function recalculateSizePlaced() { + let previousStageQuotedAssetSizePlaced = tradingEngineStage.stageQuotedAsset.sizePlaced.value + /* + Since we changed the Actual Size in Quoted Asset, we need to fix the Size Placed in Quoted Asset + that was previously calculated with the previous value of actualSize. + */ + tradingEngineStage.stageQuotedAsset.sizePlaced.value = + tradingEngineStage.stageQuotedAsset.sizePlaced.value - + previousQuotedAssetActualSize + + tradingEngineStage.stageQuotedAsset.sizePlaced.value = + tradingEngineStage.stageQuotedAsset.sizePlaced.value + + tradingEngineOrder.orderQuotedAsset.actualSize.value + + tradingEngineStage.stageQuotedAsset.sizePlaced.value = global.PRECISE(tradingEngineStage.stageQuotedAsset.sizePlaced.value, 10) + + tradingSystem.warnings.push( + [ + [tradingEngineStage.stageQuotedAsset.sizePlaced.id, tradingEngineOrder.orderStatistics.actualRate.id], + 'Size Placed (' + previousStageQuotedAssetSizePlaced + ') recalculated (' + tradingEngineStage.stageQuotedAsset.sizePlaced.value + ') because the Actual Rate (' + tradingEngineOrder.orderStatistics.actualRate.value + ') is different than the Order Rate (' + tradingEngineOrder.rate.value + '))' + ] + ) + } + } + + async function feesToBePaidCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) { + /* + The Fees To Be Paid are the fees that will be paid once the order is 100% Filled. + */ + switch (tradingEngineOrder.type) { + case 'Market Order': { + await applyFeePercentage( + 'feesToBePaid', + tradingEngineOrder, + tradingSystemOrder, + sessionParameters.feeStructure.config.taker, + 100 + ) + break + } + case 'Limit Order': { + await applyFeePercentage( + 'feesToBePaid', + tradingEngineOrder, + tradingSystemOrder, + sessionParameters.feeStructure.config.maker, + 100 + ) + break + } + } + } + + async function percentageFilledCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) { + /* + Percentage Filled Calculation: The only relevant information we get from the exchange is the order.filled. + We know this field will go up until reaching the Actual Size. Once it reaches that number + the order will be 100% filled. + */ + tradingEngineOrder.orderStatistics.percentageFilled.value = order.filled * 100 / tradingEngineOrder.orderBaseAsset.actualSize.value + tradingEngineOrder.orderStatistics.percentageFilled.value = global.PRECISE(tradingEngineOrder.orderStatistics.percentageFilled.value, 10) + } + + async function feesPaidCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) { + /* + Within the order information received from the exchange we can not see the fees paid so we need to + calculate them ourselves. Knwowing the Fees Paid is critical to later update the balances correctly, + since at every trade we will receive less than just the Actual Size. In fact we will receive the Actual + Size minus the Fees Paid. + + The percentage at which the exchange will charge fees needs to be configured at the the Session Fees Parameters. + */ + switch (tradingEngineOrder.type) { + case 'Market Order': { + await applyFeePercentage( + 'feesPaid', + tradingEngineOrder, + tradingSystemOrder, + sessionParameters.feeStructure.config.taker, + tradingEngineOrder.orderStatistics.percentageFilled.value + ) + break + } + case 'Limit Order': { + await applyFeePercentage( + 'feesPaid', + tradingEngineOrder, + tradingSystemOrder, + sessionParameters.feeStructure.config.maker, + tradingEngineOrder.orderStatistics.percentageFilled.value + ) + break + } + } + } + + async function sizeFilledCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) { + /* + CCXT returns order.filled with an amount denominated in Base Asset. We will + take it from there for our Order Base Asset. The amount in order.filled does + not reflect the Fees Paid, even if we are paying the fees in Base Asset. + */ + tradingEngineOrder.orderBaseAsset.sizeFilled.value = order.filled + tradingEngineOrder.orderBaseAsset.sizeFilled.value = global.PRECISE(tradingEngineOrder.orderBaseAsset.sizeFilled.value, 10) + /* + For Quoted Asset, CCXT does not return any field with the size filled, so we + need to calculate that by ourselves. + */ + tradingEngineOrder.orderQuotedAsset.sizeFilled.value = + tradingEngineOrder.orderQuotedAsset.actualSize.value * tradingEngineOrder.orderStatistics.percentageFilled.value / 100 + + tradingEngineOrder.orderQuotedAsset.sizeFilled.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.sizeFilled.value, 10) + } + + async function amountReceivedCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) { + /* + This calculation is for informational purposes, so that users do not have to calculate + it by themselves. The Amount Received out of the trade depends if we are Buying or Selling + the Base Asset. If we are Buying, then the Amount Receiced will be in Base Asset. If we + are selling then it will be in Quoted Asset. + */ + + switch (true) { + case tradingSystemOrder.type === 'Market Buy Order' || tradingSystemOrder.type === 'Limit Buy Order': { + /* + In this case the Amount Received is in Base Asset. + */ + tradingEngineOrder.orderBaseAsset.amountReceived.value = + tradingEngineOrder.orderBaseAsset.sizeFilled.value - + tradingEngineOrder.orderBaseAsset.feesPaid.value + break + } + case tradingSystemOrder.type === 'Market Sell Order' || tradingSystemOrder.type === 'Limit Sell Order': { + /* + In this case the Amount Received is in Quoted Asset. + */ + tradingEngineOrder.orderQuotedAsset.amountReceived.value = + tradingEngineOrder.orderQuotedAsset.sizeFilled.value - + tradingEngineOrder.orderQuotedAsset.feesPaid.value + break + } + } + } +} \ No newline at end of file diff --git a/LowFrequencyTrading/OrdersSimulations.js b/LowFrequencyTrading/OrdersSimulations.js new file mode 100644 index 000000000..cf4fa5a20 --- /dev/null +++ b/LowFrequencyTrading/OrdersSimulations.js @@ -0,0 +1,433 @@ +exports.newOrdersSimulations = function newOrdersSimulations(bot, logger) { + /* + When we are backtesting or paper trading, we need to simulate what would have happened at the exchange. + */ + const MODULE_NAME = 'Orders Simulations' + + let thisObject = { + actualSizeSimulation: actualSizeSimulation, + actualRateSimulation: actualRateSimulation, + feesToBePaidSimulation: feesToBePaidSimulation, + amountReceivedSimulation: amountReceivedSimulation, + percentageFilledSimulation: percentageFilledSimulation, + feesPaidSimulation: feesPaidSimulation, + sizeFilledSimulation: sizeFilledSimulation, + initialize: initialize, + finalize: finalize + } + + let tradingEngine + let tradingSystem + let sessionParameters + + return thisObject + + function initialize() { + sessionParameters = bot.TRADING_SESSION.tradingParameters + tradingEngine = bot.simulationState.tradingEngine + tradingSystem = bot.simulationState.tradingSystem + } + + function finalize() { + sessionParameters = undefined + tradingEngine = undefined + tradingSystem = undefined + } + + async function actualSizeSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) { + /* + The Actual Size is the size the exchange accepts for the order, + regarless of the size sent in the request to the exchange. + + While backtesting and paper trading, we do not really need to + simulate an Actual Size. We will just copy to that field the + size of the order. + + This calculation needs to happen only once, so as to set these values. + */ + if (tradingEngineOrder.orderBaseAsset.actualSize.value !== tradingEngineOrder.orderBaseAsset.actualSize.config.initialValue) { return } + + tradingEngineOrder.orderBaseAsset.actualSize.value = tradingEngineOrder.orderBaseAsset.size.value + tradingEngineOrder.orderQuotedAsset.actualSize.value = tradingEngineOrder.orderQuotedAsset.size.value + } + + function actualRateSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) { + /* Actual Rate Simulation */ + let previousQuotedAssetActualSize = tradingEngineOrder.orderQuotedAsset.actualSize.value + + if (tradingEngineOrder.orderStatistics.actualRate.value === tradingEngineOrder.orderStatistics.actualRate.config.initialValue) { + /* + The default value for the Actual Rate is the rate of the order. + */ + tradingEngineOrder.orderStatistics.actualRate.value = tradingEngineOrder.rate.value + } + + basedOnSessionParameters() + basedOnTradingSystem() + considerBestMatchWithOrderBook() + recalculateActualSize() + recalculateSizePlaced() + + function basedOnSessionParameters() { + /* + The default way to calculate the Actual Rate is to apply the Slippage defined + at the Session Parameters Definition. We are not going to apply the default if there + is a definition for this order on how to calculate the actual rate. + */ + if (tradingSystemOrder.simulatedExchangeEvents !== undefined) { + if (tradingSystemOrder.simulatedExchangeEvents.simulatedActualRate !== undefined) { + return + } + } + switch (tradingEngineOrder.type) { + case 'Market Order': { + + /* Actual Rate is simulated based on the Session Paremeters */ + let slippageAmount = tradingEngineOrder.rate.value * sessionParameters.slippage.config.marketOrderRate / 100 + if (slippageAmount !== 0) { + switch (tradingSystemOrder.type) { + case 'Market Sell Order': { + tradingEngineOrder.orderStatistics.actualRate.value = tradingEngineOrder.rate.value - slippageAmount + tradingEngineOrder.orderStatistics.actualRate.value = global.PRECISE(tradingEngineOrder.orderStatistics.actualRate.value, 10) + + tradingSystem.warnings.push( + [ + [sessionParameters.slippage.id, tradingEngineOrder.rate.id, tradingEngineOrder.orderStatistics.actualRate.id], + 'slippage.config.marketOrderRate Slippage Amount (' + slippageAmount + ') substracted from Order Rate (' + tradingEngineOrder.rate.value + ') to get the Actual Rate (' + tradingEngineOrder.orderStatistics.actualRate.value + ')' + ] + ) + break + } + case 'Market Buy Order': { + tradingEngineOrder.orderStatistics.actualRate.value = tradingEngineOrder.rate.value + slippageAmount + tradingEngineOrder.orderStatistics.actualRate.value = global.PRECISE(tradingEngineOrder.orderStatistics.actualRate.value, 10) + + tradingSystem.warnings.push( + [ + [sessionParameters.slippage.id, tradingEngineOrder.rate.id, tradingEngineOrder.orderStatistics.actualRate.id], + 'slippage.config.marketOrderRate Slippage Amount (' + slippageAmount + ') added to Order Rate (' + tradingEngineOrder.rate.value + ') to get the Actual Rate (' + tradingEngineOrder.orderStatistics.actualRate.value + ')' + ] + ) + break + } + } + } + break + } + case 'Limit Order': { + /* In Limit Orders the actual rate is the rate of the order, there is no slippage */ + tradingEngineOrder.orderStatistics.actualRate.value = tradingEngineOrder.rate.value + break + } + } + } + + function basedOnTradingSystem() { + /* + If the user specified a Formula to simulate the Actual Rate for this order, + we use that formula here. Note that if this Formula exists, this will override + the calculation based on Slippage. + */ + if (tradingSystemOrder.simulatedExchangeEvents !== undefined) { + if (tradingSystemOrder.simulatedExchangeEvents.simulatedActualRate !== undefined) { + if (tradingSystemOrder.simulatedExchangeEvents.simulatedActualRate.formula !== undefined) { + /* Calculate this only once for this order */ + if (tradingEngineOrder.orderStatistics.actualRate.value === tradingEngineOrder.orderStatistics.actualRate.config.initialValue) { + let newValue = tradingSystem.formulas.get(tradingSystemOrder.simulatedExchangeEvents.simulatedActualRate.formula.id) + newValue = global.PRECISE(newValue, 10) + + tradingSystem.warnings.push( + [ + tradingSystemOrder.simulatedExchangeEvents.simulatedActualRate.formula.id, + 'Actual Rate (' + tradingEngineOrder.orderStatistics.actualRate.value + ') calculated with this Formula (' + newValue + ')' + ] + ) + tradingEngineOrder.orderStatistics.actualRate.value = newValue + } + } + } + } + } + + function considerBestMatchWithOrderBook() { + /* + Until here we have got the Actual Rate based on the formula definition or based on session parameters slippage. + The last check is about watching what happened in the market. Let's remember that the exchange will + fill the order with the best possible matches at it's order book. That means that if the rate + we set for the order was too low (for a sale order) or too hight (for a buy order), the actual rate + should be better than expected. + + We don't know what it is at the order book, but whe know that the last candle includes trades that bounced between + the min an max of the candle. That could tell us the above the max and below the min, the order would be matched + with orders at the order book. For that reason, we do this: + */ + switch (true) { + case tradingSystemOrder.type === 'Market Buy Order' || tradingSystemOrder.type === 'Limit Buy Order': { + if (tradingEngineOrder.orderStatistics.actualRate.value > tradingEngine.current.episode.candle.max.value) { + tradingSystem.warnings.push( + [ + [tradingEngineOrder.orderStatistics.actualRate.id, tradingEngine.current.episode.candle.max.id], + 'Actual Rate (' + tradingEngineOrder.orderStatistics.actualRate.value + ') too high. Changed to candle.max (' + tradingEngine.current.episode.candle.max.value + ')' + ] + ) + tradingEngineOrder.orderStatistics.actualRate.value = tradingEngine.current.episode.candle.max.value + } + break + } + case tradingSystemOrder.type === 'Market Sell Order' || tradingSystemOrder.type === 'Limit Sell Order': { + if (tradingEngineOrder.orderStatistics.actualRate.value < tradingEngine.current.episode.candle.min.value) { + tradingSystem.warnings.push( + [ + [tradingEngineOrder.orderStatistics.actualRate.id, tradingEngine.current.episode.candle.min.id], + 'Actual Rate (' + tradingEngineOrder.orderStatistics.actualRate.value + ') too low. Changed to candle.min (' + tradingEngine.current.episode.candle.max.value + ')' + ] + ) + tradingEngineOrder.orderStatistics.actualRate.value = tradingEngine.current.episode.candle.min.value + } + break + } + } + tradingEngineOrder.orderStatistics.actualRate.value = global.PRECISE(tradingEngineOrder.orderStatistics.actualRate.value, 10) + } + + function recalculateActualSize() { + if (tradingEngineOrder.orderStatistics.actualRate.value === tradingEngineOrder.rate.value) { return } + /* + Now that we have an Actual Rate, we need to adjust the Actual Size in Quoted Asset, since + Base Asset is what the exchange accepts as input and give information at. Quotes Asset properties are always calculated. + */ + tradingEngineOrder.orderQuotedAsset.actualSize.value = + tradingEngineOrder.orderBaseAsset.actualSize.value * + tradingEngineOrder.orderStatistics.actualRate.value + + tradingEngineOrder.orderQuotedAsset.actualSize.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.actualSize.value, 10) + + tradingSystem.warnings.push( + [ + [tradingEngineOrder.orderQuotedAsset.actualSize.id, tradingEngineOrder.orderStatistics.actualRate.id], + 'Actual Size (' + previousQuotedAssetActualSize + ') recalculated (' + tradingEngineOrder.orderQuotedAsset.actualSize.value + ') because the Actual Rate (' + tradingEngineOrder.orderStatistics.actualRate.value + ') is different than the Order Rate (' + tradingEngineOrder.rate.value + '))' + ] + ) + } + + function recalculateSizePlaced() { + if (tradingEngineOrder.orderStatistics.actualRate.value === tradingEngineOrder.rate.value) { return } + /* + Since the Actual Rate might have changed, we need to recalculate the Size Placed, where we accumulate all the Size Placed of + all orders of a Stage. For Base Asset there is nothing to do, since the Actual Rate does not have an inpact on it. + + For Quoted Asset, we need to first unaccount what this same orded added before to Size Placed (with the precious Actual Size) and + account with the new Actual Size. + */ + let previousStageQuotedAssetSizePlaced = tradingEngineStage.stageQuotedAsset.sizePlaced.value + + tradingEngineStage.stageQuotedAsset.sizePlaced.value = + tradingEngineStage.stageQuotedAsset.sizePlaced.value - + previousQuotedAssetActualSize + + tradingEngineStage.stageQuotedAsset.sizePlaced.value = + tradingEngineStage.stageQuotedAsset.sizePlaced.value + + tradingEngineOrder.orderQuotedAsset.actualSize.value + + tradingEngineStage.stageQuotedAsset.sizePlaced.value = global.PRECISE(tradingEngineStage.stageQuotedAsset.sizePlaced.value, 10) + + tradingSystem.warnings.push( + [ + [tradingEngineStage.stageQuotedAsset.sizePlaced.id, tradingEngineOrder.orderStatistics.actualRate.id], + 'Size Placed (' + previousStageQuotedAssetSizePlaced + ') recalculated (' + tradingEngineStage.stageQuotedAsset.sizePlaced.value + ') because the Actual Rate (' + tradingEngineOrder.orderStatistics.actualRate.value + ') is different than the Order Rate (' + tradingEngineOrder.rate.value + '))' + ] + ) + } + } + + async function feesToBePaidSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) { + /* + The Fees To Be Paid are the fees that will be paid once the order is 100% Filled. + */ + switch (tradingEngineOrder.type) { + case 'Market Order': { + await applyFeePercentage( + 'feesToBePaid', + tradingEngineOrder, + tradingSystemOrder, + sessionParameters.feeStructure.config.taker, + 100 + ) + break + } + case 'Limit Order': { + await applyFeePercentage( + 'feesToBePaid', + tradingEngineOrder, + tradingSystemOrder, + sessionParameters.feeStructure.config.maker, + 100 + ) + break + } + } + } + + function percentageFilledSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) { + /* + When we are live, it is the exchange who knows which order was filled and in + which amount. Since we are backtesting here, we need to simulate what happens + at the exchange. We can not simulate that an order is filled if the candle we + see did not touch the order. + + For limit buy orders we are going to check if the price was below the order actual rate, and + for limit sell orders we are going to check if the price was above the order actual rate. + + If we have a partial filling defined specifically for this order, + we are going to use it to estimate how much of the order we will consider filled. + */ + let orderWasHit + switch (tradingSystemOrder.type) { + case 'Limit Buy Order': { + if (tradingEngine.current.episode.candle.min.value <= tradingEngineOrder.orderStatistics.actualRate.value) { + orderWasHit = true + } + break + } + case 'Limit Sell Order': { + if (tradingEngine.current.episode.candle.max.value >= tradingEngineOrder.orderStatistics.actualRate.value) { + orderWasHit = true + } + break + } + case 'Market Buy Order': { + orderWasHit = true + break + } + case 'Market Sell Order': { + orderWasHit = true + break + } + } + if (orderWasHit === true) { + /* + Initially, we will assume that 100% of the order was filled. This might be overwritten + if there is a configuration for this order that details how much is must be considered + filled each time we check this. + */ + tradingEngineOrder.orderStatistics.percentageFilled.value = 100 + + /* Based on the Trading System Definition */ + if (tradingSystemOrder.simulatedExchangeEvents !== undefined) { + if (tradingSystemOrder.simulatedExchangeEvents.simulatedPartialFill !== undefined) { + if (tradingSystemOrder.simulatedExchangeEvents.simulatedPartialFill.config.fillProbability !== undefined) { + + /* Percentage Filled */ + let percentageFilled = tradingSystemOrder.simulatedExchangeEvents.simulatedPartialFill.config.fillProbability * 100 + if (tradingEngineOrder.orderStatistics.percentageFilled.value + percentageFilled > 100) { + percentageFilled = 100 - tradingEngineOrder.orderStatistics.percentageFilled.value + } + tradingEngineOrder.orderStatistics.percentageFilled.value = tradingEngineOrder.orderStatistics.percentageFilled.value + percentageFilled + tradingEngineOrder.orderStatistics.percentageFilled.value = global.PRECISE(tradingEngineOrder.orderStatistics.percentageFilled.value, 10) + } + } + } + } + } + + function feesPaidSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) { + /* + The way the Fess Paid are calculated is as follows: From the order Actual Size we apply the % of fee, + that would give us the total fees for that order. To that we apply the % filled of the orders, + that give use the final result of the fees paid for the current % filled. + + But before that, we will see if there is an specific definition on how to simulate the fees for + this order, or if there is not, we use the Fees Structure defined at the Session Parameters. + */ + + /* Based on the Trading System Definition */ + if (tradingSystemOrder.simulatedExchangeEvents !== undefined) { + if (tradingSystemOrder.simulatedExchangeEvents.simulatedFeesPaid !== undefined) { + if (tradingSystemOrder.simulatedExchangeEvents.simulatedFeesPaid.config.percentage !== undefined) { + applyFeePercentage( + 'feesPaid', + tradingEngineOrder, + tradingSystemOrder, + tradingSystemOrder.simulatedExchangeEvents.simulatedFeesPaid.config.percentage, + tradingEngineOrder.orderStatistics.percentageFilled.value + ) + return + } + } + } + + /* + Another way to simulate the Fees Paid is by using the Session Paremeters configuration for Fees. + */ + switch (tradingEngineOrder.type) { + case 'Market Order': { + applyFeePercentage( + 'feesPaid', + tradingEngineOrder, + tradingSystemOrder, + sessionParameters.feeStructure.config.taker, + tradingEngineOrder.orderStatistics.percentageFilled.value + ) + break + } + case 'Limit Order': { + applyFeePercentage( + 'feesPaid', + tradingEngineOrder, + tradingSystemOrder, + sessionParameters.feeStructure.config.maker, + tradingEngineOrder.orderStatistics.percentageFilled.value + ) + break + } + } + } + + function sizeFilledSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) { + /* + Size Filled is the amount of Base Asset or Quoted Asset that at the exchange matches + another counterparty order. We calculate this using the Percentage Filled we already simulated. + */ + + tradingEngineOrder.orderBaseAsset.sizeFilled.value = + tradingEngineOrder.orderBaseAsset.actualSize.value * + tradingEngineOrder.orderStatistics.percentageFilled.value / 100 + + tradingEngineOrder.orderQuotedAsset.sizeFilled.value = + tradingEngineOrder.orderQuotedAsset.actualSize.value * + tradingEngineOrder.orderStatistics.percentageFilled.value / 100 + + tradingEngineOrder.orderBaseAsset.sizeFilled.value = global.PRECISE(tradingEngineOrder.orderBaseAsset.sizeFilled.value, 10) + tradingEngineOrder.orderQuotedAsset.sizeFilled.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.sizeFilled.value, 10) + } + + async function amountReceivedSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) { + /* + This simulation is for informational purposes, so that users do not have to calculate + it by themselves. The Amount Received out of the trade depends if we are Buying or Selling + the Base Asset. If we are Buying, then the Amount Receiced will be in Base Asset. If we + are selling then it will be in Quoted Asset. + */ + + switch (true) { + case tradingSystemOrder.type === 'Market Buy Order' || tradingSystemOrder.type === 'Limit Buy Order': { + /* + In this case the Amount Received is in Base Asset. + */ + tradingEngineOrder.orderBaseAsset.amountReceived.value = + tradingEngineOrder.orderBaseAsset.sizeFilled.value - + tradingEngineOrder.orderBaseAsset.feesPaid.value + break + } + case tradingSystemOrder.type === 'Market Sell Order' || tradingSystemOrder.type === 'Limit Sell Order': { + /* + In this case the Amount Received is in Quoted Asset. + */ + tradingEngineOrder.orderQuotedAsset.amountReceived.value = + tradingEngineOrder.orderQuotedAsset.sizeFilled.value - + tradingEngineOrder.orderQuotedAsset.feesPaid.value + break + } + } + } +} \ No newline at end of file diff --git a/LowFrequencyTrading/Snapshots.js b/LowFrequencyTrading/Snapshots.js index 6f1b02c5a..933d0f37c 100644 --- a/LowFrequencyTrading/Snapshots.js +++ b/LowFrequencyTrading/Snapshots.js @@ -45,7 +45,7 @@ exports.newSnapshots = function newSnapshots(bot, logger) { function initialize() { tradingEngine = bot.simulationState.tradingEngine tradingSystem = bot.simulationState.tradingSystem - sessionParameters = bot.SESSION.parameters + sessionParameters = bot.TRADING_SESSION.tradingParameters } function finalize() { @@ -322,7 +322,7 @@ exports.newSnapshots = function newSnapshots(bot, logger) { fileContent = "" + fileContent + ""; let fileName = pFileName + '.csv'; - let filePath = bot.filePathRoot + "/Output/" + bot.SESSION.folderName + "/" + SNAPSHOTS_FOLDER_NAME; + let filePath = bot.filePathRoot + "/Output/" + bot.TRADING_SESSION.folderName + "/" + SNAPSHOTS_FOLDER_NAME; filePath += '/' + fileName fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated); diff --git a/LowFrequencyTrading/TradingEngine.js b/LowFrequencyTrading/TradingEngine.js index b9dc6f889..b91ed2247 100644 --- a/LowFrequencyTrading/TradingEngine.js +++ b/LowFrequencyTrading/TradingEngine.js @@ -25,7 +25,7 @@ exports.newTradingEngine = function newTradingEngine(bot, logger) { function initialize() { tradingEngine = bot.simulationState.tradingEngine - sessionParameters = bot.SESSION.parameters + sessionParameters = bot.TRADING_SESSION.tradingParameters initializeNodeMap(tradingEngine) @@ -66,6 +66,8 @@ exports.newTradingEngine = function newTradingEngine(bot, logger) { function setCurrentCycle(cycle) { tradingEngine.current.episode.cycle.value = cycle + tradingEngine.current.episode.cycle.lastBegin.value = tradingEngine.current.episode.cycle.begin.value + tradingEngine.current.episode.cycle.lastEnd.value = tradingEngine.current.episode.cycle.end.value switch (cycle) { case 'First': { tradingEngine.current.episode.cycle.begin.value = @@ -89,6 +91,17 @@ exports.newTradingEngine = function newTradingEngine(bot, logger) { break } } + /* + We can not have the last begin or last end to be zero, because that would prevent + objects starting with lastBegin in zero to being saved on file. For that reason + we will do this: + */ + if (tradingEngine.current.episode.cycle.lastBegin.value === 0) { + tradingEngine.current.episode.cycle.lastBegin.value = tradingEngine.current.episode.cycle.begin.value + } + if (tradingEngine.current.episode.cycle.lastEnd.value === 0) { + tradingEngine.current.episode.cycle.lastEnd.value = tradingEngine.current.episode.cycle.end.value + } } function cloneValues(originNode, destinationNode) { diff --git a/LowFrequencyTrading/TradingEpisode.js b/LowFrequencyTrading/TradingEpisode.js index 76bb7a8d3..05ea301c0 100644 --- a/LowFrequencyTrading/TradingEpisode.js +++ b/LowFrequencyTrading/TradingEpisode.js @@ -23,7 +23,7 @@ exports.newTradingEpisode = function newTradingEpisode(bot, logger, tradingEngin function initialize() { tradingSystem = bot.simulationState.tradingSystem tradingEngine = bot.simulationState.tradingEngine - sessionParameters = bot.SESSION.parameters + sessionParameters = bot.TRADING_SESSION.tradingParameters } function finalize() { @@ -47,24 +47,27 @@ exports.newTradingEpisode = function newTradingEpisode(bot, logger, tradingEngin /* This function is called each time the simulation starts. That does not mean the Episode must be opened there, since it might happen that it is looping at the end of the market - or the task / session was restarted. + or the task / session was restarted. We will only execute if the episode was never opened before. */ - if (bot.FIRST_EXECUTION === true) { - /* Getting the begin Balance from the session configuration */ - tradingEngine.current.episode.episodeBaseAsset.beginBalance.value = sessionParameters.sessionBaseAsset.config.initialBalance - tradingEngine.current.episode.episodeQuotedAsset.beginBalance.value = sessionParameters.sessionQuotedAsset.config.initialBalance + if (tradingEngine.current.episode.begin.value !== tradingEngine.current.episode.begin.config.initialValue) { return } - /* The current balance is also the begin balance, that is how this starts. */ - tradingEngine.current.episode.episodeBaseAsset.balance.value = sessionParameters.sessionBaseAsset.config.initialBalance - tradingEngine.current.episode.episodeQuotedAsset.balance.value = sessionParameters.sessionQuotedAsset.config.initialBalance + /* Starting begin and end */ + tradingEngine.current.episode.begin.value = tradingEngine.current.episode.candle.begin.value + tradingEngine.current.episode.end.value = tradingEngine.current.episode.candle.end.value - /* Recording the opening at the Trading Engine Data Structure */ - tradingEngine.current.episode.status.value = 'Open' - tradingEngine.current.episode.serialNumber.value = 1 - tradingEngine.current.episode.identifier.value = global.UNIQUE_ID() - tradingEngine.current.episode.begin.value = tradingEngine.current.episode.candle.begin.value - tradingEngine.current.episode.beginRate.value = tradingEngine.current.episode.candle.close.value - } + /* Getting the begin Balance from the session configuration */ + tradingEngine.current.episode.episodeBaseAsset.beginBalance.value = sessionParameters.sessionBaseAsset.config.initialBalance + tradingEngine.current.episode.episodeQuotedAsset.beginBalance.value = sessionParameters.sessionQuotedAsset.config.initialBalance + + /* The current balance is also the begin balance, that is how this starts. */ + tradingEngine.current.episode.episodeBaseAsset.balance.value = sessionParameters.sessionBaseAsset.config.initialBalance + tradingEngine.current.episode.episodeQuotedAsset.balance.value = sessionParameters.sessionQuotedAsset.config.initialBalance + + /* Recording the opening at the Trading Engine Data Structure */ + tradingEngine.current.episode.status.value = 'Open' + tradingEngine.current.episode.serialNumber.value = 1 + tradingEngine.current.episode.identifier.value = global.UNIQUE_ID() + tradingEngine.current.episode.beginRate.value = tradingEngine.current.episode.candle.close.value } function updateExitType(exitType) { @@ -85,7 +88,7 @@ exports.newTradingEpisode = function newTradingEpisode(bot, logger, tradingEngin function updateEnds() { if (tradingEngine.current.episode.status.value === 'Open') { - tradingEngine.current.episode.end.value = tradingEngine.current.episode.candle.end.value + tradingEngine.current.episode.end.value = tradingEngine.current.episode.end.value + sessionParameters.timeFrame.config.value tradingEngine.current.episode.endRate.value = tradingEngine.current.episode.candle.close.value tradingEngine.current.episode.episodeBaseAsset.endBalance.value = tradingEngine.current.episode.episodeBaseAsset.balance.value tradingEngine.current.episode.episodeQuotedAsset.endBalance.value = tradingEngine.current.episode.episodeQuotedAsset.balance.value @@ -160,7 +163,7 @@ exports.newTradingEpisode = function newTradingEpisode(bot, logger, tradingEngin tradingEngine.current.episode.episodeBaseAsset.hitRatio.value = global.PRECISE(tradingEngine.current.episode.episodeBaseAsset.hitRatio.value, 10) tradingEngine.current.episode.episodeQuotedAsset.hitRatio.value = global.PRECISE(tradingEngine.current.episode.episodeQuotedAsset.hitRatio.value, 10) } - + /* Updating Annualized Rate Of Return diff --git a/LowFrequencyTrading/TradingExecution.js b/LowFrequencyTrading/TradingExecution.js index 7e6619ec3..b1e5e3e25 100644 --- a/LowFrequencyTrading/TradingExecution.js +++ b/LowFrequencyTrading/TradingExecution.js @@ -24,7 +24,7 @@ exports.newTradingExecution = function newTradingExecution(bot, logger, tradingE function initialize() { tradingSystem = bot.simulationState.tradingSystem tradingEngine = bot.simulationState.tradingEngine - sessionParameters = bot.SESSION.parameters + sessionParameters = bot.TRADING_SESSION.tradingParameters tradingOrdersModule.initialize() } diff --git a/LowFrequencyTrading/TradingOrders.js b/LowFrequencyTrading/TradingOrders.js index 76d1cabe0..2c432eb57 100644 --- a/LowFrequencyTrading/TradingOrders.js +++ b/LowFrequencyTrading/TradingOrders.js @@ -27,7 +27,7 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM function initialize() { tradingSystem = bot.simulationState.tradingSystem tradingEngine = bot.simulationState.tradingEngine - sessionParameters = bot.SESSION.parameters + sessionParameters = bot.TRADING_SESSION.tradingParameters exchangeAPIModule.initialize() announcementsModule.initialize() @@ -148,7 +148,7 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM if (tradingEngine.current.episode.cycle.value === 'Second') { continue } /* Simulate Events that happens at the Exchange, if needed. */ - simulateExchangeEvents(tradingEngineStage, tradingSystemOrder, tradingEngineOrder) + simulateCheckExchangeEvents(tradingEngineStage, tradingSystemOrder, tradingEngineOrder) /* Check Events that happens at the Exchange, if needed. */ let allGood = await checkExchangeEvents(tradingEngineStage, tradingSystemOrder, tradingEngineOrder) @@ -159,7 +159,12 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM need to cancel it, since we could end up with inconsisten information at the accounting level. */ if (tradingSystemOrder.cancelOrderEvent !== undefined) { - tradingSystem.warnings.push([tradingSystemOrder.cancelOrderEvent.id, 'Skipping Cancel Event Checking because cheking the order at the exchange failed.']) + tradingSystem.warnings.push( + [ + tradingSystemOrder.cancelOrderEvent.id, + 'Skipping Cancel Event Checking because cheking the order at the exchange failed.' + ] + ) } return } @@ -236,10 +241,16 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM if (tradingEngineOrder.orderBaseAsset.size === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderBaseAsset.size === undefined', tradingEngineOrder) } if (tradingEngineOrder.orderBaseAsset.sizeFilled === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderBaseAsset.sizeFilled === undefined', tradingEngineOrder) } if (tradingEngineOrder.orderBaseAsset.feesPaid === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderBaseAsset.feesPaid === undefined', tradingEngineOrder) } + if (tradingEngineOrder.orderBaseAsset.actualSize === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderBaseAsset.actualSize === undefined', tradingEngineOrder) } + if (tradingEngineOrder.orderBaseAsset.amountReceived === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderBaseAsset.amountReceived === undefined', tradingEngineOrder) } + if (tradingEngineOrder.orderBaseAsset.feesToBePaid === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderBaseAsset.feesToBePaid === undefined', tradingEngineOrder) } if (tradingEngineOrder.orderQuotedAsset.size === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderQuotedAsset.size === undefined', tradingEngineOrder) } if (tradingEngineOrder.orderQuotedAsset.sizeFilled === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderQuotedAsset.sizeFilled === undefined', tradingEngineOrder) } if (tradingEngineOrder.orderQuotedAsset.feesPaid === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderQuotedAsset.feesPaid === undefined', tradingEngineOrder) } + if (tradingEngineOrder.orderQuotedAsset.actualSize === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderQuotedAsset.actualSize === undefined', tradingEngineOrder) } + if (tradingEngineOrder.orderQuotedAsset.amountReceived === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderQuotedAsset.amountReceived === undefined', tradingEngineOrder) } + if (tradingEngineOrder.orderQuotedAsset.feesToBePaid === undefined) { badDefinitionUnhandledException(undefined, 'tradingEngineOrder.orderQuotedAsset.feesToBePaid === undefined', tradingEngineOrder) } } async function tryToOpenOrder(tradingEngineStage, executionAlgorithm, tradingSystemOrder, tradingEngineOrder, situationName) { @@ -248,15 +259,35 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM await calculateOrderSize() /* Check Size: We are not going to create Orders which size is equal or less to zero. */ - if (tradingEngineOrder.orderBaseAsset.size.value <= 0 || tradingEngineOrder.orderQuotedAsset.size.value <= 0) { - tradingSystem.warnings.push([tradingSystemOrder.id, 'Could not open this order because its size would be zero.']) + if (tradingEngineOrder.orderBaseAsset.size.value <= 0) { + tradingSystem.warnings.push( + [ + [tradingEngineOrder.orderBaseAsset.size.id, tradingSystemOrder.id], + 'Could not open this order because its size is zero.' + ] + ) + return + } + + if (tradingEngineOrder.orderQuotedAsset.size.value <= 0) { + tradingSystem.warnings.push( + [ + [tradingEngineOrder.orderQuotedAsset.size.id, tradingSystemOrder.id ], + 'Could not open this order because its size is zero.' + ] + ) return } /* Place Order at the Exchange, if needed. */ let result = await createOrderAtExchange(tradingSystemOrder, tradingEngineOrder) if (result !== true) { - tradingSystem.warnings.push([tradingSystemOrder.id, 'Could not open this order because something failed placing the order at the Exchange.']) + tradingSystem.warnings.push( + [ + [tradingSystemOrder.id, tradingEngineOrder], + 'Could not open this order because something failed placing the order at the Exchange.' + ] + ) return } @@ -308,53 +339,133 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM } async function calculateOrderSize() { - /* Validate that this config exists */ - if (executionAlgorithm.config.percentageOfStageTargetSize === undefined) { badDefinitionUnhandledException(undefined, 'executionAlgorithm.config.percentageOfStageTargetSize === undefined', executionAlgorithm) } - if (isNaN(executionAlgorithm.config.percentageOfStageTargetSize) === true) { badDefinitionUnhandledException(undefined, 'isNaN(executionAlgorithm.config.percentageOfStageTargetSize) === true', executionAlgorithm) } + let algorithmSizeInBaseAsset + let algorithmSizeInQuotedAsset - let algorithmSizeInBaseAsset = tradingEngineStage.stageBaseAsset.targetSize.value * executionAlgorithm.config.percentageOfStageTargetSize / 100 - let algorithmSizeInQuotedAsset = tradingEngineStage.stageQuotedAsset.targetSize.value * executionAlgorithm.config.percentageOfStageTargetSize / 100 + requiredConfigurationValidation() + notPassingTargetSizeValidation() + notNegativeBalanceValidation() - /* Validate that this config exists */ - if (tradingSystemOrder.config.percentageOfAlgorithmSize === undefined) { badDefinitionUnhandledException(undefined, 'tradingSystemOrder.config.percentageOfAlgorithmSize === undefined', tradingSystemOrder) } - if (isNaN(tradingSystemOrder.config.percentageOfAlgorithmSize) === true) { badDefinitionUnhandledException(undefined, 'isNaN(tradingSystemOrder.config.percentageOfAlgorithmSize) === true', tradingSystemOrder) } - - /* - The Size calculation depends on how the user defined the size of the position. - The user could have defined the size of the position in Base Asset or Quoted Asset. - */ - switch (tradingEngineStage.stageDefinedIn.value) { - case 'Base Asset': { - /* Size in Base Asset */ - tradingEngineOrder.orderBaseAsset.size.value = algorithmSizeInBaseAsset * tradingSystemOrder.config.percentageOfAlgorithmSize / 100 - - /* Check that the Size calculated would not surpass Stage Target Size */ - if ( - tradingEngineOrder.orderBaseAsset.size.value + tradingEngineStage.stageBaseAsset.sizePlaced.value > - tradingEngineStage.stageBaseAsset.targetSize.value - ) { - tradingEngineOrder.orderBaseAsset.size.value = tradingEngineStage.stageBaseAsset.targetSize.value - tradingEngineStage.stageBaseAsset.sizePlaced.value - } - - /* Size in Quoted Asset */ - tradingEngineOrder.orderQuotedAsset.size.value = tradingEngineOrder.orderBaseAsset.size.value * tradingEngineOrder.rate.value - break + function requiredConfigurationValidation() { + /* Validate that this config exists */ + if (executionAlgorithm.config.percentageOfStageTargetSize === undefined) { + badDefinitionUnhandledException(undefined, 'executionAlgorithm.config.percentageOfStageTargetSize === undefined', executionAlgorithm) + } + if (isNaN(executionAlgorithm.config.percentageOfStageTargetSize) === true) { + badDefinitionUnhandledException(undefined, 'isNaN(executionAlgorithm.config.percentageOfStageTargetSize) === true', executionAlgorithm) } - case 'Quoted Asset': { - /* Size in Quoted Asset */ - tradingEngineOrder.orderQuotedAsset.size.value = algorithmSizeInQuotedAsset * tradingSystemOrder.config.percentageOfAlgorithmSize / 100 - /* Check that the Size calculated would not surpass Stage Target Size */ - if ( - tradingEngineOrder.orderQuotedAsset.size.value + tradingEngineStage.stageQuotedAsset.sizePlaced.value > - tradingEngineStage.stageQuotedAsset.targetSize.value - ) { - tradingEngineOrder.orderQuotedAsset.size.value = tradingEngineStage.stageQuotedAsset.targetSize.value - tradingEngineStage.stageQuotedAsset.sizePlaced.value + algorithmSizeInBaseAsset = tradingEngineStage.stageBaseAsset.targetSize.value * executionAlgorithm.config.percentageOfStageTargetSize / 100 + algorithmSizeInQuotedAsset = tradingEngineStage.stageQuotedAsset.targetSize.value * executionAlgorithm.config.percentageOfStageTargetSize / 100 + + /* Validate that this config exists */ + if (tradingSystemOrder.config.percentageOfAlgorithmSize === undefined) { + badDefinitionUnhandledException(undefined, 'tradingSystemOrder.config.percentageOfAlgorithmSize === undefined', tradingSystemOrder) + } + if (isNaN(tradingSystemOrder.config.percentageOfAlgorithmSize) === true) { + badDefinitionUnhandledException(undefined, 'isNaN(tradingSystemOrder.config.percentageOfAlgorithmSize) === true', tradingSystemOrder) + } + } + + function notPassingTargetSizeValidation() { + /* + The Size calculation depends on how the user defined the size of the position. + The user could have defined the size of the position in Base Asset or Quoted Asset. + */ + switch (tradingEngineStage.stageDefinedIn.value) { + case 'Base Asset': { + /* Size in Base Asset */ + tradingEngineOrder.orderBaseAsset.size.value = algorithmSizeInBaseAsset * tradingSystemOrder.config.percentageOfAlgorithmSize / 100 + tradingEngineOrder.orderBaseAsset.size.value = global.PRECISE(tradingEngineOrder.orderBaseAsset.size.value, 10) + + /* Check that the Size calculated would not surpass Stage Target Size */ + if ( + tradingEngineOrder.orderBaseAsset.size.value + tradingEngineStage.stageBaseAsset.sizePlaced.value > + tradingEngineStage.stageBaseAsset.targetSize.value + ) { + let previousValue = tradingEngineOrder.orderBaseAsset.size.value + tradingEngineOrder.orderBaseAsset.size.value = + tradingEngineStage.stageBaseAsset.targetSize.value - + tradingEngineStage.stageBaseAsset.sizePlaced.value + + tradingSystem.warnings.push( + [ + [tradingEngineStage.stageBaseAsset.targetSize.id, tradingEngineOrder.orderBaseAsset.size.id, tradingEngineStage.stageBaseAsset.sizePlaced.id], + 'Order size (' + previousValue + ') shrinked (' + tradingEngineOrder.orderBaseAsset.size.value + ') so that the Size Placed (' + tradingEngineStage.stageBaseAsset.sizePlaced.value + ') does not exceed the Target Size (' + tradingEngineStage.stageBaseAsset.targetSize.value + ') for the stage.' + ] + ) + } + + /* Size in Quoted Asset */ + tradingEngineOrder.orderQuotedAsset.size.value = tradingEngineOrder.orderBaseAsset.size.value * tradingEngineOrder.rate.value + break } + case 'Quoted Asset': { + /* Size in Quoted Asset */ + tradingEngineOrder.orderQuotedAsset.size.value = algorithmSizeInQuotedAsset * tradingSystemOrder.config.percentageOfAlgorithmSize / 100 + tradingEngineOrder.orderQuotedAsset.size.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.size.value, 10) - /* Size in Base Asset */ - tradingEngineOrder.orderBaseAsset.size.value = tradingEngineOrder.orderQuotedAsset.size.value / tradingEngineOrder.rate.value - break + /* Check that the Size calculated would not surpass Stage Target Size */ + if ( + tradingEngineOrder.orderQuotedAsset.size.value + tradingEngineStage.stageQuotedAsset.sizePlaced.value > + tradingEngineStage.stageQuotedAsset.targetSize.value + ) { + let previousValue = tradingEngineOrder.orderQuotedAsset.size.value + tradingEngineOrder.orderQuotedAsset.size.value = + tradingEngineStage.stageQuotedAsset.targetSize.value - + tradingEngineStage.stageQuotedAsset.sizePlaced.value + + tradingSystem.warnings.push( + [ + [tradingEngineStage.stageQuotedAsset.targetSize.id, tradingEngineOrder.orderQuotedAsset.size.id, tradingEngineStage.stageQuotedAsset.sizePlaced.id], + 'Order size (' + previousValue + ') shrinked (' + tradingEngineOrder.orderQuotedAsset.size.value + ') so that the Size Placed (' + tradingEngineStage.stageQuotedAsset.sizePlaced.value + ') does not exceed the Target Size (' + tradingEngineStage.stageQuotedAsset.targetSize.value + ') for the stage.' + ] + ) + } + + /* Size in Base Asset */ + tradingEngineOrder.orderBaseAsset.size.value = tradingEngineOrder.orderQuotedAsset.size.value / tradingEngineOrder.rate.value + + break + } + } + } + + function notNegativeBalanceValidation() { + /* Check that the Size calculated would not leave a negative balance */ + switch (true) { + case tradingSystemOrder.type === 'Market Buy Order' || tradingSystemOrder.type === 'Limit Buy Order': { + if ( + tradingEngine.current.episode.episodeQuotedAsset.balance.value - tradingEngineOrder.orderQuotedAsset.size.value < 0 + ) { + tradingSystem.warnings.push( + [ + [tradingEngine.current.episode.episodeQuotedAsset.balance.id, tradingEngineOrder.orderQuotedAsset.size.id], + 'Order Size Quoted Asset (' + tradingEngineOrder.orderQuotedAsset.size.value + ') changed to Balance Quoted Asset (' + tradingEngine.current.episode.episodeQuotedAsset.balance.value + ') so that the Balance does not drop below zero.' + ] + ) + + tradingEngineOrder.orderQuotedAsset.size.value = tradingEngine.current.episode.episodeQuotedAsset.balance.value + tradingEngineOrder.orderBaseAsset.size.value = tradingEngineOrder.orderQuotedAsset.size.value / tradingEngineOrder.rate.value + } + break + } + case tradingSystemOrder.type === 'Market Sell Order' || tradingSystemOrder.type === 'Limit Sell Order': { + if ( + tradingEngine.current.episode.episodeBaseAsset.balance.value - tradingEngineOrder.orderBaseAsset.size.value < 0 + ) { + tradingSystem.warnings.push( + [ + [tradingEngine.current.episode.episodeBaseAsset.balance.id, tradingEngineOrder.orderBaseAsset.size.id], + 'Order Size Base Asset (' + tradingEngineOrder.orderBaseAsset.size.value + ') changed to Balance Base Asset (' + tradingEngine.current.episode.episodeBaseAsset.balance.value + ') so that the Balance does not drop below zero.' + ] + ) + + tradingEngineOrder.orderBaseAsset.size.value = tradingEngine.current.episode.episodeBaseAsset.balance.value + tradingEngineOrder.orderQuotedAsset.size.value = tradingEngineOrder.orderBaseAsset.size.value * tradingEngineOrder.rate.value + } + break + } } } @@ -365,7 +476,7 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM async function createOrderAtExchange(tradingSystemOrder, tradingEngineOrder) { /* Filter by Session Type */ - switch (bot.SESSION.type) { + switch (bot.TRADING_SESSION.type) { case 'Backtesting Session': { return true } @@ -389,283 +500,10 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM } } - function simulateExchangeEvents(tradingEngineStage, tradingSystemOrder, tradingEngineOrder) { - - /* Filter by Session Type */ - switch (bot.SESSION.type) { - case 'Backtesting Session': { - break - } - case 'Live Trading Session': { - return - } - case 'Fordward Testing Session': { - return - } - case 'Paper Trading Session': { - break - } - } - - let previousQuotedAssetSize = tradingEngineOrder.orderQuotedAsset.size.value - let previousBaseAssetSizeFilled = tradingEngineOrder.orderBaseAsset.sizeFilled.value - let previousQuotedAssetSizeFilled = tradingEngineOrder.orderQuotedAsset.sizeFilled.value - let previousBaseAssetFeesPaid = tradingEngineOrder.orderBaseAsset.feesPaid.value - let previousQuotedAssetFeesPaid = tradingEngineOrder.orderQuotedAsset.feesPaid.value - - actualRateSimulation() - percentageFilledSimulation() - feesPaidSimulation() - sizeFilledSimulation() - - doTheAccounting( - tradingEngineStage, - tradingSystemOrder, - tradingEngineOrder, - previousBaseAssetSizeFilled, - previousQuotedAssetSizeFilled, - previousBaseAssetFeesPaid, - previousQuotedAssetFeesPaid - ) - - /* If the Stage is Closing and this order is still open, we need to cancel it now */ - if (tradingEngineStage.status.value === 'Closing' && tradingEngineOrder.status.value !== 'Closed') { - simulateCancelOrder(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, 'Closing Stage') - } - - function actualRateSimulation() { - /* Actual Rate Simulation */ - let calculatedBasedOnTradingSystem = false - - basedOnTradingSystem() - basedOnSessionParameters() - considerBestMatchWithOrderBook() - recalculateQuotedAsset() - recalculateSizePlaced() - - function basedOnTradingSystem() { - /* Based on the Trading System Definition */ - if (tradingSystemOrder.simulatedExchangeEvents !== undefined) { - if (tradingSystemOrder.simulatedExchangeEvents.simulatedActualRate !== undefined) { - if (tradingSystemOrder.simulatedExchangeEvents.simulatedActualRate.formula !== undefined) { - /* Calculate this only once for this order */ - if (tradingEngineOrder.orderStatistics.actualRate.value === tradingEngineOrder.orderStatistics.actualRate.config.initialValue) { - tradingEngineOrder.orderStatistics.actualRate.value = tradingSystem.formulas.get(tradingSystemOrder.simulatedExchangeEvents.simulatedActualRate.formula.id) - if (tradingEngineOrder.orderStatistics.actualRate.value !== undefined) { - calculatedBasedOnTradingSystem = true - } - } - } - } - } - } - - function basedOnSessionParameters() { - /* Based on the Session Parameters Definition */ - if (calculatedBasedOnTradingSystem === false) { - switch (tradingEngineOrder.type) { - case 'Market Order': { - /* Actual Rate is simulated based on the Session Paremeters */ - let slippageAmount = tradingEngineOrder.rate.value * sessionParameters.slippage.config.positionRate / 100 - switch (tradingSystemOrder.type) { - case 'Market Sell Order': { - tradingEngineOrder.orderStatistics.actualRate.value = tradingEngineOrder.rate.value - slippageAmount - break - } - case 'Market Buy Order': { - tradingEngineOrder.orderStatistics.actualRate.value = tradingEngineOrder.rate.value + slippageAmount - break - } - } - break - } - case 'Limit Order': { - /* In Limit Orders the actual rate is the rate of the order, there is no slippage */ - tradingEngineOrder.orderStatistics.actualRate.value = tradingEngineOrder.rate.value - break - } - } - } - } - - function considerBestMatchWithOrderBook() { - /* - Until here we have got the rate based on the formula definition or based on session parameters slippage. - THe last check is about watching what happened in the market. Let's remember that the exchange will - fill the order with the best possible matches at it's order book. That means that if the rate - we set for the order was too low (for a sale order) or too hight (for a buy order), the actual rate - should be better than expected. - */ - switch (true) { - case tradingSystemOrder.type === 'Market Buy Order' || tradingSystemOrder.type === 'Limit Buy Order': { - if (tradingEngineOrder.orderStatistics.actualRate.value > tradingEngine.current.episode.candle.max.value) { - tradingEngineOrder.orderStatistics.actualRate.value = tradingEngine.current.episode.candle.max.value - } - break - } - case tradingSystemOrder.type === 'Market Sell Order' || tradingSystemOrder.type === 'Limit Sell Order': { - if (tradingEngineOrder.orderStatistics.actualRate.value < tradingEngine.current.episode.candle.min.value) { - tradingEngineOrder.orderStatistics.actualRate.value = tradingEngine.current.episode.candle.min.value - } - break - } - } - tradingEngineOrder.orderStatistics.actualRate.value = global.PRECISE(tradingEngineOrder.orderStatistics.actualRate.value, 10) - } - - function recalculateQuotedAsset() { - /* - Recalculate the Quouted Asset Size based on the newly calculated Actual Rate. - */ - tradingEngineOrder.orderQuotedAsset.size.value = tradingEngineOrder.orderBaseAsset.size.value * tradingEngineOrder.orderStatistics.actualRate.value - tradingEngineOrder.orderQuotedAsset.size.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.size.value, 10) - } - - function recalculateSizePlaced() { - /* - Recaulculate the Size Placed Based on the newly calculated Quoted Asset Size. - */ - tradingEngineStage.stageQuotedAsset.sizePlaced.value = - tradingEngineStage.stageQuotedAsset.sizePlaced.value - - previousQuotedAssetSize - - tradingEngineStage.stageQuotedAsset.sizePlaced.value = - tradingEngineStage.stageQuotedAsset.sizePlaced.value + - tradingEngineOrder.orderQuotedAsset.size.value - - tradingEngineStage.stageQuotedAsset.sizePlaced.value = global.PRECISE(tradingEngineStage.stageQuotedAsset.sizePlaced.value, 10) - } - } - - function percentageFilledSimulation() { - /* - Order Filling Simulation: the way we are going to handle this is the following. - For buy orders we are going to check if the price was below the order rate, and - for sell orders we are going to check if the price was above the order rate. - - If we have a partial filling defined we are going to use it to estimate how much - of the order we will consider filled. - */ - let orderWasHit - switch (tradingSystemOrder.type) { - case 'Limit Buy Order': { - if (tradingEngine.current.episode.candle.min.value <= tradingEngineOrder.rate.value) { - orderWasHit = true - } - break - } - case 'Limit Sell Order': { - if (tradingEngine.current.episode.candle.max.value >= tradingEngineOrder.rate.value) { - orderWasHit = true - } - break - } - case 'Market Buy Order': { - orderWasHit = true - break - } - case 'Market Sell Order': { - orderWasHit = true - break - } - } - if (orderWasHit === true) { - let calculatedBasedOnTradingSystem = false - - /* Based on the Trading System Definition */ - if (tradingSystemOrder.simulatedExchangeEvents !== undefined) { - if (tradingSystemOrder.simulatedExchangeEvents.simulatedPartialFill !== undefined) { - if (tradingSystemOrder.simulatedExchangeEvents.simulatedPartialFill.config.fillProbability !== undefined) { - - /* Percentage Filled */ - let percentageFilled = tradingSystemOrder.simulatedExchangeEvents.simulatedPartialFill.config.fillProbability * 100 - if (tradingEngineOrder.orderStatistics.percentageFilled.value + percentageFilled > 100) { - percentageFilled = 100 - tradingEngineOrder.orderStatistics.percentageFilled.value - } - tradingEngineOrder.orderStatistics.percentageFilled.value = tradingEngineOrder.orderStatistics.percentageFilled.value + percentageFilled - tradingEngineOrder.orderStatistics.percentageFilled.value = global.PRECISE(tradingEngineOrder.orderStatistics.percentageFilled.value, 10) - calculatedBasedOnTradingSystem = true - } - } - } - /* - If there is no definition for this at the trading system we will assume orders - are 100% filled every single time. - */ - if (calculatedBasedOnTradingSystem === false) { - /* We will assume that 100% of the order was filled */ - tradingEngineOrder.orderStatistics.percentageFilled.value = 100 - } - /* Check if we need to close the order */ - if (tradingEngineOrder.orderStatistics.percentageFilled.value === 100) { - - /* Close this Order */ - tradingEngineOrder.status.value = 'Closed' - tradingEngineOrder.exitType.value = 'Filled' - - /* Initialize this */ - tradingEngine.current.episode.distanceToEvent.closeOrder.value = 1 - - updateEndsWithCycle(tradingEngineOrder) - tradingSystem.infos.push([tradingSystemOrder.id, 'percentageFilledSimulation -> Closing Order with Exit Type ' + tradingEngineOrder.exitType.value]) - } - } - } - - function feesPaidSimulation() { - /* - The way the Fess Paid are calculated is as follows: From the order size we apply the % of fee, - that would give us the total fees for that order. To that we apply the % filled of the orders, - that give use the final result of the fees paid for the current % filled. - - But before that, we will see if there is an specific definition on how to simulate the fees for - this order, or if there is not, we use the fees session parameters definitions. - */ - - /* Based on the Trading System Definition */ - if (tradingSystemOrder.simulatedExchangeEvents !== undefined) { - if (tradingSystemOrder.simulatedExchangeEvents.simulatedFeesPaid !== undefined) { - if (tradingSystemOrder.simulatedExchangeEvents.simulatedFeesPaid.config.percentage !== undefined) { - applyFeePercentage(tradingEngineOrder, tradingSystemOrder, tradingSystemOrder.simulatedExchangeEvents.simulatedFeesPaid.config.percentage) - return - } - } - } - - /* - Another way to simulate the Fees Paid is by using the Session Paremeters configuration for Fees. - */ - switch (tradingEngineOrder.type) { - case 'Market Order': { - applyFeePercentage(tradingEngineOrder, tradingSystemOrder, sessionParameters.feeStructure.config.taker) - break - } - case 'Limit Order': { - applyFeePercentage(tradingEngineOrder, tradingSystemOrder, sessionParameters.feeStructure.config.maker) - break - } - } - } - - function sizeFilledSimulation() { - /* Size Filled */ - tradingEngineOrder.orderBaseAsset.sizeFilled.value = - tradingEngineOrder.orderBaseAsset.size.value * - tradingEngineOrder.orderStatistics.percentageFilled.value / 100 - - tradingEngineOrder.orderQuotedAsset.sizeFilled.value = - tradingEngineOrder.orderQuotedAsset.size.value * - tradingEngineOrder.orderStatistics.percentageFilled.value / 100 - - tradingEngineOrder.orderBaseAsset.sizeFilled.value = global.PRECISE(tradingEngineOrder.orderBaseAsset.sizeFilled.value, 10) - tradingEngineOrder.orderQuotedAsset.sizeFilled.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.sizeFilled.value, 10) - } - } - async function checkExchangeEvents(tradingEngineStage, tradingSystemOrder, tradingEngineOrder) { /* Filter by Session Type */ - switch (bot.SESSION.type) { + switch (bot.TRADING_SESSION.type) { case 'Backtesting Session': { return true } @@ -683,7 +521,12 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM let order = await exchangeAPIModule.getOrder(tradingSystemOrder, tradingEngineOrder) if (order === undefined) { - tradingSystem.warnings.push([tradingSystemOrder.id, 'Could not verify the status of this order at the exchange.']) + tradingSystem.warnings.push( + [ + [tradingSystemOrder.id, tradingEngineOrder.id], + 'Could not verify the status of this order at the exchange.' + ] + ) return false } @@ -704,7 +547,7 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM await updateEndsWithCycle(tradingEngineOrder) tradingSystem.infos.push([tradingSystemOrder.id, 'checkExchangeEvents -> Closing Order with Exit Type ' + tradingEngineOrder.exitType.value]) - + await syncWithExchange(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order) return } @@ -767,20 +610,98 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM return true } - async function syncWithExchange(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order) { + function simulateCheckExchangeEvents(tradingEngineStage, tradingSystemOrder, tradingEngineOrder) { + + /* Filter by Session Type */ + switch (bot.TRADING_SESSION.type) { + case 'Backtesting Session': { + break + } + case 'Live Trading Session': { + return + } + case 'Fordward Testing Session': { + return + } + case 'Paper Trading Session': { + break + } + } + + simulateSyncWithExchange(tradingEngineStage, tradingSystemOrder, tradingEngineOrder) + + /* Check if we need to close the order */ + if (tradingEngineOrder.orderStatistics.percentageFilled.value === 100) { + + /* Close this Order */ + tradingEngineOrder.status.value = 'Closed' + tradingEngineOrder.exitType.value = 'Filled' + + /* Initialize this */ + tradingEngine.current.episode.distanceToEvent.closeOrder.value = 1 + + updateEndsWithCycle(tradingEngineOrder) + tradingSystem.infos.push([tradingSystemOrder.id, 'percentageFilledSimulation -> Closing Order with Exit Type ' + tradingEngineOrder.exitType.value]) + } + + /* If the Stage is Closing and this order is still open, we need to cancel it now */ + if (tradingEngineStage.status.value === 'Closing' && tradingEngineOrder.status.value !== 'Closed') { + simulateCancelOrder(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, 'Closing Stage') + } + } + + function simulateSyncWithExchange(tradingEngineStage, tradingSystemOrder, tradingEngineOrder) { - let previousBaseAssetSize = tradingEngineOrder.orderBaseAsset.size.value - let previousQuotedAssetSize = tradingEngineOrder.orderQuotedAsset.size.value let previousBaseAssetSizeFilled = tradingEngineOrder.orderBaseAsset.sizeFilled.value let previousQuotedAssetSizeFilled = tradingEngineOrder.orderQuotedAsset.sizeFilled.value let previousBaseAssetFeesPaid = tradingEngineOrder.orderBaseAsset.feesPaid.value let previousQuotedAssetFeesPaid = tradingEngineOrder.orderQuotedAsset.feesPaid.value - await actualSizeCalculation() - await actualRateCalculation() - await percentageFilledCalculation() - await feesPaidCalculation() - await sizeFilledCalculation() + const ORDERS_SIMULATIONS_MODULE = require('./OrdersSimulations.js') + let ordersSimulationsModule = ORDERS_SIMULATIONS_MODULE.newOrdersSimulations(bot, logger) + ordersSimulationsModule.initialize() + + ordersSimulationsModule.actualSizeSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) + ordersSimulationsModule.actualRateSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) + ordersSimulationsModule.feesToBePaidSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) + ordersSimulationsModule.percentageFilledSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) + ordersSimulationsModule.feesPaidSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) + ordersSimulationsModule.sizeFilledSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) + ordersSimulationsModule.amountReceivedSimulation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, applyFeePercentage) + + ordersSimulationsModule.finalize() + + doTheAccounting( + tradingEngineStage, + tradingSystemOrder, + tradingEngineOrder, + previousBaseAssetSizeFilled, + previousQuotedAssetSizeFilled, + previousBaseAssetFeesPaid, + previousQuotedAssetFeesPaid + ) + } + + async function syncWithExchange(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order) { + + let previousBaseAssetSizeFilled = tradingEngineOrder.orderBaseAsset.sizeFilled.value + let previousQuotedAssetSizeFilled = tradingEngineOrder.orderQuotedAsset.sizeFilled.value + let previousBaseAssetFeesPaid = tradingEngineOrder.orderBaseAsset.feesPaid.value + let previousQuotedAssetFeesPaid = tradingEngineOrder.orderQuotedAsset.feesPaid.value + + const ORDERS_CALCULATIONS_MODULE = require('./OrdersCalculations.js') + let ordersCalculationsModule = ORDERS_CALCULATIONS_MODULE.newOrdersCalculations(bot, logger) + ordersCalculationsModule.initialize() + + await ordersCalculationsModule.actualSizeCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) + await ordersCalculationsModule.actualRateCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) + await ordersCalculationsModule.feesToBePaidCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) + await ordersCalculationsModule.percentageFilledCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) + await ordersCalculationsModule.feesPaidCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) + await ordersCalculationsModule.sizeFilledCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) + await ordersCalculationsModule.amountReceivedCalculation(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, order, applyFeePercentage) + + ordersCalculationsModule.finalize() await doTheAccounting( tradingEngineStage, @@ -791,155 +712,6 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM previousBaseAssetFeesPaid, previousQuotedAssetFeesPaid ) - - return - - async function actualSizeCalculation() { - /* - When we submit the order to the exchange, we do it specifying the order size - in Base Asset. The exchange might take that size or might change it a little - bit if for example, if it does not accept as many decimals as we are sending. - If we identify the situation in which the size accepted is different than the - size we have accounted for, we need to make several adjustments so that the - accounting syncronizes with reality. - */ - if (tradingEngineOrder.orderBaseAsset.size.value === order.amount) { return } - /* - We will sync the Base Asset size, and recalculate the Quoted Asset size too. - */ - tradingEngineOrder.orderBaseAsset.size.value = order.amount - tradingEngineOrder.orderQuotedAsset.size.value = tradingEngineOrder.orderBaseAsset.size.value * tradingEngineOrder.rate.value - - tradingEngineOrder.orderBaseAsset.size.value = global.PRECISE(tradingEngineOrder.orderBaseAsset.size.value, 10) - tradingEngineOrder.orderQuotedAsset.size.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.size.value, 10) - /* - Finaly we need to unaccount the previous size placed and correctly account - for the the new sizes we now have. - */ - tradingEngineStage.stageBaseAsset.sizePlaced.value = - tradingEngineStage.stageBaseAsset.sizePlaced.value - - previousBaseAssetSize - - tradingEngineStage.stageBaseAsset.sizePlaced.value = - tradingEngineStage.stageBaseAsset.sizePlaced.value + - tradingEngineOrder.orderBaseAsset.size.value - - tradingEngineStage.stageQuotedAsset.sizePlaced.value = - tradingEngineStage.stageQuotedAsset.sizePlaced.value - - previousQuotedAssetSize - - tradingEngineStage.stageQuotedAsset.sizePlaced.value = - tradingEngineStage.stageQuotedAsset.sizePlaced.value + - tradingEngineOrder.orderQuotedAsset.size.value - - tradingEngineStage.stageBaseAsset.sizePlaced.value = global.PRECISE(tradingEngineStage.stageBaseAsset.sizePlaced.value, 10) - tradingEngineStage.stageQuotedAsset.sizePlaced.value = global.PRECISE(tradingEngineStage.stageQuotedAsset.sizePlaced.value, 10) - - previousBaseAssetSize = tradingEngineOrder.orderBaseAsset.size.value - previousQuotedAssetSize = tradingEngineOrder.orderQuotedAsset.size.value - } - - async function actualRateCalculation() { - /* - Actual Rate Calculation: Let's remember that the exchange may not take the rate - we send it with the order (in Limit orders) and for any reason change it a little bit. - For example we might have sent a rate with too many decimals and the exchange truncate - them. Another thing that might happen is that we send a rate that is matching orders already - at the order book. In this case we would get an actual rate not only different than the one - we sent, but also, better since the trading engine inside the exchange would match out orders - with the first orders best by price. - - In the case of Market Orders we don't even send a rate. For all those reason - we get and store the Actual Rate at which the order was filled. - */ - if (order.average !== undefined) { - /* - We will use the average whenever is available. - */ - tradingEngineOrder.orderStatistics.actualRate.value = order.average - } else { - /* - Otherwise we use the order.price. - */ - tradingEngineOrder.orderStatistics.actualRate.value = order.price - } - - tradingEngineOrder.orderStatistics.actualRate.value = global.PRECISE(tradingEngineOrder.orderStatistics.actualRate.value, 10) - - if (tradingEngineOrder.orderStatistics.actualRate.value === tradingEngineOrder.rate.value) { return } - /* - Now we know the Actual Rate at which the order was filled. Since the actual rate - is not the same as the Rate we defined for the order, we need to syncronize - the Order Size for Quoted Asset since it was calculated with the Order Size that we - now know it is not the once really used at the exchange. We will recalculate the - Order Size in Quoted Asset and not in Base Asset, since the Order Size in Base Asset - is the one we sent to the exchange, so that is fixed, regardless of how we initially - got the Order Size in Base Asset, eigher from direct imput from the user or calculated - from the Order Size in Quoted Asset input it by the user. So here we go. - */ - tradingEngineOrder.orderQuotedAsset.size.value = tradingEngineOrder.orderBaseAsset.size.value * tradingEngineOrder.orderStatistics.actualRate.value - tradingEngineOrder.orderQuotedAsset.size.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.size.value, 10) - /* - Great, this up-to-date order size in Base Asset and the Actual Rate can now be used for example to calculate - the orders fees in Quoted Assets, and it is perfectly accurate. The next problem - we need to fix is that the sizePlaced for Quoted Asset that we already have calculated - was based on the Order Rate, not the Actual Rate, so we need to fix that by anuncounting - the previous size and accounting the new one, all this only for Quoted Asset. - */ - tradingEngineStage.stageQuotedAsset.sizePlaced.value = - tradingEngineStage.stageQuotedAsset.sizePlaced.value - - previousQuotedAssetSize - - tradingEngineStage.stageQuotedAsset.sizePlaced.value = - tradingEngineStage.stageQuotedAsset.sizePlaced.value + - tradingEngineOrder.orderQuotedAsset.size.value - - tradingEngineStage.stageQuotedAsset.sizePlaced.value = global.PRECISE(tradingEngineStage.stageQuotedAsset.sizePlaced.value, 10) - } - - async function percentageFilledCalculation() { - /* Percentage Filled Calculation */ - tradingEngineOrder.orderStatistics.percentageFilled.value = order.filled * 100 / (order.filled + order.remaining) - tradingEngineOrder.orderStatistics.percentageFilled.value = global.PRECISE(tradingEngineOrder.orderStatistics.percentageFilled.value, 10) - } - - async function feesPaidCalculation() { - /* - The exchange fees are taken from the Base Asset or the Quoted Asset depending if we - are buying or selling. Within the order information received we can not see the fees paid so we need to - calculate them by ourselves. To do this it is needed to be used the Session Fees Parameters. - */ - switch (tradingEngineOrder.type) { - case 'Market Order': { - await applyFeePercentage(tradingEngineOrder, tradingSystemOrder, sessionParameters.feeStructure.config.taker) - break - } - case 'Limit Order': { - await applyFeePercentage(tradingEngineOrder, tradingSystemOrder, sessionParameters.feeStructure.config.maker) - break - } - } - } - - async function sizeFilledCalculation() { - /* - CCXT returns order.filled with an amount denominated in Base Asset. We will - take it from there for our Order Base Asset. The amount in order.filled does - not account for the fees, which is exactly what we need in this case. - */ - tradingEngineOrder.orderBaseAsset.sizeFilled.value = order.filled - tradingEngineOrder.orderBaseAsset.sizeFilled.value = global.PRECISE(tradingEngineOrder.orderBaseAsset.sizeFilled.value, 10) - /* - For Quoted Asset, CCXT does not return any field with the size filled, so we - need to calculate that by ourselves. Since we know the Size in Quoted Asset thet - is already adjusted with the Actual Rate and the Percentage Filled, we will use all - this in this formula to get the Size Filled in Quoted Asset. - */ - tradingEngineOrder.orderQuotedAsset.sizeFilled.value = - tradingEngineOrder.orderQuotedAsset.size.value * tradingEngineOrder.orderStatistics.percentageFilled.value / 100 - - tradingEngineOrder.orderQuotedAsset.sizeFilled.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.sizeFilled.value, 10) - } } async function doTheAccounting( @@ -1003,41 +775,7 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM /* Balances Update */ switch (true) { /* - Let's remember that for Buy orders, the fees are paid in Base Asset. So we - actually don't receive the whole Size Filled in Base Asset, but the Size - Filled minus the Fees Paid. For the case of Quoted Asset, we actually - need to susbstract the whole Size Filled from the balance. - */ - case tradingSystemOrder.type === 'Market Buy Order' || tradingSystemOrder.type === 'Limit Buy Order': { - - /* Balance Base Asset: Undo the previous accounting */ - tradingEngine.current.episode.episodeBaseAsset.balance.value = - tradingEngine.current.episode.episodeBaseAsset.balance.value - - previousBaseAssetSizeFilled + - previousBaseAssetFeesPaid - - /* Balance Base Asset: Account the current filling and fees */ - tradingEngine.current.episode.episodeBaseAsset.balance.value = - tradingEngine.current.episode.episodeBaseAsset.balance.value + - tradingEngineOrder.orderBaseAsset.sizeFilled.value - - tradingEngineOrder.orderBaseAsset.feesPaid.value - - /* Balance Quoted Asset: Undo the previous accounting */ - tradingEngine.current.episode.episodeQuotedAsset.balance.value = - tradingEngine.current.episode.episodeQuotedAsset.balance.value + - previousQuotedAssetSizeFilled - - /* Balance Quoted Asset: Account the current filling and fees */ - tradingEngine.current.episode.episodeQuotedAsset.balance.value = - tradingEngine.current.episode.episodeQuotedAsset.balance.value - - tradingEngineOrder.orderQuotedAsset.sizeFilled.value - break - } - /* - For Sell orders, the fees are being paid in Quote Asset, so we never - acutally receive the whole Filled Size in Quote Asset, but the Filled - Size minus the Fees Paid. But we do expend the whole Filled Size in - Base Asset. + For Sell orders, the fees are being paid in Quote Asset. */ case tradingSystemOrder.type === 'Market Sell Order' || tradingSystemOrder.type === 'Limit Sell Order': { @@ -1064,6 +802,34 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM tradingEngineOrder.orderQuotedAsset.feesPaid.value break } + /* + Let's remember that for Buy orders, the fees are paid in Base Asset. + */ + case tradingSystemOrder.type === 'Market Buy Order' || tradingSystemOrder.type === 'Limit Buy Order': { + + /* Balance Base Asset: Undo the previous accounting */ + tradingEngine.current.episode.episodeBaseAsset.balance.value = + tradingEngine.current.episode.episodeBaseAsset.balance.value - + previousBaseAssetSizeFilled + + previousBaseAssetFeesPaid + + /* Balance Base Asset: Account the current filling and fees */ + tradingEngine.current.episode.episodeBaseAsset.balance.value = + tradingEngine.current.episode.episodeBaseAsset.balance.value + + tradingEngineOrder.orderBaseAsset.sizeFilled.value - + tradingEngineOrder.orderBaseAsset.feesPaid.value + + /* Balance Quoted Asset: Undo the previous accounting */ + tradingEngine.current.episode.episodeQuotedAsset.balance.value = + tradingEngine.current.episode.episodeQuotedAsset.balance.value + + previousQuotedAssetSizeFilled + + /* Balance Quoted Asset: Account the current filling and fees */ + tradingEngine.current.episode.episodeQuotedAsset.balance.value = + tradingEngine.current.episode.episodeQuotedAsset.balance.value - + tradingEngineOrder.orderQuotedAsset.sizeFilled.value + break + } } tradingEngine.current.episode.episodeBaseAsset.balance.value = global.PRECISE(tradingEngine.current.episode.episodeBaseAsset.balance.value, 10) tradingEngine.current.episode.episodeQuotedAsset.balance.value = global.PRECISE(tradingEngine.current.episode.episodeQuotedAsset.balance.value, 10) @@ -1073,7 +839,7 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM function simulateCancelOrder(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, exitType) { /* Filter by Session Type */ - switch (bot.SESSION.type) { + switch (bot.TRADING_SESSION.type) { case 'Backtesting Session': { break } @@ -1104,7 +870,7 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM async function exchangeCancelOrder(tradingEngineStage, tradingSystemOrder, tradingEngineOrder, exitType) { /* Filter by Session Type */ - switch (bot.SESSION.type) { + switch (bot.TRADING_SESSION.type) { case 'Backtesting Session': { return } @@ -1134,7 +900,12 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM let order = await exchangeAPIModule.getOrder(tradingSystemOrder, tradingEngineOrder) if (order === undefined) { - tradingSystem.warnings.push([tradingSystemOrder.id, 'Could not verify the status of this order at the exchange, and syncronize the accounting.']) + tradingSystem.warnings.push( + [ + [tradingSystemOrder.id, tradingEngineOrder.id], + 'Could not verify the status of this order at the exchange, and syncronize the accounting.' + ] + ) return false } @@ -1164,15 +935,15 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM Size Placed accumulates for each asset, the order size placed at the exchange. A cancelation means that only the part filled can be considered placed, so we need to substract from the stage size the remainder. To achieve this with the information we currently have, we are going first - to unaccount the order size, and the account only the sizeFilled + the feesPaid. + to unaccount the order actual size, and the account only the sizeFilled + the feesPaid. */ tradingEngineStage.stageBaseAsset.sizePlaced.value = tradingEngineStage.stageBaseAsset.sizePlaced.value - - tradingEngineOrder.orderBaseAsset.size.value + tradingEngineOrder.orderBaseAsset.actualSize.value tradingEngineStage.stageQuotedAsset.sizePlaced.value = tradingEngineStage.stageQuotedAsset.sizePlaced.value - - tradingEngineOrder.orderQuotedAsset.size.value + tradingEngineOrder.orderQuotedAsset.actualSize.value tradingEngineStage.stageBaseAsset.sizePlaced.value = tradingEngineStage.stageBaseAsset.sizePlaced.value + @@ -1186,32 +957,40 @@ exports.newTradingOrders = function newTradingOrders(bot, logger, tradingEngineM tradingEngineStage.stageQuotedAsset.sizePlaced.value = global.PRECISE(tradingEngineStage.stageQuotedAsset.sizePlaced.value, 10) } - async function applyFeePercentage(tradingEngineOrder, tradingSystemOrder, feePercentage) { + async function applyFeePercentage(feesNodePropertyName, tradingEngineOrder, tradingSystemOrder, feePercentage, percentageFilled) { + /* + The exchange fees are taken from the Base Asset or the Quoted Asset depending if we + are buying or selling. + */ + let feesNode + switch (true) { case tradingSystemOrder.type === 'Market Buy Order' || tradingSystemOrder.type === 'Limit Buy Order': { /* - In this case the fees are taken from the Base Asset you receive. + In this case the fees are taken from the Base Asset you receive as a result of the trade at the exchange. */ - tradingEngineOrder.orderBaseAsset.feesPaid.value = - tradingEngineOrder.orderBaseAsset.size.value * + feesNode = tradingEngineOrder.orderBaseAsset[feesNodePropertyName] + feesNode.value = + tradingEngineOrder.orderBaseAsset.actualSize.value * feePercentage / 100 * - tradingEngineOrder.orderStatistics.percentageFilled.value / 100 + percentageFilled / 100 break } case tradingSystemOrder.type === 'Market Sell Order' || tradingSystemOrder.type === 'Limit Sell Order': { /* - In this case the fees are taken from the Quoted Asset you receive. + In this case the fees are taken from the Quoted Asset you receive as a result of the trade at the exchange. */ - tradingEngineOrder.orderQuotedAsset.feesPaid.value = - tradingEngineOrder.orderQuotedAsset.size.value * + feesNode = tradingEngineOrder.orderQuotedAsset[feesNodePropertyName] + feesNode.value = + tradingEngineOrder.orderQuotedAsset.actualSize.value * feePercentage / 100 * - tradingEngineOrder.orderStatistics.percentageFilled.value / 100 + percentageFilled / 100 break } } - tradingEngineOrder.orderBaseAsset.feesPaid.value = global.PRECISE(tradingEngineOrder.orderBaseAsset.feesPaid.value, 10) - tradingEngineOrder.orderQuotedAsset.feesPaid.value = global.PRECISE(tradingEngineOrder.orderQuotedAsset.feesPaid.value, 10) + feesNode.value = global.PRECISE(feesNode.value, 10) + feesNode.value = global.PRECISE(feesNode.value, 10) } async function checkOrderEvent(event, order, executionAlgorithm, executionNode) { diff --git a/LowFrequencyTrading/TradingOutput.js b/LowFrequencyTrading/TradingOutput.js index 1f582d090..7cd76c81c 100644 --- a/LowFrequencyTrading/TradingOutput.js +++ b/LowFrequencyTrading/TradingOutput.js @@ -3,7 +3,7 @@ exports.newTradingOutput = function newTradingOutput(bot, logger, tradingEngineM This module will load if necesary all the data outputs so that they can be appended with new records if needed. After running the simulation, it will save all the data outputs. */ - const MODULE_NAME = 'Trading Bot' + const MODULE_NAME = 'Trading Output' let thisObject = { initialize: initialize, @@ -133,7 +133,7 @@ exports.newTradingOutput = function newTradingOutput(bot, logger, tradingEngineM if (dataset.config.type === 'Market Files') { let fileName = 'Data.json' - let filePath = bot.filePathRoot + '/Output/' + bot.SESSION.folderName + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + let filePath = bot.filePathRoot + '/Output/' + bot.TRADING_SESSION.folderName + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel await readOutputFile(fileName, filePath, dataset.parentNode.config.codeName) } @@ -149,7 +149,7 @@ exports.newTradingOutput = function newTradingOutput(bot, logger, tradingEngineM let dateForPath = tradingProcessDate.getUTCFullYear() + '/' + utilities.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '/' + utilities.pad(tradingProcessDate.getUTCDate(), 2); let fileName = 'Data.json' - let filePath = bot.filePathRoot + '/Output/' + bot.SESSION.folderName + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath + let filePath = bot.filePathRoot + '/Output/' + bot.TRADING_SESSION.folderName + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath await readOutputFile(fileName, filePath, dataset.parentNode.config.codeName) } @@ -190,7 +190,7 @@ exports.newTradingOutput = function newTradingOutput(bot, logger, tradingEngineM if (dataset.config.type === 'Market Files') { let fileName = 'Data.json' - let filePath = bot.filePathRoot + '/Output/' + bot.SESSION.folderName + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + let filePath = bot.filePathRoot + '/Output/' + bot.TRADING_SESSION.folderName + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel await writeOutputFile(fileName, filePath, dataset.parentNode.config.codeName) } @@ -206,7 +206,7 @@ exports.newTradingOutput = function newTradingOutput(bot, logger, tradingEngineM let dateForPath = tradingProcessDate.getUTCFullYear() + '/' + utilities.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '/' + utilities.pad(tradingProcessDate.getUTCDate(), 2); let fileName = 'Data.json' - let filePath = bot.filePathRoot + '/Output/' + bot.SESSION.folderName + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath + let filePath = bot.filePathRoot + '/Output/' + bot.TRADING_SESSION.folderName + '/' + dataset.parentNode.config.codeName + '/' + dataset.config.codeName + '/' + timeFrameLabel + "/" + dateForPath await writeOutputFile(fileName, filePath, dataset.parentNode.config.codeName) } diff --git a/LowFrequencyTrading/TradingPosition.js b/LowFrequencyTrading/TradingPosition.js index 8476a4685..8c606d203 100644 --- a/LowFrequencyTrading/TradingPosition.js +++ b/LowFrequencyTrading/TradingPosition.js @@ -31,7 +31,7 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng function initialize() { tradingSystem = bot.simulationState.tradingSystem tradingEngine = bot.simulationState.tradingEngine - sessionParameters = bot.SESSION.parameters + sessionParameters = bot.TRADING_SESSION.tradingParameters } function finalize() { @@ -51,11 +51,14 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng function openPosition(situationName) { + /* Starting begin and end */ + tradingEngine.current.position.begin.value = tradingEngine.current.episode.cycle.lastBegin.value + tradingEngine.current.position.end.value = tradingEngine.current.episode.cycle.lastEnd.value + /* Recording the opening at the Trading Engine Data Structure */ tradingEngine.current.position.status.value = 'Open' tradingEngine.current.position.serialNumber.value = tradingEngine.current.episode.episodeCounters.positions.value + 1 tradingEngine.current.position.identifier.value = global.UNIQUE_ID() - tradingEngine.current.position.begin.value = tradingEngine.current.episode.candle.end.value tradingEngine.current.position.beginRate.value = tradingEngine.current.episode.candle.close.value tradingEngine.current.position.positionBaseAsset.beginBalance.value = tradingEngine.current.episode.episodeBaseAsset.balance.value tradingEngine.current.position.positionQuotedAsset.beginBalance.value = tradingEngine.current.episode.episodeQuotedAsset.balance.value @@ -79,11 +82,20 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng function closingPosition(exitType) { tradingEngine.current.position.status.value = 'Closing' tradingEngine.current.position.exitType.value = exitType + + /* + By the time we figured out that the stop loss or take profit were hit, or + for whatever reason the position needs to be closed, the take profit and + stop loss values for the next candle have already been calculated. In order + to avoid being plotted, we will put them in zero. + */ + tradingEngine.current.position.stopLoss.value = 0 + tradingEngine.current.position.takeProfit.value = 0 } function closePosition() { tradingEngine.current.position.status.value = 'Closed' - tradingEngine.current.position.end.value = tradingEngine.current.episode.candle.end.value + tradingEngine.current.position.end.value = tradingEngine.current.episode.cycle.lastEnd.value tradingEngine.current.position.endRate.value = tradingEngine.current.episode.candle.close.value tradingEngine.current.position.positionBaseAsset.endBalance.value = tradingEngine.current.episode.episodeBaseAsset.balance.value tradingEngine.current.position.positionQuotedAsset.endBalance.value = tradingEngine.current.episode.episodeQuotedAsset.balance.value @@ -94,28 +106,68 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng tradingEngineModule.cloneValues(tradingEngine.current.position, tradingEngine.last.position) cycleBasedStatistics() - + /* Updating Hits & Fails */ if (tradingEngine.current.position.positionBaseAsset.profitLoss.value > 0) { tradingEngine.current.episode.episodeBaseAsset.hits.value++ - } + } if (tradingEngine.current.position.positionBaseAsset.profitLoss.value < 0) { tradingEngine.current.episode.episodeBaseAsset.fails.value++ } if (tradingEngine.current.position.positionQuotedAsset.profitLoss.value > 0) { tradingEngine.current.episode.episodeQuotedAsset.hits.value++ - } + } if (tradingEngine.current.position.positionQuotedAsset.profitLoss.value < 0) { tradingEngine.current.episode.episodeQuotedAsset.fails.value++ } } function applyStopLossFormula(formulas, formulaId) { + updateStopLossTakeProfitFinalValue(tradingEngine.current.position.stopLoss) tradingEngine.current.position.stopLoss.value = formulas.get(formulaId) + updateStopLossTakeProfitInitialValue(tradingEngine.current.position.stopLoss) + updateStopLossTakeProfitBeginEnd(tradingEngine.current.position.stopLoss) } function applyTakeProfitFormula(formulas, formulaId) { + updateStopLossTakeProfitFinalValue(tradingEngine.current.position.takeProfit) tradingEngine.current.position.takeProfit.value = formulas.get(formulaId) + updateStopLossTakeProfitInitialValue(tradingEngine.current.position.takeProfit) + updateStopLossTakeProfitBeginEnd(tradingEngine.current.position.takeProfit) + } + + function updateStopLossTakeProfitInitialValue(node) { + /* + We store the first Stop Loss or Take Profit value in a separate node. + To know if it is the first one we compare its current value to the initial value. + */ + if (node.initialValue.value === node.initialValue.config.initialValue) { + node.initialValue.value = node.value + } + } + + function updateStopLossTakeProfitFinalValue(node) { + /* + We set the final value just before calcualting the new value, since the new calculated + value is going to be discarded if the position is closed after it is calculated, but + during the same candle. + */ + node.finalValue.value = node.value + } + + function updateStopLossTakeProfitBeginEnd(node) { + /* + Both the Stop Loss and the Take Profit have their own Begin and End. + Ther reason for this is because they represent targets to be checked + at the next candle, so it does not apply that they share the begin and + end of the position itself. + */ + node.begin.value = + tradingEngine.current.episode.candle.begin.value + + sessionParameters.timeFrame.config.value + node.end.value = + tradingEngine.current.episode.candle.end.value + + sessionParameters.timeFrame.config.value } function updateStopLoss(phase) { @@ -186,7 +238,7 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng const message = 'Target Size In Base Asset cannot be undefined. Fix this please.' badDefinitionUnhandledException(undefined, message, tradingSystemStageNode.initialTargets.targetSizeInBaseAsset.formula) } - if (value === undefined) { + if (value === 0) { const message = 'Target Size In Base Asset cannot be zero. Fix this please.' badDefinitionUnhandledException(undefined, message, tradingSystemStageNode.initialTargets.targetSizeInBaseAsset.formula) } @@ -222,7 +274,7 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng const message = 'Target Size In Quoted Asset cannot be undefined. Fix this please.' badDefinitionUnhandledException(undefined, message, tradingSystemStageNode.initialTargets.targetSizeInQuotedAsset.formula) } - if (value === undefined) { + if (value === 0) { const message = 'Target Size In Quoted Asset cannot be zero. Fix this please.' badDefinitionUnhandledException(undefined, message, tradingSystemStageNode.initialTargets.targetSizeInQuotedAsset.formula) } @@ -244,8 +296,8 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng /* Remember how the end user defined this stage. */ tradingEngineStageNode.stageDefinedIn.value = 'Quoted Asset' } else { - const errorText = 'You need to specify a Formula for this.' - badDefinitionUnhandledException(undefined, message, tradingSystemStageNode.initialTargets.targetSizeInQuotedAsset.formula) + const message = 'You need to specify a Formula for this.' + badDefinitionUnhandledException(undefined, message, tradingSystemStageNode.initialTargets.targetSizeInQuotedAsset) } } } @@ -253,7 +305,7 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng function updateEnds() { if (tradingEngine.current.position.status.value === 'Open') { - tradingEngine.current.position.end.value = tradingEngine.current.episode.candle.end.value + tradingEngine.current.position.end.value = tradingEngine.current.position.end.value + sessionParameters.timeFrame.config.value tradingEngine.current.position.endRate.value = tradingEngine.current.episode.candle.close.value tradingEngine.current.position.positionBaseAsset.endBalance.value = tradingEngine.current.episode.episodeBaseAsset.balance.value tradingEngine.current.position.positionQuotedAsset.endBalance.value = tradingEngine.current.episode.episodeQuotedAsset.balance.value @@ -305,7 +357,7 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng /* Hit Fail Calculation */ if (tradingEngine.current.position.positionBaseAsset.ROI.value > 0) { tradingEngine.current.position.positionBaseAsset.hitFail.value = 'Hit' - } + } if (tradingEngine.current.position.positionBaseAsset.ROI.value < 0) { tradingEngine.current.position.positionBaseAsset.hitFail.value = 'Fail' } @@ -314,7 +366,7 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng } if (tradingEngine.current.position.positionQuotedAsset.ROI.value > 0) { tradingEngine.current.position.positionQuotedAsset.hitFail.value = 'Hit' - } + } if (tradingEngine.current.position.positionQuotedAsset.ROI.value < 0) { tradingEngine.current.position.positionQuotedAsset.hitFail.value = 'Fail' } @@ -348,7 +400,7 @@ exports.newTradingPosition = function newTradingPosition(bot, logger, tradingEng /* Hit Fail Calculation */ if (tradingEngine.current.position.positionStatistics.ROI.value > 0) { tradingEngine.current.position.positionStatistics.hitFail.value = 'Hit' - } + } if (tradingEngine.current.position.positionStatistics.ROI.value < 0) { tradingEngine.current.position.positionStatistics.hitFail.value = 'Fail' } diff --git a/LowFrequencyTrading/TradingProcess.js b/LowFrequencyTrading/TradingProcess.js index bddb9883a..1032c04ea 100644 --- a/LowFrequencyTrading/TradingProcess.js +++ b/LowFrequencyTrading/TradingProcess.js @@ -103,7 +103,7 @@ This variable tell us which day we are standing at, specially while working with Daily Files. From this Date is that we are going to load the Daily Files. */ - bot.simulationState.tradingEngine.current.episode.processDate.value = global.REMOVE_TIME(bot.SESSION.parameters.timeRange.config.initialDatetime).valueOf() + bot.simulationState.tradingEngine.current.episode.processDate.value = global.REMOVE_TIME(bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime).valueOf() } /* @@ -119,7 +119,7 @@ await processSingleFiles() if (await processMarketFiles() === false) { - bot.sessionHeartBeat(undefined, undefined, 'Waiting for Data Mining to be run') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waiting for Data Mining to be run') callBackFunction(global.DEFAULT_RETRY_RESPONSE) return } @@ -141,17 +141,17 @@ With all the indicators data files loaded, we will build the chart object data structure that will be used in user-defied conditions and formulas. */ - bot.sessionHeartBeat(undefined, undefined, 'Waking up') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waking up') buildCharts(chart) if (checkThereAreCandles(chart) === true) { - bot.sessionHeartBeat(undefined, undefined, 'Running') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Running') await generateOutput(chart) - bot.sessionHeartBeat(undefined, undefined, 'Saving') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Saving') await writeProcessFiles() - bot.sessionHeartBeat(undefined, undefined, 'Sleeping') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Sleeping') } else { - bot.sessionHeartBeat(undefined, undefined, 'Waiting for Data Mining to be up to date') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waiting for Data Mining to be up to date. No candles found at.') callBackFunction(global.DEFAULT_RETRY_RESPONSE) return } @@ -159,7 +159,7 @@ } else { /* We are processing Daily Files */ do { - global.EMIT_SESSION_STATUS(bot.SESSION_STATUS, bot.sessionKey) + global.EMIT_SESSION_STATUS(bot.TRADING_SESSION_STATUS, bot.TRADING_SESSIONKey) /* We update the Trading Process Date with the date calculated at the simulation. We will use this date to load indicator and output files. After that we will @@ -171,9 +171,9 @@ if (checkStopTaskGracefully() === false) { break } if (checkStopProcessing() === false) { break } - bot.sessionHeartBeat(undefined, undefined, 'Waking up') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waking up') if (await processDailyFiles() === false) { - bot.sessionHeartBeat(undefined, undefined, 'Waiting for Data Mining to be run') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waiting for Data Mining to be run') callBackFunction(global.DEFAULT_RETRY_RESPONSE) return } @@ -186,13 +186,13 @@ The process of generating the output includes the trading simulation. */ if (checkThereAreCandles(chart) === true) { - bot.sessionHeartBeat(undefined, undefined, 'Running') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Running') await generateOutput(chart) - bot.sessionHeartBeat(undefined, undefined, 'Saving') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Saving') await writeProcessFiles() - bot.sessionHeartBeat(undefined, undefined, 'Sleeping') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Sleeping') } else { - bot.sessionHeartBeat(undefined, undefined, 'Waiting for Data Mining to be up to date') + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waiting for Data Mining to be up to date') callBackFunction(global.DEFAULT_RETRY_RESPONSE) return } @@ -219,7 +219,7 @@ callBackFunction(global.DEFAULT_OK_RESPONSE) function checkThereAreCandles(chart) { - let sessionParameters = bot.SESSION.parameters + let sessionParameters = bot.TRADING_SESSION.tradingParameters let propertyName = 'at' + sessionParameters.timeFrame.config.label.replace('-', '') let candles = chart[propertyName].candles @@ -236,7 +236,7 @@ let statusReport; /* We are going to use the start date as beging of market date. */ - contextVariables.dateBeginOfMarket = global.REMOVE_TIME(bot.SESSION.parameters.timeRange.config.initialDatetime) + contextVariables.dateBeginOfMarket = global.REMOVE_TIME(bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime) /* Here we get the status report from the bot who knows which is the end of the market. */ @@ -371,7 +371,7 @@ dataFiles = new Map(); - if (bot.SESSION.parameters.timeFrame.config.label === timeFrameLabel) { + if (bot.TRADING_SESSION.tradingParameters.timeFrame.config.label === timeFrameLabel) { currentTimeFrame = global.marketFilesPeriods[n][0]; currentTimeFrameLabel = global.marketFilesPeriods[n][1]; } @@ -384,7 +384,7 @@ } if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.config.singularVariableName) !== true) { - if (!(bot.SESSION.parameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { + if (!(bot.TRADING_SESSION.tradingParameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { continue } } @@ -431,8 +431,8 @@ let dataRecord = dataFile[i] let begin = dataRecord[beginIndex] let end = dataRecord[endIndex] - if (end + timeFrame < bot.SESSION.parameters.timeRange.config.initialDatetime - 1) { continue } // /1 because we need the previous closed element - if (begin > bot.SESSION.parameters.timeRange.config.finalDatetime) { continue } + if (end + timeFrame < bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime - 1) { continue } // /1 because we need the previous closed element + if (begin > bot.TRADING_SESSION.tradingParameters.timeRange.config.finalDatetime) { continue } result.push(dataRecord) } return result @@ -468,7 +468,7 @@ } } - if (bot.SESSION.parameters.timeFrame.config.label === timeFrameLabel) { + if (bot.TRADING_SESSION.tradingParameters.timeFrame.config.label === timeFrameLabel) { currentTimeFrame = global.dailyFilePeriods[n][0]; currentTimeFrameLabel = global.dailyFilePeriods[n][1]; } @@ -488,7 +488,7 @@ } if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.config.singularVariableName) !== true) { - if (!(bot.SESSION.parameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { + if (!(bot.TRADING_SESSION.tradingParameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { continue } } @@ -653,7 +653,7 @@ let fileContent = JSON.stringify(dataRange); let fileName = '/Data.Range.json'; - let filePath = bot.filePathRoot + "/Output/" + bot.SESSION.folderName + "/" + productCodeName + "/" + 'Multi-Period-Daily' + fileName; + let filePath = bot.filePathRoot + "/Output/" + bot.TRADING_SESSION.folderName + "/" + productCodeName + "/" + 'Multi-Period-Daily' + fileName; let response = await fileStorage.asyncCreateTextFile(filePath, fileContent + '\n') @@ -692,7 +692,7 @@ let fileContent = JSON.stringify(timeFramesArray) let fileName = '/Time.Frames.json'; - let filePath = bot.filePathRoot + "/Output/" + bot.SESSION.folderName + "/" + productCodeName + "/" + processType + fileName; + let filePath = bot.filePathRoot + "/Output/" + bot.TRADING_SESSION.folderName + "/" + productCodeName + "/" + processType + fileName; let response = await fileStorage.asyncCreateTextFile(filePath, fileContent + '\n') if (response.err.result !== global.DEFAULT_OK_RESPONSE.result) { @@ -728,12 +728,12 @@ function checkIfSessionMustStop() { - if (bot.SESSION.type === 'Backtesting Session') { + if (bot.TRADING_SESSION.type === 'Backtesting Session') { /* Backtests needs only one execution of this process to complete. */ if (FULL_LOG === true) { logger.write(MODULE_NAME, '[IMPORTANT] checkIfSessionMustStop -> Backtesting Session Finished. Stopping the Session now. ') } - bot.SESSION.stop('Backtesting Session Finished.') + bot.TRADING_SESSION.stop('Backtesting Session Finished.') } } } diff --git a/LowFrequencyTrading/TradingRecords.js b/LowFrequencyTrading/TradingRecords.js index f3de8553c..8777fc2b6 100644 --- a/LowFrequencyTrading/TradingRecords.js +++ b/LowFrequencyTrading/TradingRecords.js @@ -20,7 +20,7 @@ exports.newTradingRecords = function newTradingRecords(bot, logger) { function initialize(pOutputDatasetsMap) { tradingEngine = bot.simulationState.tradingEngine tradingSystem = bot.simulationState.tradingSystem - sessionParameters = bot.SESSION.parameters + sessionParameters = bot.TRADING_SESSION.tradingParameters outputDatasetsMap = pOutputDatasetsMap // These are the files turned into arrays, stored in a Map by Product codeName. } @@ -143,16 +143,19 @@ exports.newTradingRecords = function newTradingRecords(bot, logger) { the Root Node specifically for this property only. */ if (recordProperty.config.nodePath !== undefined) { + if (recordProperty.config.nodePath === "tradingEngine.exchangeOrders.marketBuyOrders.marketOrders[index].orderBaseAsset") { + let a = tradingEngine + } try { propertyRootNode = eval(recordProperty.config.nodePath) } catch (err) { - badDefinitionUnhandledException(err, 'Error Evaluation Record Property nodePath -> nodePath = ' + recordProperty.config.nodePath, product, recordProperty) + badDefinitionUnhandledException(err, 'Error Evaluating Record Property nodePath -> nodePath = ' + recordProperty.config.nodePath, product, recordProperty) } } /* The Target Node is the node from where we are going to exctract the value. We will use the codeName of the Record Property to match it with - the any of the properties of the Root Node to get the Target Node. + any of the properties of the Root Node to get the Target Node. */ let targetNode try { @@ -284,8 +287,31 @@ exports.newTradingRecords = function newTradingRecords(bot, logger) { } } } else { - /* When we are not dealing with objects, we add every record to the existing file. */ - outputDatasetArray.push(record) + /* + When we are not dealing with objects, we add every record to the existing file except + for the ones that are filtered out at the Product Definition. + */ + if (product.config.codeName === "Trading-System-Errors") { + let a = 1 + } + if (product.config.propertyNameThatDefinesStatus !== undefined && product.config.propertyValueThatPreventsSavingObject !== undefined) { + for (let j = 0; j < product.record.properties.length; j++) { + let recordProperty = product.record.properties[j] + if (recordProperty.config.codeName === product.config.propertyNameThatDefinesStatus) { + let propertyValue = record[j] + + if (product.config.propertyValueThatPreventsSavingObject === "->Empty Array->" && propertyValue.length === 0) { + break + } + if (propertyValue !== product.config.propertyValueThatPreventsSavingObject) { + outputDatasetArray.push(record) + } + break + } + } + } else { + outputDatasetArray.push(record) + } } } diff --git a/LowFrequencyTrading/TradingSimulation.js b/LowFrequencyTrading/TradingSimulation.js index 186f2872e..3c262e76a 100644 --- a/LowFrequencyTrading/TradingSimulation.js +++ b/LowFrequencyTrading/TradingSimulation.js @@ -4,7 +4,7 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, tradin the execution at each loop cycle of the Trading System Protocol. */ const FULL_LOG = true - const MODULE_NAME = 'Trading Simulation -> ' + bot.SESSION.name + const MODULE_NAME = 'Trading Simulation -> ' + bot.TRADING_SESSION.name let thisObject = { finalize: finalize, @@ -28,7 +28,7 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, tradin let tradingSystem = bot.simulationState.tradingSystem let tradingEngine = bot.simulationState.tradingEngine - let sessionParameters = bot.SESSION.parameters + let sessionParameters = bot.TRADING_SESSION.tradingParameters if (FULL_LOG === true) { logger.write(MODULE_NAME, '[INFO] runSimulation -> initialDatetime = ' + sessionParameters.timeRange.config.initialDatetime) } if (FULL_LOG === true) { logger.write(MODULE_NAME, '[INFO] runSimulation -> finalDatetime = ' + sessionParameters.timeRange.config.finalDatetime) } @@ -53,7 +53,6 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, tradin /* Variables needed for heartbeat functionality */ let heartBeatDate let previousHeartBeatDate - let firstLoopExecution = true /* Estimation of the Initial Candle to Process in this Run. */ @@ -72,8 +71,8 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, tradin This will happen when the sessionParameters.timeRange.config.initialDatetime is beyond the last candle available, meaning that the dataSet needs to be updated with more up-to-date data. */ - bot.SESSION.stop('Data is not up-to-date enough. Please start the Masters Data Mining Operation.') - if (FULL_LOG === true) { logger.write(MODULE_NAME, '[IMPORTANT] runSimulation -> Data is not up-to-date enough. Stopping the Session now. ' ) } + bot.TRADING_SESSION.stop('Data is not up-to-date enough. Please start the Masters Data Mining Operation.') + if (FULL_LOG === true) { logger.write(MODULE_NAME, '[IMPORTANT] runSimulation -> Data is not up-to-date enough. Stopping the Session now. ') } return } } else { @@ -107,7 +106,7 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, tradin tradingEngine.current.episode.candle.index.value = i /* This is the current candle the Simulation is working at. */ - let candle = candles[tradingEngine.current.episode.candle.index.value] + let candle = candles[tradingEngine.current.episode.candle.index.value] if (FULL_LOG === true) { logger.write(MODULE_NAME, '[INFO] runSimulation -> loop -> Candle Begin @ ' + (new Date(candle.begin)).toLocaleString()) } if (FULL_LOG === true) { logger.write(MODULE_NAME, '[INFO] runSimulation -> loop -> Candle End @ ' + (new Date(candle.end)).toLocaleString()) } @@ -117,10 +116,8 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, tradin /* We emit a heart beat so that the UI can now where we are at the overal process.*/ heartBeat() - if (firstLoopExecution === true) { - tradingEpisodeModule.openEpisode() - firstLoopExecution = false - } + /* Opening the Episode, if needed. */ + tradingEpisodeModule.openEpisode() if (checkInitialDatetime() === false) { if (FULL_LOG === true) { logger.write(MODULE_NAME, '[INFO] runSimulation -> loop -> Candle Before the Initia Date Time @ ' + (new Date(candle.begin)).toLocaleString()) } @@ -213,16 +210,16 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, tradin if (checkFinalDatetime() === false) { closeEpisode('Final Datetime Reached') breakLoop = true - bot.SESSION.stop('Final Datetime Reached') - if (FULL_LOG === true) { logger.write(MODULE_NAME, '[IMPORTANT] runSimulation -> Final Datetime Reached. Stopping the Session now. ' ) } + bot.TRADING_SESSION.stop('Final Datetime Reached') + if (FULL_LOG === true) { logger.write(MODULE_NAME, '[IMPORTANT] runSimulation -> Final Datetime Reached. Stopping the Session now. ') } return } if (checkMinimunAndMaximunBalance() === false) { closeEpisode('Min or Max Balance Reached') breakLoop = true - bot.SESSION.stop('Min or Max Balance Reached') - if (FULL_LOG === true) { logger.write(MODULE_NAME, '[IMPORTANT] runSimulation -> Min or Max Balance Reached. Stopping the Session now. ' ) } + bot.TRADING_SESSION.stop('Min or Max Balance Reached') + if (FULL_LOG === true) { logger.write(MODULE_NAME, '[IMPORTANT] runSimulation -> Min or Max Balance Reached. Stopping the Session now. ') } return } } @@ -279,7 +276,7 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, tradin let processingDate = heartBeatDate.getUTCFullYear() + '-' + utilities.pad(heartBeatDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(heartBeatDate.getUTCDate(), 2) - if (FULL_LOG === true) { logger.write(MODULE_NAME, '[INFO] runSimulation -> loop -> Simulation ' + bot.sessionKey + ' Loop # ' + tradingEngine.current.episode.candle.index.value + ' @ ' + processingDate) } + if (FULL_LOG === true) { logger.write(MODULE_NAME, '[INFO] runSimulation -> loop -> Simulation ' + bot.TRADING_SESSIONKey + ' Loop # ' + tradingEngine.current.episode.candle.index.value + ' @ ' + processingDate) } /* Logging to console and disk */ if (global.areEqualDates(currentDate, new Date()) === false) { @@ -287,23 +284,23 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, tradin } /* Date only hearbeat */ - if (sessionParameters.heartbeats.config.date === true && sessionParameters.heartbeats.config.candleIndex === false) { + if (sessionParameters.heartbeats.config.date === true && sessionParameters.heartbeats.config.candleIndex === false) { hartbeatText = hartbeatText + currentDateString bot.processHeartBeat(hartbeatText, percentage) return } } - + /* When the Candle Index nees to be shown, then we can not send the hearbet only when the dates changes, we have to send it for every candle. It might also contain the date information. */ - if (sessionParameters.heartbeats.config.candleIndex === true ) { - if (sessionParameters.heartbeats.config.date === true ) { + if (sessionParameters.heartbeats.config.candleIndex === true) { + if (sessionParameters.heartbeats.config.date === true) { hartbeatText = hartbeatText + currentDateString } - hartbeatText = hartbeatText + ' Candle # ' + tradingEngine.current.episode.candle.index.value + hartbeatText = hartbeatText + ' Candle # ' + tradingEngine.current.episode.candle.index.value bot.processHeartBeat(hartbeatText, percentage) } } @@ -420,7 +417,7 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, tradin The second +1 is because we need to compare the next candle (remember that the loops allways avoid the last candle of the dataset available.) */ - if (tradingEngine.current.episode.candle.index.value + 1 + 1 === candles.length ) { + if (tradingEngine.current.episode.candle.index.value + 1 + 1 === candles.length) { /* When processing daily files, we need a mechanism to turn from one day to the next one. That mechanism is the one implemented here. If we detect that the next candle is the last candle of diff --git a/LowFrequencyTrading/TradingStages.js b/LowFrequencyTrading/TradingStages.js index b3465bf18..c62fd4b4c 100644 --- a/LowFrequencyTrading/TradingStages.js +++ b/LowFrequencyTrading/TradingStages.js @@ -47,7 +47,7 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM function initialize() { tradingSystem = bot.simulationState.tradingSystem tradingEngine = bot.simulationState.tradingEngine - sessionParameters = bot.SESSION.parameters + sessionParameters = bot.TRADING_SESSION.tradingParameters tradingStrategyModule.initialize() tradingPositionModule.initialize() @@ -160,7 +160,7 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM announcementsModule.makeAnnoucements(triggerStage.triggerOn) announcementsModule.makeAnnoucements(triggerStage) - if (bot.SESSION.type === 'Backtesting Session') { + if (bot.TRADING_SESSION.type === 'Backtesting Session') { if (sessionParameters.snapshots !== undefined) { if (sessionParameters.snapshots.config.strategy === true) { snapshotsModule.strategyEntry() @@ -245,7 +245,7 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM announcementsModule.makeAnnoucements(triggerStage.takePosition) announcementsModule.makeAnnoucements(strategy.openStage) - if (bot.SESSION.type === 'Backtesting Session') { + if (bot.TRADING_SESSION.type === 'Backtesting Session') { if (sessionParameters.snapshots !== undefined) { if (sessionParameters.snapshots.config.position === true) { snapshotsModule.positionEntry() @@ -393,6 +393,8 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM function runManageStage() { + if (tradingEngine.current.episode.cycle.value !== 'First') { return } + runWhenStatusIsOpening() runWhenStatusIsOpen() @@ -428,144 +430,110 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM /* Checking if Stop or Take Profit were hit */ checkStopLossOrTakeProfitWasHit() } - } - function calculateStopLoss() { - let strategy = tradingSystem.tradingStrategies[tradingEngine.current.strategy.index.value] - let manageStage = strategy.manageStage - let phase + function calculateStopLoss() { + let strategy = tradingSystem.tradingStrategies[tradingEngine.current.strategy.index.value] + let manageStage = strategy.manageStage + let phase - if (manageStage !== undefined) { - if (manageStage.managedStopLoss !== undefined) { - phase = manageStage.managedStopLoss.phases[tradingEngine.current.position.stopLoss.stopLossPhase.value - 1] - } - } - - if (phase !== undefined) { - if (phase.formula !== undefined) { - let previousValue = tradingEngine.current.position.stopLoss.value - tradingPositionModule.applyStopLossFormula(tradingSystem.formulas, phase.formula.id) - - if (tradingEngine.current.position.stopLoss.value !== previousValue) { - announcementsModule.makeAnnoucements(phase) + if (manageStage !== undefined) { + if (manageStage.managedStopLoss !== undefined) { + phase = manageStage.managedStopLoss.phases[tradingEngine.current.position.stopLoss.stopLossPhase.value - 1] } } - } - } - function calculateTakeProfit() { - let strategy = tradingSystem.tradingStrategies[tradingEngine.current.strategy.index.value] - let manageStage = strategy.manageStage - let phase + if (phase !== undefined) { + if (phase.formula !== undefined) { + let previousValue = tradingEngine.current.position.stopLoss.value + tradingPositionModule.applyStopLossFormula(tradingSystem.formulas, phase.formula.id) - if (manageStage !== undefined) { - if (manageStage.managedTakeProfit !== undefined) { - phase = manageStage.managedTakeProfit.phases[tradingEngine.current.position.takeProfit.takeProfitPhase.value - 1] - } - } - - if (phase !== undefined) { - if (phase.formula !== undefined) { - let previousValue = tradingEngine.current.position.takeProfit.value - tradingPositionModule.applyTakeProfitFormula(tradingSystem.formulas, phase.formula.id) - - if (tradingEngine.current.position.takeProfit.value !== previousValue) { - announcementsModule.makeAnnoucements(phase) - } - } - } - } - - function calculateStopLossPosition() { - /* - The position of the Stop Loss (above or below the price) is needed in order to know - later if the price hit the Stop Loss or not at every Simulation candle. When we get - the first values of the simulation of the Stop Loss we check if it is above or below - the Position Rate, and we assign the values Above or Below to it. - */ - if (tradingEngine.current.position.stopLoss.stopLossPosition.value === tradingEngine.current.position.stopLoss.stopLossPosition.config.initialValue) { - if (tradingEngine.current.position.stopLoss.value > tradingEngine.current.position.entryTargetRate.value) { - tradingEngine.current.position.stopLoss.stopLossPosition.value = 'Above' - } else { - tradingEngine.current.position.stopLoss.stopLossPosition.value = 'Below' - } - } - } - - function calculateTakeProfitPosition() { - /* - The position of the Take Profit (above or below the price) is needed in order to know - later if the price hit the Take Profit or not at every Simulation candle. When we get - the first values of the simulation of the Take Profits we check if it is above or below - the Position Rates, and we assign the values Above or Below to it. - */ - if (tradingEngine.current.position.takeProfit.takeProfitPosition.value === tradingEngine.current.position.takeProfit.takeProfitPosition.config.initialValue) { - if (tradingEngine.current.position.takeProfit.value > tradingEngine.current.position.entryTargetRate.value) { - tradingEngine.current.position.takeProfit.takeProfitPosition.value = 'Above' - } else { - tradingEngine.current.position.takeProfit.takeProfitPosition.value = 'Below' - } - } - } - - function checkStopPhasesEvents() { - let strategy = tradingSystem.tradingStrategies[tradingEngine.current.strategy.index.value] - let manageStage = strategy.manageStage - let parentNode - let phaseIndex - let phase - let stopLoss - - if (manageStage !== undefined) { - if (manageStage.managedStopLoss !== undefined) { - parentNode = manageStage - phaseIndex = tradingEngine.current.position.stopLoss.stopLossPhase.value - 1 - stopLoss = manageStage.managedStopLoss - phase = stopLoss.phases[phaseIndex] - } - } - - if (parentNode !== undefined) { - if (phase === undefined) { return } // trying to jump to a phase that does not exists. - - /* Check the next Phase Event. */ - let nextPhaseEvent = phase.nextPhaseEvent - if (nextPhaseEvent !== undefined) { - for (let k = 0; k < nextPhaseEvent.situations.length; k++) { - let situation = nextPhaseEvent.situations[k] - let passed - if (situation.conditions.length > 0) { - passed = true - } - - passed = tradingSystem.checkConditions(situation, passed) - - tradingSystem.values.push([situation.id, passed]) - if (passed) { - tradingSystem.highlights.push(situation.id) - tradingSystem.highlights.push(nextPhaseEvent.id) - tradingSystem.highlights.push(phase.id) - tradingSystem.highlights.push(stopLoss.id) - tradingSystem.highlights.push(parentNode.id) - tradingSystem.highlights.push(manageStage.id) - - tradingPositionModule.updateStopLoss(tradingEngine.current.position.stopLoss.stopLossPhase.value + 1) - - announcementsModule.makeAnnoucements(nextPhaseEvent) - - /* Reset this counter */ - tradingEngine.current.episode.distanceToEvent.nextPhase.value = 1 - return // only one event can pass at the time + if (tradingEngine.current.position.stopLoss.value !== previousValue) { + announcementsModule.makeAnnoucements(phase) } } } + } - /* Check the Move to Phase Events. */ - for (let n = 0; n < phase.moveToPhaseEvents.length; n++) { - let moveToPhaseEvent = phase.moveToPhaseEvents[n] - if (moveToPhaseEvent !== undefined) { - for (let k = 0; k < moveToPhaseEvent.situations.length; k++) { - let situation = moveToPhaseEvent.situations[k] + function calculateTakeProfit() { + let strategy = tradingSystem.tradingStrategies[tradingEngine.current.strategy.index.value] + let manageStage = strategy.manageStage + let phase + + if (manageStage !== undefined) { + if (manageStage.managedTakeProfit !== undefined) { + phase = manageStage.managedTakeProfit.phases[tradingEngine.current.position.takeProfit.takeProfitPhase.value - 1] + } + } + + if (phase !== undefined) { + if (phase.formula !== undefined) { + let previousValue = tradingEngine.current.position.takeProfit.value + tradingPositionModule.applyTakeProfitFormula(tradingSystem.formulas, phase.formula.id) + + if (tradingEngine.current.position.takeProfit.value !== previousValue) { + announcementsModule.makeAnnoucements(phase) + } + } + } + } + + function calculateStopLossPosition() { + /* + The position of the Stop Loss (above or below the price) is needed in order to know + later if the price hit the Stop Loss or not at every Simulation candle. When we get + the first values of the simulation of the Stop Loss we check if it is above or below + the Position Rate, and we assign the values Above or Below to it. + */ + if (tradingEngine.current.position.stopLoss.stopLossPosition.value === tradingEngine.current.position.stopLoss.stopLossPosition.config.initialValue) { + if (tradingEngine.current.position.stopLoss.value > tradingEngine.current.position.entryTargetRate.value) { + tradingEngine.current.position.stopLoss.stopLossPosition.value = 'Above' + } else { + tradingEngine.current.position.stopLoss.stopLossPosition.value = 'Below' + } + } + } + + function calculateTakeProfitPosition() { + /* + The position of the Take Profit (above or below the price) is needed in order to know + later if the price hit the Take Profit or not at every Simulation candle. When we get + the first values of the simulation of the Take Profits we check if it is above or below + the Position Rates, and we assign the values Above or Below to it. + */ + if (tradingEngine.current.position.takeProfit.takeProfitPosition.value === tradingEngine.current.position.takeProfit.takeProfitPosition.config.initialValue) { + if (tradingEngine.current.position.takeProfit.value > tradingEngine.current.position.entryTargetRate.value) { + tradingEngine.current.position.takeProfit.takeProfitPosition.value = 'Above' + } else { + tradingEngine.current.position.takeProfit.takeProfitPosition.value = 'Below' + } + } + } + + function checkStopPhasesEvents() { + let strategy = tradingSystem.tradingStrategies[tradingEngine.current.strategy.index.value] + let manageStage = strategy.manageStage + let parentNode + let phaseIndex + let phase + let stopLoss + + if (manageStage !== undefined) { + if (manageStage.managedStopLoss !== undefined) { + parentNode = manageStage + phaseIndex = tradingEngine.current.position.stopLoss.stopLossPhase.value - 1 + stopLoss = manageStage.managedStopLoss + phase = stopLoss.phases[phaseIndex] + } + } + + if (parentNode !== undefined) { + if (phase === undefined) { return } // trying to jump to a phase that does not exists. + + /* Check the next Phase Event. */ + let nextPhaseEvent = phase.nextPhaseEvent + if (nextPhaseEvent !== undefined) { + for (let k = 0; k < nextPhaseEvent.situations.length; k++) { + let situation = nextPhaseEvent.situations[k] let passed if (situation.conditions.length > 0) { passed = true @@ -576,95 +544,95 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM tradingSystem.values.push([situation.id, passed]) if (passed) { tradingSystem.highlights.push(situation.id) - tradingSystem.highlights.push(moveToPhaseEvent.id) + tradingSystem.highlights.push(nextPhaseEvent.id) tradingSystem.highlights.push(phase.id) tradingSystem.highlights.push(stopLoss.id) tradingSystem.highlights.push(parentNode.id) tradingSystem.highlights.push(manageStage.id) - let moveToPhase = moveToPhaseEvent.referenceParent - if (moveToPhase !== undefined) { - for (let q = 0; q < stopLoss.phases.length; q++) { - if (stopLoss.phases[q].id === moveToPhase.id) { - tradingPositionModule.updateStopLoss(q + 1) - } - } - } else { - tradingSystem.errors.push([moveToPhaseEvent.id, 'This Node needs to reference a Phase.']) - continue + tradingPositionModule.updateStopLoss(tradingEngine.current.position.stopLoss.stopLossPhase.value + 1) + + announcementsModule.makeAnnoucements(nextPhaseEvent) + + /* Reset this counter */ + tradingEngine.current.episode.distanceToEvent.nextPhase.value = 1 + return // only one event can pass at the time + } + } + } + + /* Check the Move to Phase Events. */ + for (let n = 0; n < phase.moveToPhaseEvents.length; n++) { + let moveToPhaseEvent = phase.moveToPhaseEvents[n] + if (moveToPhaseEvent !== undefined) { + for (let k = 0; k < moveToPhaseEvent.situations.length; k++) { + let situation = moveToPhaseEvent.situations[k] + let passed + if (situation.conditions.length > 0) { + passed = true } - announcementsModule.makeAnnoucements(moveToPhaseEvent) - - /* Reset this counter */ - tradingEngine.current.episode.distanceToEvent.moveToPhase.value = 1 - return // only one event can pass at the time + passed = tradingSystem.checkConditions(situation, passed) + + tradingSystem.values.push([situation.id, passed]) + if (passed) { + tradingSystem.highlights.push(situation.id) + tradingSystem.highlights.push(moveToPhaseEvent.id) + tradingSystem.highlights.push(phase.id) + tradingSystem.highlights.push(stopLoss.id) + tradingSystem.highlights.push(parentNode.id) + tradingSystem.highlights.push(manageStage.id) + + let moveToPhase = moveToPhaseEvent.referenceParent + if (moveToPhase !== undefined) { + for (let q = 0; q < stopLoss.phases.length; q++) { + if (stopLoss.phases[q].id === moveToPhase.id) { + tradingPositionModule.updateStopLoss(q + 1) + } + } + } else { + tradingSystem.errors.push([moveToPhaseEvent.id, 'This Node needs to reference a Phase.']) + continue + } + + announcementsModule.makeAnnoucements(moveToPhaseEvent) + + /* Reset this counter */ + tradingEngine.current.episode.distanceToEvent.moveToPhase.value = 1 + return // only one event can pass at the time + } } } } } } - } - function checkTakeProfitPhaseEvents() { - let strategy = tradingSystem.tradingStrategies[tradingEngine.current.strategy.index.value] - let openStage = strategy.openStage - let manageStage = strategy.manageStage - let parentNode - let phaseIndex - let phase - let takeProfit + function checkTakeProfitPhaseEvents() { + let strategy = tradingSystem.tradingStrategies[tradingEngine.current.strategy.index.value] + let openStage = strategy.openStage + let manageStage = strategy.manageStage + let parentNode + let phaseIndex + let phase + let takeProfit - if (manageStage !== undefined) { - if (manageStage.managedTakeProfit !== undefined) { - parentNode = manageStage - phaseIndex = tradingEngine.current.position.takeProfit.takeProfitPhase.value - 1 - takeProfit = manageStage.managedTakeProfit - phase = takeProfit.phases[phaseIndex] - } - } - - if (parentNode !== undefined) { - if (phase === undefined) { return } // trying to jump to a phase that does not exists. - - /* Check the next Phase Event. */ - let nextPhaseEvent = phase.nextPhaseEvent - if (nextPhaseEvent !== undefined) { - for (let k = 0; k < nextPhaseEvent.situations.length; k++) { - let situation = nextPhaseEvent.situations[k] - let passed - if (situation.conditions.length > 0) { - passed = true - } - - passed = tradingSystem.checkConditions(situation, passed) - - tradingSystem.values.push([situation.id, passed]) - if (passed) { - tradingSystem.highlights.push(situation.id) - tradingSystem.highlights.push(nextPhaseEvent.id) - tradingSystem.highlights.push(phase.id) - tradingSystem.highlights.push(takeProfit.id) - tradingSystem.highlights.push(parentNode.id) - tradingSystem.highlights.push(manageStage.id) - - tradingPositionModule.updateTakeProfit(tradingEngine.current.position.takeProfit.takeProfitPhase.value + 1) - - announcementsModule.makeAnnoucements(nextPhaseEvent) - - /* Reset this counter */ - tradingEngine.current.episode.distanceToEvent.nextPhase.value = 1 - return // only one event can pass at the time - } + if (manageStage !== undefined) { + if (manageStage.managedTakeProfit !== undefined) { + parentNode = manageStage + phaseIndex = tradingEngine.current.position.takeProfit.takeProfitPhase.value - 1 + takeProfit = manageStage.managedTakeProfit + phase = takeProfit.phases[phaseIndex] } } - /* Check the Move to Phase Events. */ - for (let n = 0; n < phase.moveToPhaseEvents.length; n++) { - let moveToPhaseEvent = phase.moveToPhaseEvents[n] - if (moveToPhaseEvent !== undefined) { - for (let k = 0; k < moveToPhaseEvent.situations.length; k++) { - let situation = moveToPhaseEvent.situations[k] + if (parentNode !== undefined) { + if (phase === undefined) { return } // trying to jump to a phase that does not exists. + + /* Check the next Phase Event. */ + let nextPhaseEvent = phase.nextPhaseEvent + if (nextPhaseEvent !== undefined) { + for (let k = 0; k < nextPhaseEvent.situations.length; k++) { + let situation = nextPhaseEvent.situations[k] let passed if (situation.conditions.length > 0) { passed = true @@ -675,84 +643,118 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM tradingSystem.values.push([situation.id, passed]) if (passed) { tradingSystem.highlights.push(situation.id) - tradingSystem.highlights.push(moveToPhaseEvent.id) + tradingSystem.highlights.push(nextPhaseEvent.id) tradingSystem.highlights.push(phase.id) tradingSystem.highlights.push(takeProfit.id) tradingSystem.highlights.push(parentNode.id) tradingSystem.highlights.push(manageStage.id) - let moveToPhase = moveToPhaseEvent.referenceParent - if (moveToPhase !== undefined) { - for (let q = 0; q < takeProfit.phases.length; q++) { - if (takeProfit.phases[q].id === moveToPhase.id) { - tradingPositionModule.updateTakeProfit(q + 1) - } - } - } else { - tradingSystem.errors.push([moveToPhaseEvent.id, 'This Node needs to reference a Phase.']) - continue - } + tradingPositionModule.updateTakeProfit(tradingEngine.current.position.takeProfit.takeProfitPhase.value + 1) - announcementsModule.makeAnnoucements(moveToPhaseEvent) + announcementsModule.makeAnnoucements(nextPhaseEvent) /* Reset this counter */ - tradingEngine.current.episode.distanceToEvent.moveToPhase.value = 1 + tradingEngine.current.episode.distanceToEvent.nextPhase.value = 1 return // only one event can pass at the time } } } + + /* Check the Move to Phase Events. */ + for (let n = 0; n < phase.moveToPhaseEvents.length; n++) { + let moveToPhaseEvent = phase.moveToPhaseEvents[n] + if (moveToPhaseEvent !== undefined) { + for (let k = 0; k < moveToPhaseEvent.situations.length; k++) { + let situation = moveToPhaseEvent.situations[k] + let passed + if (situation.conditions.length > 0) { + passed = true + } + + passed = tradingSystem.checkConditions(situation, passed) + + tradingSystem.values.push([situation.id, passed]) + if (passed) { + tradingSystem.highlights.push(situation.id) + tradingSystem.highlights.push(moveToPhaseEvent.id) + tradingSystem.highlights.push(phase.id) + tradingSystem.highlights.push(takeProfit.id) + tradingSystem.highlights.push(parentNode.id) + tradingSystem.highlights.push(manageStage.id) + + let moveToPhase = moveToPhaseEvent.referenceParent + if (moveToPhase !== undefined) { + for (let q = 0; q < takeProfit.phases.length; q++) { + if (takeProfit.phases[q].id === moveToPhase.id) { + tradingPositionModule.updateTakeProfit(q + 1) + } + } + } else { + tradingSystem.errors.push([moveToPhaseEvent.id, 'This Node needs to reference a Phase.']) + continue + } + + announcementsModule.makeAnnoucements(moveToPhaseEvent) + + /* Reset this counter */ + tradingEngine.current.episode.distanceToEvent.moveToPhase.value = 1 + return // only one event can pass at the time + } + } + } + } } } - } - function checkStopLossOrTakeProfitWasHit() { - { - let strategy = tradingSystem.tradingStrategies[tradingEngine.current.strategy.index.value] - /* - Checking what happened since the last execution. We need to know if the Stop Loss - or our Take Profit were hit. - */ + function checkStopLossOrTakeProfitWasHit() { + { + let strategy = tradingSystem.tradingStrategies[tradingEngine.current.strategy.index.value] + /* + Checking what happened since the last execution. We need to know if the Stop Loss + or our Take Profit were hit. + */ - /* Stop Loss condition: Here we verify if the Stop Loss was hitted or not. */ - if ( - ( - tradingEngine.current.position.stopLoss.stopLossPosition.value === 'Above' && - tradingEngine.current.episode.candle.max.value >= tradingEngine.current.position.stopLoss.value - ) || - ( - tradingEngine.current.position.stopLoss.stopLossPosition.value === 'Below' && - tradingEngine.current.episode.candle.min.value <= tradingEngine.current.position.stopLoss.value - ) - ) { - logger.write(MODULE_NAME, '[INFO] checkStopLossOrTakeProfitWasHit -> Stop Loss was hit.') + /* Stop Loss condition: Here we verify if the Stop Loss was hitted or not. */ + if ( + ( + tradingEngine.current.position.stopLoss.stopLossPosition.value === 'Above' && + tradingEngine.current.episode.candle.max.value >= tradingEngine.current.position.stopLoss.value + ) || + ( + tradingEngine.current.position.stopLoss.stopLossPosition.value === 'Below' && + tradingEngine.current.episode.candle.min.value <= tradingEngine.current.position.stopLoss.value + ) + ) { + logger.write(MODULE_NAME, '[INFO] checkStopLossOrTakeProfitWasHit -> Stop Loss was hit.') - tradingPositionModule.closingPosition('Stop Loss') - changeStageStatus('Close Stage', 'Opening') - changeStageStatus('Manage Stage', 'Closed') - announcementsModule.makeAnnoucements(strategy.closeStage) + tradingPositionModule.closingPosition('Stop Loss') + changeStageStatus('Close Stage', 'Opening') + changeStageStatus('Manage Stage', 'Closed') + announcementsModule.makeAnnoucements(strategy.closeStage) + return + } + + /* Take Profit condition: Here we verify if the Take Profit was hit or not. */ + if ( + ( + tradingEngine.current.position.takeProfit.takeProfitPosition.value === 'Below' && + tradingEngine.current.episode.candle.min.value <= tradingEngine.current.position.takeProfit.value + ) || + ( + tradingEngine.current.position.takeProfit.takeProfitPosition.value === 'Above' && + tradingEngine.current.episode.candle.max.value >= tradingEngine.current.position.takeProfit.value + ) + ) { + logger.write(MODULE_NAME, '[INFO] checkStopLossOrTakeProfitWasHit -> Take Profit was hit.') + + tradingPositionModule.closingPosition('Take Profit') + changeStageStatus('Close Stage', 'Opening') + changeStageStatus('Manage Stage', 'Closed') + announcementsModule.makeAnnoucements(strategy.closeStage) + return + } return } - - /* Take Profit condition: Here we verify if the Take Profit was hit or not. */ - if ( - ( - tradingEngine.current.position.takeProfit.takeProfitPosition.value === 'Below' && - tradingEngine.current.episode.candle.min.value <= tradingEngine.current.position.takeProfit.value - ) || - ( - tradingEngine.current.position.takeProfit.takeProfitPosition.value === 'Above' && - tradingEngine.current.episode.candle.max.value >= tradingEngine.current.position.takeProfit.value - ) - ) { - logger.write(MODULE_NAME, '[INFO] checkStopLossOrTakeProfitWasHit -> Take Profit was hit.') - - tradingPositionModule.closingPosition('Take Profit') - changeStageStatus('Close Stage', 'Opening') - changeStageStatus('Manage Stage', 'Closed') - announcementsModule.makeAnnoucements(strategy.closeStage) - return - } - return } } } @@ -786,12 +788,12 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM to be known before we start placing orders at the Close Stage. */ if ( - tradingEngine.current.strategyCloseStage.status.value === 'Open' && - ( - tradingEngine.current.strategyOpenStage.status.value === 'Closed' || - tradingEngine.current.strategyOpenStage.status.value == tradingEngine.current.strategyOpenStage.status.config.initialValue - ) - ) { + tradingEngine.current.strategyCloseStage.status.value === 'Open' && + ( + tradingEngine.current.strategyOpenStage.status.value === 'Closed' || + tradingEngine.current.strategyOpenStage.status.value == tradingEngine.current.strategyOpenStage.status.config.initialValue + ) + ) { /* This will happen as long as the Close Stage is Open. */ @@ -854,7 +856,7 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM function closePositionAndStrategy() { /* Taking Position Snapshot */ - if (bot.SESSION.type === 'Backtesting Session') { + if (bot.TRADING_SESSION.type === 'Backtesting Session') { if (sessionParameters.snapshots !== undefined) { if (sessionParameters.snapshots.config.position === true) { snapshotsModule.positionExit() @@ -863,7 +865,7 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM } /* Taking Strategy Snapshot */ - if (bot.SESSION.type === 'Backtesting Session') { + if (bot.TRADING_SESSION.type === 'Backtesting Session') { if (sessionParameters.snapshots !== undefined) { if (sessionParameters.snapshots.config.strategy === true) { snapshotsModule.strategyExit() @@ -886,19 +888,28 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM function checkIfStageNeedsToBeClosed(tradingEngineStage, tradingSystemStage, stageName) { /* - The Stage is closed when the placedSize reaches the targetSize, and the fillSize - + feesPaid reaches the placedSize. This depends on how the user defined the stage: - either on Base Asset or Quoted Asset. - This can happens at any time when we update the sizeFilled and the feesPaid values - when we see at the exchange that orders were filled. - Note that the comparison is made with a Rounding Factor in order to avoid rounding problems. + The Stage is closed when the fillSize + feesPaid reaches the targetSize. + + The Target Size is defined by the end user either in Base Asset or Quoted Asset. + Whatever the user chooses defines also the way we check if the Target was met. + Why? Because the counterparty target is estimated with the Target Rate and it + is not syncronized with the reality we later learn when the exchange tell us the + actual rate each of the orders where executed at. + + For the same reason that the user does not know exactly which is the rate the orders + will actually be filled, then we can not expect that the total filled will really + reach the target size, so we introduce a rounding factor so that when it is close + enough we will consider the target to have been reached. */ - const ROUNDING_ERROR_CORRECTION_FACTOR = 1.001 + const ROUNDING_ERROR_CORRECTION_FACTOR = 1.001 switch (tradingEngineStage.stageDefinedIn.value) { case 'Base Asset': { if ( - tradingEngineStage.stageBaseAsset.sizeFilled.value * ROUNDING_ERROR_CORRECTION_FACTOR + ( + tradingEngineStage.stageBaseAsset.sizeFilled.value + ) + * ROUNDING_ERROR_CORRECTION_FACTOR >= tradingEngineStage.stageBaseAsset.targetSize.value ) { @@ -910,7 +921,10 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM } case 'Quoted Asset': { if ( - tradingEngineStage.stageQuotedAsset.sizeFilled.value * ROUNDING_ERROR_CORRECTION_FACTOR + ( + tradingEngineStage.stageQuotedAsset.sizeFilled.value + ) + * ROUNDING_ERROR_CORRECTION_FACTOR >= tradingEngineStage.stageQuotedAsset.targetSize.value ) { @@ -921,9 +935,11 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM break } } + function positionFilled() { changeStageStatus(stageName, 'Closed', 'Position Size Filled') } + function checkCloseStageEvent(tradingSystemStage) { /* Check the Close Stage Event */ tradingSystem.evalConditions(tradingSystemStage, 'Close Stage Event') @@ -990,35 +1006,50 @@ exports.newTradingStages = function newTradingStages(bot, logger, tradingEngineM closeStage(stage) } + /* + The way the begining and end of the Stages object works is as follows: + This object begins with the last cycle, that is the current cycle diplaced + backwards to the current candle. And it ends at the end of the last cycle. + During mantainance, we add the time frame of the candle, since we assume it + will survive the 2 cycles without the end being updated. If it closes, + then we update the end to the end of the last cycle, overrinding the + mantainance if needed. If we assume that in the same cycle there will be + no closing and opening of the same type of object, then there can be no + overlap. + */ function openStage(stage) { /* Recording the opening at the Trading Engine Data Structure */ - stage.begin.value = tradingEngine.current.episode.cycle.begin.value + stage.begin.value = tradingEngine.current.episode.cycle.lastBegin.value + stage.end.value = tradingEngine.current.episode.cycle.lastEnd.value stage.beginRate.value = tradingEngine.current.episode.candle.close.value - stage.end.value = tradingEngine.current.episode.cycle.end.value } function closeStage(stage) { /* Recording the closing at the Trading Engine Data Structure */ - stage.end.value = tradingEngine.current.episode.cycle.end.value + stage.end.value = tradingEngine.current.episode.cycle.lastEnd.value stage.endRate.value = tradingEngine.current.episode.candle.close.value } } function updateEnds() { + /* + Note that we can not use the cycle here because this is executed via mantain + which in turn is executed before the first cycle for this candle is set. + */ if (tradingEngine.current.strategyTriggerStage.status.value === 'Open') { - tradingEngine.current.strategyTriggerStage.end.value = tradingEngine.current.episode.cycle.end.value + tradingEngine.current.strategyTriggerStage.end.value = tradingEngine.current.strategyTriggerStage.end.value + sessionParameters.timeFrame.config.value tradingEngine.current.strategyTriggerStage.endRate.value = tradingEngine.current.episode.candle.close.value } if (tradingEngine.current.strategyOpenStage.status.value === 'Open') { - tradingEngine.current.strategyOpenStage.end.value = tradingEngine.current.episode.cycle.end.value + tradingEngine.current.strategyOpenStage.end.value = tradingEngine.current.strategyOpenStage.end.value + sessionParameters.timeFrame.config.value tradingEngine.current.strategyOpenStage.endRate.value = tradingEngine.current.episode.candle.close.value } if (tradingEngine.current.strategyManageStage.status.value === 'Open') { - tradingEngine.current.strategyManageStage.end.value = tradingEngine.current.episode.cycle.end.value + tradingEngine.current.strategyManageStage.end.value = tradingEngine.current.strategyManageStage.end.value + sessionParameters.timeFrame.config.value tradingEngine.current.strategyManageStage.endRate.value = tradingEngine.current.episode.candle.close.value } if (tradingEngine.current.strategyCloseStage.status.value === 'Open') { - tradingEngine.current.strategyCloseStage.end.value = tradingEngine.current.episode.cycle.end.value + tradingEngine.current.strategyCloseStage.end.value = tradingEngine.current.strategyCloseStage.end.value + sessionParameters.timeFrame.config.value tradingEngine.current.strategyCloseStage.endRate.value = tradingEngine.current.episode.candle.close.value } } diff --git a/LowFrequencyTrading/TradingStrategy.js b/LowFrequencyTrading/TradingStrategy.js index 5225f705d..fbd170845 100644 --- a/LowFrequencyTrading/TradingStrategy.js +++ b/LowFrequencyTrading/TradingStrategy.js @@ -13,15 +13,18 @@ exports.newTradingStrategy = function newTradingStrategy(bot, logger, tradingEng } let tradingEngine + let sessionParameters return thisObject function initialize() { tradingEngine = bot.simulationState.tradingEngine + sessionParameters = bot.TRADING_SESSION.tradingParameters } function finalize() { tradingEngine = undefined + sessionParameters = undefined } function mantain() { @@ -34,11 +37,14 @@ exports.newTradingStrategy = function newTradingStrategy(bot, logger, tradingEng } function openStrategy(index, situationName, strategyName) { + /* Starting begin and end */ + tradingEngine.current.strategy.begin.value = tradingEngine.current.episode.cycle.lastBegin.value + tradingEngine.current.strategy.end.value = tradingEngine.current.episode.cycle.lastEnd.value + + /* Recording the opening at the Trading Engine Data Structure */ tradingEngine.current.strategy.status.value = 'Open' tradingEngine.current.strategy.serialNumber.value = tradingEngine.current.episode.episodeCounters.strategies.value tradingEngine.current.strategy.identifier.value = global.UNIQUE_ID() - tradingEngine.current.strategy.begin.value = tradingEngine.current.episode.candle.end.value - tradingEngine.current.strategy.end.value = tradingEngine.current.episode.candle.end.value tradingEngine.current.strategy.beginRate.value = tradingEngine.current.episode.candle.min.value tradingEngine.current.strategy.index.value = index @@ -52,7 +58,7 @@ exports.newTradingStrategy = function newTradingStrategy(bot, logger, tradingEng function closeStrategy(exitType) { tradingEngine.current.strategy.status.value = 'Closed' tradingEngine.current.strategy.exitType.value = exitType - tradingEngine.current.strategy.end.value = tradingEngine.current.episode.candle.end.value + tradingEngine.current.strategy.end.value = tradingEngine.current.episode.cycle.lastEnd.value tradingEngine.current.strategy.endRate.value = tradingEngine.current.episode.candle.min.value /* Now that the strategy is closed, it is the right time to move this strategy from current to last at the Trading Engine data structure. @@ -62,7 +68,7 @@ exports.newTradingStrategy = function newTradingStrategy(bot, logger, tradingEng function updateEnds() { if (tradingEngine.current.strategy.status.value === 'Open') { - tradingEngine.current.strategy.end.value = tradingEngine.current.episode.candle.end.value + tradingEngine.current.strategy.end.value = tradingEngine.current.strategy.end.value + sessionParameters.timeFrame.config.value tradingEngine.current.strategy.endRate.value = tradingEngine.current.episode.candle.close.value } } diff --git a/LowFrequencyTrading/TradingSystem.js b/LowFrequencyTrading/TradingSystem.js index d90c558ae..ddafc6f33 100644 --- a/LowFrequencyTrading/TradingSystem.js +++ b/LowFrequencyTrading/TradingSystem.js @@ -17,7 +17,7 @@ exports.newTradingSystem = function newTradingSystem(bot, logger, tradingEngineM let tradingSystem let tradingEngine let sessionParameters - let dynamicIndicators + let dynamicIndicators const TRADING_STAGES_MODULE = require('./TradingStages.js') let tradingStagesModule = TRADING_STAGES_MODULE.newTradingStages(bot, logger, tradingEngineModule) @@ -28,7 +28,7 @@ exports.newTradingSystem = function newTradingSystem(bot, logger, tradingEngineM function initialize() { tradingSystem = bot.simulationState.tradingSystem tradingEngine = bot.simulationState.tradingEngine - sessionParameters = bot.SESSION.parameters + sessionParameters = bot.TRADING_SESSION.tradingParameters tradingSystem.conditions = new Map() tradingSystem.formulas = new Map() @@ -131,8 +131,8 @@ exports.newTradingSystem = function newTradingSystem(bot, logger, tradingEngineM for (let i = 0; i < tradingSystem.dynamicIndicators.indicatorFunctions.length; i++) { let indicatorFunction = tradingSystem.dynamicIndicators.indicatorFunctions[i] - if (indicatorFunction.formula === undefined) { return} - if (indicatorFunction.config.codeName === undefined) { return} + if (indicatorFunction.formula === undefined) { return } + if (indicatorFunction.config.codeName === undefined) { return } dynamicIndicators[indicatorFunction.config.codeName] = tradingSystem.formulas.get(indicatorFunction.formula.id) } } @@ -257,15 +257,7 @@ exports.newTradingSystem = function newTradingSystem(bot, logger, tradingEngineM will not be considered an error. */ value = false - - if (code.indexOf('previous') > -1 && err.message.indexOf('of undefined') > -1 - ) { - /* - We are not going to set an error for the casess we are using previous and the error is that the indicator is undefined. - */ - } else { - error = err.message - } + error = err.message } tradingSystem.conditions.set(node.id, value) @@ -297,19 +289,12 @@ exports.newTradingSystem = function newTradingSystem(bot, logger, tradingEngineM will not be considered an error. */ value = 0 - - if (node.code.indexOf('previous') > -1 && err.message.indexOf('of undefined') > -1 - ) { - /* - We are not going to set an error for the casess we are using previous and the error is that the indicator is undefined. - */ - } else { - error = err.message - } + error = err.message } if (error !== undefined) { tradingSystem.errors.push([node.id, error]) + logger.write(MODULE_NAME, '[INFO] evalFormula -> error = ' + error) return } if (value !== undefined) { @@ -323,7 +308,6 @@ exports.newTradingSystem = function newTradingSystem(bot, logger, tradingEngineM } logger.write(MODULE_NAME, '[INFO] evalFormula -> value = ' + value) - logger.write(MODULE_NAME, '[INFO] evalFormula -> error = ' + error) } } diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index cdc4a892b..4d0e4b922 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -67,7 +67,6 @@ function getContextVariables() { try { let thisReport; - let reportKey; let statusReport; /* We look first for the bot who knows the begining of the market in order to get when the market starts. @@ -89,7 +88,7 @@ thisReport = statusReport.file; if (thisReport.beginingOfMarket === undefined) { - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey) + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> thisReport.beginingOfMarket === undefined ") logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> It is too early too run this process since the trade history of the market is not there yet.") let customOK = { @@ -122,7 +121,7 @@ thisReport = statusReport.file; if (thisReport.lastFile === undefined) { - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey) + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> thisReport.lastFile === undefined") let customOK = { result: global.CUSTOM_OK_RESPONSE.result, @@ -139,7 +138,7 @@ statusReport = statusDependencies.reportsByMainUtility.get("Self Reference") if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. - logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Self Reference -> Status Report does not exist or Self Reference not defined. Retrying Later. ") + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Self Reference -> Status Report does not exist or Self Reference not defined or badly configured. Check that at the Status Dependency you have chosen Self Reference at the config. Retrying Later. ") callBackFunction(global.DEFAULT_RETRY_RESPONSE) return } @@ -157,7 +156,7 @@ beginingOfMarket = new Date(thisReport.beginingOfMarket) if (beginingOfMarket.valueOf() !== contextVariables.dateBeginOfMarket.valueOf()) { // Reset Mechanism for Begining of the Market - logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Reset Mechanism for Begining of the Market Activated. -> reportKey = " + reportKey) + logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Reset Mechanism for Begining of the Market Activated.") beginingOfMarket = new Date(contextVariables.dateBeginOfMarket.valueOf()) startFromBegining() diff --git a/ProcessExecutionEvents.js b/ProcessExecutionEvents.js index 9b94c0365..ae74beee2 100644 --- a/ProcessExecutionEvents.js +++ b/ProcessExecutionEvents.js @@ -11,7 +11,7 @@ let thisObject = { networkNode: undefined, - initialize, initialize, + initialize: initialize, finalize: finalize, start: start, finish: finish diff --git a/ProcessOutput.js b/ProcessOutput.js index 9f6f0ee71..c4751f557 100644 --- a/ProcessOutput.js +++ b/ProcessOutput.js @@ -32,6 +32,9 @@ if (botNode === undefined) { botNode = global.FIND_NODE_IN_NODE_MESH(outputDataset, 'Sensor Bot') } + if (botNode === undefined) { + botNode = global.FIND_NODE_IN_NODE_MESH(outputDataset, 'Learning Bot') + } /* Some validations to verify that everything is in place. */ if (outputDataset.referenceParent !== undefined) { @@ -86,7 +89,7 @@ let product = outputDataset.referenceParent.parentNode.config.codeName let dataset = outputDataset.referenceParent.config.codeName - key = dataMine + "-" + botName + "-" + product + "-" + dataset + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset + let key = dataMine + "-" + botName + "-" + product + "-" + dataset + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset let event = { lastFile: lastFile, timeFrames: timeFrames @@ -119,6 +122,9 @@ if (botNode === undefined) { botNode = global.FIND_NODE_IN_NODE_MESH(outputDataset, 'Sensor Bot') } + if (botNode === undefined) { + botNode = global.FIND_NODE_IN_NODE_MESH(outputDataset, 'Learning Bot') + } /* Some validations to verify that everything is in place. */ if (outputDataset.referenceParent !== undefined) { @@ -138,11 +144,11 @@ } if (botNode.parentNode !== undefined) { if (botNode.parentNode.config.codeName === undefined) { - logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Data Mine not defined. Data Mine = " + JSON.stringify(botNode.parentNode)); + logger.write(MODULE_NAME, "[ERROR] raiseEvents -> codeName at Mine not defined. Mine = " + JSON.stringify(botNode.parentNode)); throw (global.DEFAULT_FAIL_RESPONSE); } } else { - logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Bot not attached to a Data Mine or Trading Mine. Bot = " + JSON.stringify(botNode)); + logger.write(MODULE_NAME, "[ERROR] raiseEvents -> Bot not attached to a Data Mine | Trading Mine | Learning Mine. Bot = " + JSON.stringify(botNode)); throw (global.DEFAULT_FAIL_RESPONSE); } } else { @@ -165,7 +171,7 @@ let product = outputDataset.referenceParent.parentNode.config.codeName let dataset = outputDataset.referenceParent.config.codeName - key = dataMine + "-" + botName + "-" + product + "-" + dataset + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset + let key = dataMine + "-" + botName + "-" + product + "-" + dataset + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset let event = { lastFile: lastFile, timeFrames: timeFrames diff --git a/ReinforcementLearning/LearningBot.js b/ReinforcementLearning/LearningBot.js new file mode 100644 index 000000000..ad887979f --- /dev/null +++ b/ReinforcementLearning/LearningBot.js @@ -0,0 +1,685 @@ +exports.newLearningBot = function newLearningBot(bot, parentLogger) { + + const MODULE_NAME = "Learning Bot"; + const FULL_LOG = true; + + const TRADING_PROCESS_MODULE = require(global.ROOT_DIR + '/LowFrequencyTrading/MachineLearningProcess.js'); + const FILE_STORAGE = require('./FileStorage.js'); + const SESSION = require(global.ROOT_DIR + 'TradingSession'); + + let fileStorage = FILE_STORAGE.newFileStorage(parentLogger); + let session = SESSION.newLearningSession(bot, parentLogger) + + const DEBUG_MODULE = require(global.ROOT_DIR + 'DebugLog'); + let logger; // We need this here in order for the loopHealth function to work and be able to rescue the loop when it gets in trouble. + + let nextLoopTimeoutHandle; + + let thisObject = { + initialize: initialize, + run: run + }; + + let processConfig; + + return thisObject; + + function initialize(pProcessConfig, callBackFunction) { + /* This function is exactly the same in the 3 modules representing the 2 different bot types loops. */ + try { + processConfig = pProcessConfig; + if (bot.definedByUI === true) { + /* The code of the bot is defined at the UI. No need to load a file with the code. */ + session.initialize(processConfig, callBackFunction) + } + + } catch (err) { + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function run(callBackFunction) { + try { + /* Some initial values*/ + let fixedTimeLoopIntervalHandle; + bot.STOP_SESSION = true; + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, '[IMPORTANT] run -> Stopping the Session now. ') } + + /* Heartbeats sent to the UI */ + bot.processHeartBeat = processHeartBeat + + loop(); + + function loop() { + try { + processHeartBeat(undefined, undefined, "Running...") + function pad(str, max) { + str = str.toString(); + return str.length < max ? pad(" " + str, max) : str; + } + + /* For each loop we want to create a new log file. */ + if (logger !== undefined) { + logger.finalize() + } + logger = DEBUG_MODULE.newDebugLog(); + global.LOGGER_MAP.set(MODULE_NAME, logger) + logger.bot = bot; + logger.initialize(); + + bot.loopCounter++; + bot.loopStartTime = new Date().valueOf(); + + let nextWaitTime; + + /* We tell the UI that we are running. */ + processHeartBeat(undefined, undefined, "Running...") + + /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ + const UTILITIES = require(global.ROOT_DIR + 'CloudUtilities'); + const STATUS_REPORT = require(global.ROOT_DIR + 'StatusReport'); + const STATUS_DEPENDENCIES = require(global.ROOT_DIR + 'StatusDependencies'); + const DATA_DEPENDENCIES = require(global.ROOT_DIR + 'DataDependencies'); + const DATA_SET = require(global.ROOT_DIR + 'DataSet'); + const PROCESS_EXECUTION_EVENTS = require(global.ROOT_DIR + 'ProcessExecutionEvents'); + const PROCESS_OUTPUT = require(global.ROOT_DIR + 'ProcessOutput'); + + /* We define the datetime for the process that we are running now. This will be the official processing time for both the infraestructure and the bot. */ + bot.processDatetime = new Date(); // This will be considered the process date and time, so as to have it consistenly all over the execution. + + /* High level log entry */ + console.log(new Date().toISOString() + " " + pad(bot.exchange, 20) + " " + pad(bot.market.baseAsset + '/' + bot.market.quotedAsset, 10) + " " + pad(bot.codeName, 30) + " " + pad(bot.process, 30) + + " Main Loop # " + pad(Number(bot.loopCounter), 8) + " " + bot.processNode.session.type + " " + bot.processNode.session.name) + + /* Checking if we need to need to emit any event */ + if (bot.SESSION_STATUS === 'Idle' && bot.STOP_SESSION === false) { + bot.SESSION_STATUS = 'Running' + } + + if (bot.SESSION_STATUS === 'Running' && bot.STOP_SESSION === true) { + bot.SESSION_STATUS = 'Stopped' + } + + global.EMIT_SESSION_STATUS (bot.SESSION_STATUS, bot.sessionKey) + + /* Checking if we should process this loop or not.*/ + if (bot.STOP_SESSION === true) { + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be run."); } + + console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + + " Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be run. "); + + nextWaitTime = 'Waiting for Session'; + loopControl(nextWaitTime); + return + } + + /* We will prepare first the infraestructure needed for the bot to run. There are 3 modules we need to sucessfullly initialize first. */ + + let processExecutionEvents + let userBot; + let processFramework; + let statusDependencies; + let dataDependencies; + + initializeProcessExecutionEvents(); + + function initializeProcessExecutionEvents() { + try { + processExecutionEvents = PROCESS_EXECUTION_EVENTS.newProcessExecutionEvents(bot, logger) + processExecutionEvents.initialize(processConfig, onInizialized); + + function onInizialized(err) { + try { + switch (err.result) { + case global.DEFAULT_OK_RESPONSE.result: { + logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeProcessExecutionEvents -> onInizialized -> Execution finished well."); + startProcessExecutionEvents() + return; + } + case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeProcessExecutionEvents -> onInizialized -> Retry Later. Requesting Execution Retry."); + nextWaitTime = 'Retry'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeProcessExecutionEvents -> onInizialized -> Operation Failed. Aborting the process."); + global.unexpectedError = err.message + processStopped() + return + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeProcessExecutionEvents -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeProcessExecutionEvents -> onInizialized -> Unhandled err.result received. -> err = " + err.message); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeProcessExecutionEvents -> onInizialized -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeProcessExecutionEvents -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + + function startProcessExecutionEvents() { + try { + processExecutionEvents.start(onStarted); + + function onStarted(err) { + try { + switch (err.result) { + case global.DEFAULT_OK_RESPONSE.result: { + logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessExecutionEvents -> onStarted -> Execution finished well."); + + if (global.STOP_TASK_GRACEFULLY === true) { + loopControl() + return + } + + initializeStatusDependencies(); + return; + } + case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. + logger.write(MODULE_NAME, "[WARN] run -> loop -> startProcessExecutionEvents -> onStarted -> Retry Later. Requesting Execution Retry."); + nextWaitTime = 'Retry'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessExecutionEvents -> onStarted -> Operation Failed. Aborting the process."); + global.unexpectedError = err.message + processStopped() + return + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessExecutionEvents -> onStarted -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessExecutionEvents -> onStarted -> Unhandled err.result received. -> err = " + err.message); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessExecutionEvents -> onStarted -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessExecutionEvents -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + + function initializeStatusDependencies() { + try { + statusDependencies = STATUS_DEPENDENCIES.newStatusDependencies(bot, logger, STATUS_REPORT, UTILITIES, PROCESS_OUTPUT); + statusDependencies.initialize(onInizialized); + + function onInizialized(err) { + try { + switch (err.result) { + case global.DEFAULT_OK_RESPONSE.result: { + logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeStatusDependencies -> onInizialized -> Execution finished well."); + initializeDataDependencies(); + return; + } + case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeStatusDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); + nextWaitTime = 'Retry'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Operation Failed. Aborting the process."); + global.unexpectedError = err.message + processStopped() + return + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Unhandled err.result received. -> err = " + err.message); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + + function initializeDataDependencies() { + try { + dataDependencies = DATA_DEPENDENCIES.newDataDependencies(bot, logger, DATA_SET); + dataDependencies.initialize(onInizialized); + + function onInizialized(err) { + try { + switch (err.result) { + case global.DEFAULT_OK_RESPONSE.result: { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> onInizialized -> Execution finished well."); } + switch (processConfig.framework.name) { + case 'Low-Frequency-Trading-Process': { + processFramework = TRADING_PROCESS_MODULE.newMachineLearningProcess(bot, logger, UTILITIES); + intitializeProcessFramework(); + break; + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Process Framework not Supported."); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Process Framework Name = " + processConfig.framework.name); + global.unexpectedError = err.message + processStopped() + return + } + } + return; + } + case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeDataDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); + nextWaitTime = 'Retry'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Operation Failed. Aborting the process."); + global.unexpectedError = err.message + processStopped() + return + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Unhandled err.result received. -> err = " + err.message); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + + function intitializeProcessFramework() { + try { + processFramework.initialize(processConfig, statusDependencies, dataDependencies, onInizialized); + + function onInizialized(err) { + try { + switch (err.result) { + case global.DEFAULT_OK_RESPONSE.result: { + logger.write(MODULE_NAME, "[INFO] run -> loop -> intitializeProcessFramework -> onInizialized -> Execution finished well."); + startProcessFramework(); + return; + } + case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. + logger.write(MODULE_NAME, "[WARN] run -> loop -> intitializeProcessFramework -> onInizialized -> Retry Later. Requesting Execution Retry."); + nextWaitTime = 'Retry'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> Operation Failed. Aborting the process."); + global.unexpectedError = err.message + processStopped() + return + } + case global.CUSTOM_OK_RESPONSE.result: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized > Unhandled custom response received. -> err = " + err.message); + global.unexpectedError = err.message + processStopped() + return + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> Unhandled err.result received. -> err = " + err.message); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + + function startProcessFramework() { + try { + processFramework.start(onFinished); + + function onFinished(err) { + try { + processFramework.finalize() + processFramework = undefined + dataDependencies.finalize() + dataDependencies = undefined + statusDependencies.finalize() + statusDependencies = undefined + + switch (err.result) { + case global.DEFAULT_OK_RESPONSE.result: { + logger.write(MODULE_NAME, "[INFO] run -> loop -> startProcessFramework -> onFinished -> Execution finished well."); + nextWaitTime = 'Normal'; + finishProcessExecutionEvents() + return; + } + case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. + logger.write(MODULE_NAME, "[WARN] run -> loop -> startProcessFramework -> onFinished -> Retry Later. Requesting Execution Retry."); + nextWaitTime = 'Retry'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Operation Failed. Aborting the process."); + global.unexpectedError = err.message + processStopped() + return + } + case global.CUSTOM_OK_RESPONSE.result: { + + switch (err.message) { + case "Dependency does not exist.": { + logger.write(MODULE_NAME, "[WARN] run -> loop -> startProcessFramework -> onFinished -> Dependency does not exist. This Loop will go to sleep."); + nextWaitTime = 'Sleep'; + loopControl(nextWaitTime); + return; + } + case "Dependency not ready.": { + logger.write(MODULE_NAME, "[WARN] run -> loop -> startProcessFramework -> onFinished -> Dependency not ready. This Loop will go to sleep."); + nextWaitTime = 'Sleep'; + loopControl(nextWaitTime); + return; + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Unhandled custom response received. -> err = " + err.message); + global.unexpectedError = err.message + processStopped() + return + } + } + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Unhandled err.result received. -> err = " + err.message); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + + function finishProcessExecutionEvents() { + try { + processExecutionEvents.finish(onFinished); + + function onFinished(err) { + try { + processExecutionEvents.finalize() + processExecutionEvents = undefined + + switch (err.result) { + case global.DEFAULT_OK_RESPONSE.result: { + logger.write(MODULE_NAME, "[INFO] run -> loop -> finishProcessExecutionEvents -> onFinished -> Execution finished well."); + nextWaitTime = 'Normal'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. + logger.write(MODULE_NAME, "[WARN] run -> loop -> finishProcessExecutionEvents -> onFinished -> Retry Later. Requesting Execution Retry."); + nextWaitTime = 'Retry'; + loopControl(nextWaitTime); + return; + } + case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + logger.write(MODULE_NAME, "[ERROR] run -> loop -> finishProcessExecutionEvents -> onFinished -> Operation Failed. Aborting the process."); + global.unexpectedError = err.message + processStopped() + return + } + default: { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> finishProcessExecutionEvents -> onFinished -> Unhandled err.result received. -> err.result = " + err.result); + logger.write(MODULE_NAME, "[ERROR] run -> loop -> finishProcessExecutionEvents -> onFinished -> Unhandled err.result received. -> err = " + err.message); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> finishProcessExecutionEvents -> onFinished -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> finishProcessExecutionEvents -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + + function loopControl(nextWaitTime) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> nextWaitTime = " + nextWaitTime); } + + /* We show we reached the end of the loop. */ + processHeartBeat(undefined, undefined, "Running...") + + /* Here we check if we must stop the loop gracefully. */ + shallWeStop(onStop, onContinue); + + function onStop() { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> onStop -> Stopping the Loop Gracefully. See you next time!"); } + + if (global.WRITE_LOGS_TO_FILES === 'true') { + logger.persist(); + } + + global.EVENT_SERVER_CLIENT.raiseEvent(bot.sessionKey, 'Stopped') + processStopped() + return; + } + + function onContinue() { + /* Indicator bots are going to be executed after a configured period of time after the last execution ended. This is to avoid overlapping executions. */ + switch (nextWaitTime) { + case 'Waiting for Session': { + let waitTime = processConfig.sessionRunWaitTime + nextLoopTimeoutHandle = setTimeout(loop, waitTime); + let waitingTime = waitTime / 1000 / 60 + let label = 'minute/s' + if (waitingTime < 1) { + waitingTime = waitTime / 1000 + label = 'seconds' + } + processHeartBeat(undefined, undefined, "Waiting " + waitingTime + " " + label + " for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be run. ") + logger.persist(); + } + break + case 'Normal': { + let waitTime + if (processConfig.waitsForExecutionFinishedEvent === true) { + waitTime = 0 + } else { + switch (bot.LEARNING_SESSION.type) { + case 'Live Trading Session': { + waitTime = bot.LEARNING_SESSION.learningParameters.timeFrame.config.value + break + } + case 'Fordward Tessting Session': { + waitTime = bot.LEARNING_SESSION.learningParameters.timeFrame.config.value + break + } + case 'Paper Trading Session': { + waitTime = bot.LEARNING_SESSION.learningParameters.timeFrame.config.value + break + } + case 'Backtesting Session': { + waitTime = 0 + break + } + } + } + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (waitTime / 1000 / 60) + " minute/s."); } + nextLoopTimeoutHandle = setTimeout(loop, waitTime); + let waitingTime = waitTime / 1000 / 60 + let label = 'minute/s' + if (waitingTime < 1) { + waitingTime = waitTime / 1000 + label = 'seconds' + } + processHeartBeat(undefined, undefined, "Waiting " + waitingTime + " " + label + " for next execution.") + logger.persist(); + } + break; + case 'Retry': { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.retryWaitTime / 1000) + " seconds."); } + nextLoopTimeoutHandle = setTimeout(loop, processConfig.retryWaitTime); + processHeartBeat(undefined, undefined, "Trying to recover from some problem. Waiting " + processConfig.retryWaitTime / 1000 + " seconds for next execution.") + logger.persist(); + } + break; + case 'Sleep': { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.sleepWaitTime / 60000) + " minutes."); } + nextLoopTimeoutHandle = setTimeout(loop, processConfig.sleepWaitTime); + processHeartBeat(undefined, undefined, "Waiting " + processConfig.sleepWaitTime / 60000 + " minutes for next execution.") + logger.persist(); + } + break; + case 'Coma': { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> loopControl -> Restarting Loop in " + (processConfig.comaWaitTime / 3600000) + " hours."); } + nextLoopTimeoutHandle = setTimeout(loop, processConfig.comaWaitTime); + processHeartBeat(undefined, undefined, "Waiting " + processConfig.comaWaitTime / 3600000 + " hours for next execution.") + logger.persist(); + } + break; + } + } + } + + function shallWeStop(stopCallBack, continueCallBack) { + try { + /* IMPORTANT: This function is exactly the same on the 3 modules. */ + if (!global.STOP_TASK_GRACEFULLY) { + continueCallBack(); + } else { + stopCallBack(); + } + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] run -> loop -> shallWeStop -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + + } catch (err) { + parentLogger.write(MODULE_NAME, "[ERROR] run -> loop -> err = " + err.stack); + global.unexpectedError = err.message + processStopped() + return + } + } + + function processHeartBeat(processingDate, percentage, status) { + let event = { + seconds: (new Date()).getSeconds(), + processingDate: processingDate, + percentage: percentage, + status: status + } + global.EVENT_SERVER_CLIENT.raiseEvent(bot.processKey, 'Heartbeat', event) + } + + function processStopped() { + if (global.unexpectedError !== undefined) { + global.PROCESS_ERROR(bot.processKey, undefined, "An unexpected error caused the Process to stop.") + } else { + global.EVENT_SERVER_CLIENT.raiseEvent(bot.processKey, 'Stopped') + } + sessionStopped() + logger.persist(); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + if (global.unexpectedError !== undefined) { + callBackFunction(global.DEFAULT_FAIL_RESPONSE) + } else { + callBackFunction(global.DEFAULT_OK_RESPONSE) + } + } + + function sessionStopped() { + if (bot.SESSION_STATUS === 'Running') { + global.EVENT_SERVER_CLIENT.raiseEvent(bot.sessionKey, 'Stopped') + bot.SESSION_STATUS = 'Stopped' + } + } + + } catch (err) { + parentLogger.write(MODULE_NAME, "[ERROR] run -> err = " + err.stack); + clearInterval(fixedTimeLoopIntervalHandle); + clearTimeout(nextLoopTimeoutHandle); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } +}; diff --git a/ReinforcementLearning/LearningProcess.js b/ReinforcementLearning/LearningProcess.js new file mode 100644 index 000000000..9cb44f672 --- /dev/null +++ b/ReinforcementLearning/LearningProcess.js @@ -0,0 +1,786 @@ +exports.newLearningProcess = function newLearningProcess(bot, logger, UTILITIES) { + /* + This Module will load all the process data dependencies from files and send them downstream. + After execution, will save the time range and status report of the process. + */ + const FULL_LOG = true; + const MODULE_NAME = "Learning Process"; + const GMT_SECONDS = ':00.000 GMT+0000'; + + thisObject = { + initialize: initialize, + finalize: finalize, + start: start + }; + + let utilities = UTILITIES.newCloudUtilities(logger); + + let statusDependencies; + let dataDependenciesModule; + let dataFiles = new Map(); + let multiPeriodDataFiles = new Map(); + + const FILE_STORAGE = require('../FileStorage.js'); + let fileStorage = FILE_STORAGE.newFileStorage(logger); + + const MARKOV_DECISION_PROCESS_MODULE = require('./MarkovDecisionProcess.js') + let markovDecisionProcessModule = MARKOV_DECISION_PROCESS_MODULE.newMarkovDecisionProcess(bot, logger) + + let ALGORITHM_MODULE = require("./Algorithm") + let algorithmModule = ALGORITHM_MODULE.newTradingOutput(bot, logger, markovDecisionProcessModule, UTILITIES, FILE_STORAGE) + + let processConfig; + + return thisObject; + + function initialize(pProcessConfig, pStatusDependencies, pDataDependencies, callBackFunction) { + try { + logger.fileName = MODULE_NAME + logger.initialize() + + statusDependencies = pStatusDependencies + dataDependenciesModule = pDataDependencies + processConfig = pProcessConfig + + algorithmModule.initialize() + + callBackFunction(global.DEFAULT_OK_RESPONSE) + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + function finalize() { + + markovDecisionProcessModule.finalize() + markovDecisionProcessModule = undefined + + algorithmModule.finalize() + algorithmModule = undefined + + dataFiles = undefined + multiPeriodDataFiles = undefined + statusDependencies = undefined + dataDependenciesModule = undefined + fileStorage = undefined + processConfig = undefined + thisObject = undefined + } + + async function start(callBackFunction) { + try { + let currentTimeFrame + let currentTimeFrameLabel + + /* Context Variables */ + let contextVariables = { + lastFile: undefined, // Datetime of the last file files sucessfully produced by this process. + dateBeginOfMarket: undefined, // Datetime of the first trade file in the whole market history. + dateEndOfMarket: undefined // Datetime of the last file available to be used as an input of this process. + }; + + getContextVariables() + + if (bot.FIRST_EXECUTION === true && bot.RESUME === false) { + /* + Here is where the Trading Engine and Trading Systems received are moved to the simulation state. + */ + bot.simulationState.tradingEngine = bot.TRADING_ENGINE + bot.simulationState.tradingSystem = bot.TRADING_SYSTEM + } + + /* We set up the Trading Engine Module. */ + markovDecisionProcessModule.initialize() + + /* Initializing the Trading Process Date */ + if (bot.FIRST_EXECUTION === true && bot.RESUME === false) { + /* + This funcion is going to be called many times by the Trading Bot Loop. + Only during the first execution and when the User is not resuming the execution + of a stopped session / task; we are going to initialize the Process Date Time. + This variable tell us which day we are standing at, specially while working + with Daily Files. From this Date is that we are going to load the Daily Files. + */ + bot.simulationState.tradingEngine.current.episode.processDate.value = global.REMOVE_TIME(bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime).valueOf() + } + + /* + This is the Date that is going to be used across the execution of this Trading Process. + We need this because it has a different life cycle than the processData stored at the + Trading Engine data structure. This date has to remain the same during the whole execution + of the Trading Process until the end, inclusind the writting of Data Ranges and Status Reports. + The processDate of the Trading Engine data structure on the other hand can be changed during + the simulation loop, once we discover that all candles from a certain date have benn processed. + Here is the point where we sync one and the other. + */ + let LearningProcessDate = global.REMOVE_TIME(bot.simulationState.tradingEngine.current.episode.processDate.value) + await processSingleFiles() + + if (await processMarketFiles() === false) { + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waiting for Data Mining to be run') + callBackFunction(global.DEFAULT_RETRY_RESPONSE) + return + } + /* + This is the Data Structure used at the Simulation with all indicator data. + We start creating it right here. + */ + let chart = {} + /* + Here we check if we need to get Daily Files or not. As an optimization, when + we are running on a Time Frame of 1hs or above, we are not going to load + dependencies on Daily Files. The way we recognize that is by checking if + we alreaady set a value to currentTimeFrame. We are also not going to loop + through days if we are processing market files. + */ + if (currentTimeFrame) { + /* We are processing Market Files */ + /* + With all the indicators data files loaded, we will build the chart object + data structure that will be used in user-defied conditions and formulas. + */ + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waking up') + buildCharts(chart) + + if (checkThereAreCandles(chart) === true) { + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Running') + await generateOutput(chart) + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Saving') + await writeProcessFiles() + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Sleeping') + } else { + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waiting for Data Mining to be up to date. No candles found at.') + callBackFunction(global.DEFAULT_RETRY_RESPONSE) + return + } + + } else { + /* We are processing Daily Files */ + do { + global.EMIT_SESSION_STATUS(bot.TRADING_SESSION_STATUS, bot.TRADING_SESSIONKey) + /* + We update the Trading Process Date with the date calculated at the simulation. + We will use this date to load indicator and output files. After that we will + use it to save Output Files and later the Data Ranges. This is the point where + the date calculated by the Simulation is applied at the Trading Process Level. + */ + LearningProcessDate = global.REMOVE_TIME(bot.simulationState.tradingEngine.current.episode.processDate.value) + + if (checkStopTaskGracefully() === false) { break } + if (checkStopProcessing() === false) { break } + + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waking up') + if (await processDailyFiles() === false) { + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waiting for Data Mining to be run') + callBackFunction(global.DEFAULT_RETRY_RESPONSE) + return + } + /* + With all the indicators data files loaded, we will build the chart object + data structure that will be used in user-defied conditions and formulas. + */ + buildCharts(chart) + /* + The process of generating the output includes the trading simulation. + */ + if (checkThereAreCandles(chart) === true) { + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Running') + await generateOutput(chart) + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Saving') + await writeProcessFiles() + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Sleeping') + } else { + bot.TRADING_SESSIONHeartBeat(undefined, undefined, 'Waiting for Data Mining to be up to date') + callBackFunction(global.DEFAULT_RETRY_RESPONSE) + return + } + /* + If for any reason the session was stopped, we will break this loop and exit the process. + */ + if (bot.STOP_SESSION === true) { break } + /* + When we get to the end of the market, we need to break this process loop in order + to let time pass, new information be collected from the exchange, new data built + into indicators, and eventually a new execution of this process. + */ + if (checkStopHeadOfTheMarket() === false) { break } + + } + while (true) + } + + checkIfSessionMustStop() + /* + Everything worked as expected. We return an OK code and wait for + the Bot Loop to call us again later. + */ + callBackFunction(global.DEFAULT_OK_RESPONSE) + + function checkThereAreCandles(chart) { + let sessionParameters = bot.TRADING_SESSION.tradingParameters + let propertyName = 'at' + sessionParameters.timeFrame.config.label.replace('-', '') + let candles = chart[propertyName].candles + + if (candles.length === 0) { + return false + } else { + return true + } + } + + function getContextVariables() { + try { + let thisReport; + let statusReport; + + /* We are going to use the start date as beging of market date. */ + contextVariables.dateBeginOfMarket = global.REMOVE_TIME(bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime) + /* + Here we get the status report from the bot who knows which is the end of the market. + */ + statusReport = statusDependencies.reportsByMainUtility.get("Market Ending Point") + + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return + } + + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because dependecy Status Report is corrupt. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return + } + + thisReport = statusReport.file; + + if (thisReport.lastFile === undefined) { + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> thisReport = " + JSON.stringify(thisReport)); + + let customOK = { + result: global.CUSTOM_OK_RESPONSE.result, + message: "Dependency not ready." + } + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> customOK = " + customOK.message); + callBackFunction(customOK); + return + } + + contextVariables.dateEndOfMarket = new Date(thisReport.lastFile.valueOf()); + + /* Finally we get our own Status Report. */ + statusReport = statusDependencies.reportsByMainUtility.get("Self Reference") + + if (statusReport === undefined) { // This means the status report does not exist, that could happen for instance at the begining of a month. + logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Status Report does not exist. Retrying Later. "); + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return + } + + if (statusReport.status === "Status Report is corrupt.") { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> Can not continue because self dependecy Status Report is corrupt. Aborting Process."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return + } + + thisReport = statusReport.file; + bot.simulationState = thisReport.simulationState; + if (bot.simulationState === undefined) { bot.simulationState = {} } // This should happen only when there is no status report + + if (thisReport.lastFile !== undefined) { + if (bot.RESUME !== true) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Starting from the begining because bot has just started and resume execution was true."); } + startFromBegining(); + return + } + contextVariables.lastFile = new Date(thisReport.lastFile); + return + + } else { + /* + We are here becuse either: + 1. There is no status report + 2. There is no Last File (this happens on Market Files) + */ + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Starting from the begining of the market because own status report not found or lastFile was undefined."); } + startFromBegining(); + return + } + + function startFromBegining() { + contextVariables.lastFile = new Date(contextVariables.dateBeginOfMarket.getUTCFullYear() + "-" + (contextVariables.dateBeginOfMarket.getUTCMonth() + 1) + "-" + contextVariables.dateBeginOfMarket.getUTCDate() + " " + "00:00" + GMT_SECONDS); + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> startFromBegining -> contextVariables.lastFile = " + contextVariables.lastFile); } + } + + } catch (err) { + logger.write(MODULE_NAME, "[ERROR] start -> getContextVariables -> err = " + err.stack); + if (err.message === "Cannot read property 'file' of undefined") { + logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Check the bot configuration to see if all of its statusDependencies declarations are correct. "); + logger.write(MODULE_NAME, "[HINT] start -> getContextVariables -> Dependencies loaded -> keys = " + JSON.stringify(statusDependencies.keys)); + } + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } + + async function processSingleFiles() { + dataFiles = new Map(); + + for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.nodeArray.length; dependencyIndex++) { + let dependency = dataDependenciesModule.nodeArray[dependencyIndex]; + let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]; + + if (datasetModule.node.config.codeName !== "Single-File") { + continue + } + + if (dataDependenciesModule.isItADepenency('atAnyTimeFrame', datasetModule.node.parentNode.config.singularVariableName) !== true) { + continue + } + + let fileName = "Data.json"; + let filePath = datasetModule.node.parentNode.config.codeName + '/' + datasetModule.node.config.codeName; + + /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */ + let response = await asyncGetDatasetFile(datasetModule, filePath, fileName) + + if (response.err.result !== global.DEFAULT_OK_RESPONSE.result) { + throw (response.err) + } + + let dataFile = JSON.parse(response.text); + dataFiles.set(dependency.id, dataFile); + } + + let mapKey = "Single Files" + multiPeriodDataFiles.set(mapKey, dataFiles) + } + + async function processMarketFiles() { + /* + We do market files first since if the simulation is run on daily files, there will + be a loop to get each of those files and we do not need that loop to reload market files. + + We will iterate through all posible timeFrames. + */ + for (let n = 0; n < global.marketFilesPeriods.length; n++) { + const timeFrame = global.marketFilesPeriods[n][0]; + const timeFrameLabel = global.marketFilesPeriods[n][1]; + + dataFiles = new Map(); + + if (bot.TRADING_SESSION.tradingParameters.timeFrame.config.label === timeFrameLabel) { + currentTimeFrame = global.marketFilesPeriods[n][0]; + currentTimeFrameLabel = global.marketFilesPeriods[n][1]; + } + for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.nodeArray.length; dependencyIndex++) { + let dependency = dataDependenciesModule.nodeArray[dependencyIndex]; + let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]; + + if (dependency.referenceParent.config.codeName !== "Multi-Period-Market") { + continue + } + + if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.config.singularVariableName) !== true) { + if (!(bot.TRADING_SESSION.tradingParameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { + continue + } + } + + let fileName = "Data.json"; + let filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel; + + /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */ + let response = await asyncGetDatasetFile(datasetModule, filePath, fileName) + + if (response.err.message === 'File does not exist.') { + return false + } + + if (response.err.result !== global.DEFAULT_OK_RESPONSE.result) { + throw (response.err) + } + + let dataFile = JSON.parse(response.text); + let trimmedDataFile = trimDataFile(dataFile, datasetModule.node.parentNode.record) + dataFiles.set(dependency.id, trimmedDataFile); + + function trimDataFile(dataFile, recordDefinition) { + /* + Here we will discard all the records in a file that are outside of the current time range. + We will include the las element previous to the begining of the time range. This is needed + because during the simulation, the current period is not the open one, but the previous to + the open, and if we do not include the previous to the initial datetime there will be no + current objects at the begining of the simulation for many time frames. + */ + let beginIndex + let endIndex + let result = [] + for (let i = 0; i < recordDefinition.properties.length; i++) { + let property = recordDefinition.properties[i] + if (property.config.codeName === 'begin') { + beginIndex = i + } + if (property.config.codeName === 'end') { + endIndex = i + } + } + for (let i = 0; i < dataFile.length; i++) { + let dataRecord = dataFile[i] + let begin = dataRecord[beginIndex] + let end = dataRecord[endIndex] + if (end + timeFrame < bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime - 1) { continue } // /1 because we need the previous closed element + if (begin > bot.TRADING_SESSION.tradingParameters.timeRange.config.finalDatetime) { continue } + result.push(dataRecord) + } + return result + } + } + + let mapKey = global.marketFilesPeriods[n][1]; + multiPeriodDataFiles.set(mapKey, dataFiles) + } + return true + } + + async function processDailyFiles() { + /* Telling the world we are alive and doing well and which date we are processing right now. */ + let processingDateString = LearningProcessDate.getUTCFullYear() + '-' + utilities.pad(LearningProcessDate.getUTCMonth() + 1, 2) + '-' + utilities.pad(LearningProcessDate.getUTCDate(), 2); + bot.processHeartBeat(processingDateString, undefined, "Running...") + + /* + We will iterate through all posible timeFrames. + */ + for (let n = 0; n < global.dailyFilePeriods.length; n++) { + const timeFrame = global.dailyFilePeriods[n][0]; + const timeFrameLabel = global.dailyFilePeriods[n][1]; + + if (processConfig.framework.validtimeFrames !== undefined) { + let validPeriod = false; + for (let i = 0; i < processConfig.framework.validtimeFrames.length; i++) { + let period = processConfig.framework.validtimeFrames[i]; + if (period === timeFrameLabel) { validPeriod = true } + } + if (validPeriod === false) { + continue + } + } + + if (bot.TRADING_SESSION.tradingParameters.timeFrame.config.label === timeFrameLabel) { + currentTimeFrame = global.dailyFilePeriods[n][0]; + currentTimeFrameLabel = global.dailyFilePeriods[n][1]; + } + + dataFiles = new Map(); + + /* + We will iterate through all dependencies, in order to load the + files that later will end up at the chart data structure. + */ + for (let dependencyIndex = 0; dependencyIndex < dataDependenciesModule.nodeArray.length; dependencyIndex++) { + let dependency = dataDependenciesModule.nodeArray[dependencyIndex]; + let datasetModule = dataDependenciesModule.dataSetsModulesArray[dependencyIndex]; + + if (dependency.referenceParent.config.codeName !== "Multi-Period-Daily") { + continue + } + + if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.config.singularVariableName) !== true) { + if (!(bot.TRADING_SESSION.tradingParameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) { + continue + } + } + + let dateForPath = LearningProcessDate.getUTCFullYear() + '/' + utilities.pad(LearningProcessDate.getUTCMonth() + 1, 2) + '/' + utilities.pad(LearningProcessDate.getUTCDate(), 2); + let filePath + if (dependency.referenceParent.config.codeName === "Multi-Period-Daily") { + filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel + "/" + dateForPath; + } else { + filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + dateForPath; + } + let fileName = "Data.json"; + + /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */ + let response = await asyncGetDatasetFile(datasetModule, filePath, fileName) + + if (response.err.message === 'File does not exist.') { + return false + } + if (response.err.result !== global.DEFAULT_OK_RESPONSE.result) { + throw (response.err) + } + + let dataFile = JSON.parse(response.text); + dataFiles.set(dependency.id, dataFile); + } + + let mapKey = global.dailyFilePeriods[n][1]; + multiPeriodDataFiles.set(mapKey, dataFiles) + } + return true + } + + function checkStopHeadOfTheMarket() { + /* + We need to check if we have reached the head of the market in order to know + when to break the Daily Files Process loop and give time for a new candles / + indicators to be built and continue the processing once this process is called + again. + */ + if (bot.simulationState.tradingEngine.current.episode.headOfTheMarket.value === true) { + return false + } + } + + function checkStopTaskGracefully() { + /* Validation that we dont need to stop. */ + if (global.STOP_TASK_GRACEFULLY === true) { + return false + } + } + + function checkStopProcessing() { + /* Validation that we dont need to stop. */ + if (bot.STOP_SESSION === true) { + return false + } + } + + function buildCharts(chart) { + const COMMONS = require('../Commons.js') + let commons = COMMONS.newCommons(bot, logger, UTILITIES, FILE_STORAGE) + + let mainDependency = {} + + /* The first phase here is about checking that we have everything we need at the definition level. */ + let dataDependencies = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processDependencies, 'Data Dependency') + /* + We will filter ourt declared dependencies that are not present in the workspace. + This will allow the user to have less Data Mines loaded at the workspace + that the ones that a Trading Mine depends on. + */ + dataDependencies = global.FILTER_OUT_NODES_WITHOUT_REFERENCE_PARENT_FROM_NODE_ARRAY(dataDependencies) + + if (commons.validateDataDependencies(dataDependencies, callBackFunction) !== true) { return } + + let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset') + if (commons.validateOutputDatasets(outputDatasets, callBackFunction) !== true) { return } + + /* The second phase is about transforming the inputs into a format that can be used to apply the user defined code. */ + for (let j = 0; j < global.marketFilesPeriods.length; j++) { + let timeFrameLabel = marketFilesPeriods[j][1] + let dataFiles = multiPeriodDataFiles.get(timeFrameLabel) + let products = {} + + if (dataFiles !== undefined) { + commons.inflateDatafiles(dataFiles, dataDependencies, products, mainDependency, currentTimeFrame) + + let propertyName = 'at' + timeFrameLabel.replace('-', '') + chart[propertyName] = products + } + } + + for (let j = 0; j < global.dailyFilePeriods.length; j++) { + let timeFrameLabel = global.dailyFilePeriods[j][1] + let dataFiles = multiPeriodDataFiles.get(timeFrameLabel) + let products = {} + + if (dataFiles !== undefined) { + commons.inflateDatafiles(dataFiles, dataDependencies, products, mainDependency, currentTimeFrame) + + let propertyName = 'at' + timeFrameLabel.replace('-', '') + chart[propertyName] = products + } + } + + /* Single Files */ + let dataFiles = multiPeriodDataFiles.get('Single Files') + let products = {} + + if (dataFiles !== undefined) { + commons.inflateDatafiles(dataFiles, dataDependencies, products, mainDependency, currentTimeFrame) + + let propertyName = 'atAnyTimeFrame' + chart[propertyName] = products + } + } + + async function generateOutput(chart) { + await algorithmModule.start( + chart, + currentTimeFrame, + currentTimeFrameLabel, + LearningProcessDate + ) + + /* + From here on, all other loops executions wont be the first execution and also + we will consider that it is not resuming a previous execution as well. + */ + bot.FIRST_EXECUTION = false + bot.RESUME = false + } + + async function writeProcessFiles() { + let outputDatasets = global.NODE_BRANCH_TO_ARRAY(bot.processNode.referenceParent.processOutput, 'Output Dataset') + + await writeTimeFramesFiles(currentTimeFrame, currentTimeFrameLabel) + await writeDataRanges() + + if (currentTimeFrame > global.dailyFilePeriods[0][0]) { + await writeMarketStatusReport(currentTimeFrameLabel) + } else { + await writeDailyStatusReport(currentTimeFrameLabel) + } + + async function writeDataRanges() { + for ( + let outputDatasetIndex = 0; + outputDatasetIndex < outputDatasets.length; + outputDatasetIndex++ + ) { + let productCodeName = outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; + await writeDataRange(productCodeName); + } + + async function writeDataRange(productCodeName) { + let dataRange = { + begin: contextVariables.dateBeginOfMarket.valueOf(), + end: LearningProcessDate.valueOf() + global.ONE_DAY_IN_MILISECONDS + } + + let fileContent = JSON.stringify(dataRange); + let fileName = '/Data.Range.json'; + let filePath = bot.filePathRoot + "/Output/" + bot.TRADING_SESSION.folderName + "/" + productCodeName + "/" + 'Multi-Period-Daily' + fileName; + + let response = await fileStorage.asyncCreateTextFile(filePath, fileContent + '\n') + + if (response.err.result !== global.DEFAULT_OK_RESPONSE.result) { + throw (response.err) + } + + /* + Raise the event that the Data Range was Updated. + */ + let key = bot.dataMine + "-" + bot.codeName + "-" + productCodeName + "-" + bot.exchange + "-" + bot.market.baseAsset + '/' + bot.market.quotedAsset + let event = { + dateRange: dataRange + } + + global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Data Range Updated', event) + } + } + + async function writeTimeFramesFiles(currentTimeFrame, currentTimeFrameLabel) { + + for ( + let outputDatasetIndex = 0; + outputDatasetIndex < outputDatasets.length; + outputDatasetIndex++ + ) { + let productCodeName = outputDatasets[outputDatasetIndex].referenceParent.parentNode.config.codeName; + await writeTimeFramesFile(currentTimeFrameLabel, productCodeName, 'Multi-Period-Daily') + await writeTimeFramesFile(currentTimeFrameLabel, productCodeName, 'Multi-Period-Market') + + async function writeTimeFramesFile(currentTimeFrameLabel, productCodeName, processType) { + + let timeFramesArray = [] + timeFramesArray.push(currentTimeFrameLabel) + + let fileContent = JSON.stringify(timeFramesArray) + let fileName = '/Time.Frames.json'; + + let filePath = bot.filePathRoot + "/Output/" + bot.TRADING_SESSION.folderName + "/" + productCodeName + "/" + processType + fileName; + + let response = await fileStorage.asyncCreateTextFile(filePath, fileContent + '\n') + if (response.err.result !== global.DEFAULT_OK_RESPONSE.result) { + throw (response.err) + } + } + } + } + + async function writeDailyStatusReport(currentTimeFrameLabel) { + let reportKey = bot.dataMine + "-" + bot.codeName + "-" + bot.processNode.referenceParent.config.codeName + let thisReport = statusDependencies.statusReports.get(reportKey) + + thisReport.file.lastExecution = bot.currentDaytime + thisReport.file.lastFile = LearningProcessDate + thisReport.file.simulationState = bot.simulationState + thisReport.file.timeFrames = currentTimeFrameLabel + await thisReport.asyncSave() + } + + async function writeMarketStatusReport(currentTimeFrameLabel) { + let reportKey = bot.dataMine + "-" + bot.codeName + "-" + bot.processNode.referenceParent.config.codeName + let thisReport = statusDependencies.statusReports.get(reportKey); + + thisReport.file.lastExecution = bot.processDatetime + thisReport.file.simulationState = bot.simulationState + thisReport.file.timeFrames = currentTimeFrameLabel + + logger.newInternalLoop(bot.codeName, bot.process, LearningProcessDate) + await thisReport.asyncSave() + } + } + + function checkIfSessionMustStop() { + + if (bot.TRADING_SESSION.type === 'Backtesting Session') { + /* + Backtests needs only one execution of this process to complete. + */ + if (FULL_LOG === true) { logger.write(MODULE_NAME, '[IMPORTANT] checkIfSessionMustStop -> Backtesting Session Finished. Stopping the Session now. ') } + bot.TRADING_SESSION.stop('Backtesting Session Finished.') + } + } + } + catch (err) { + /* An unhandled exception occured. in this case we return Fail and log the stack. */ + if (err.stack) { + logger.write(MODULE_NAME, "[ERROR] start -> Unhandled Exception. Will Abort this process. err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return + } + + /* Some expected file was not found. We will return a RETRY code and move on. */ + if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { + logger.write(MODULE_NAME, "[ERROR] File not Found. Will Retry the Process Loop.") + callBackFunction(global.DEFAULT_RETRY_RESPONSE); + return + } + + /* Some other handled exception occured. We return Fail and move on. */ + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write(MODULE_NAME, "[ERROR] start -> Handled Exception. Will Abort this process. err = " + err.message); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return + } + } + } + + function pad(str, max) { + str = str.toString(); + return str.length < max ? pad(" " + str, max) : str; + } + + async function asyncGetDatasetFile(datasetModule, filePath, fileName) { + + let promise = new Promise((resolve, reject) => { + + datasetModule.getTextFile(filePath, fileName, onFileReceived); + function onFileReceived(err, text) { + + let response = { + err: err, + text: text + } + resolve(response) + } + }); + + return promise + } +} diff --git a/ReinforcementLearning/MarkovDecisionProcess.js b/ReinforcementLearning/MarkovDecisionProcess.js new file mode 100644 index 000000000..ca46398dc --- /dev/null +++ b/ReinforcementLearning/MarkovDecisionProcess.js @@ -0,0 +1,218 @@ +exports.newMarkovDecisionProcess = function newMarkovDecisionProcess(bot, logger) { + /* + We call the Trading Engine to the data structure that is needed in order to exevute the + trading protocol with the specific rules defined at the Trading System. + */ + const MODULE_NAME = 'Markov Decision Process' + let thisObject = { + mantain: mantain, + reset: reset, + getNodeById: getNodeById, + cloneValues: cloneValues, + setCurrentCandle: setCurrentCandle, + setCurrentCycle: setCurrentCycle, + initializeNode: initializeNode, + initialize: initialize, + finalize: finalize + } + + let tradingEngine + let sessionParameters + let nodesMap = new Map() + + return thisObject + + function initialize() { + + tradingEngine = bot.simulationState.tradingEngine + sessionParameters = bot.LEARNING_SESSION.learningParameters + + initializeNodeMap(tradingEngine) + + if (bot.FIRST_EXECUTION === true) { + /* + Here we will go through all the nodes in the Trading Engine hiriarchy and + apply the initial value to the value property when needed + */ + initializeNode(tradingEngine) + } + } + + function finalize() { + tradingEngine = undefined + sessionParameters = undefined + nodesMap = undefined + } + + function mantain() { + } + + function reset() { + + } + + function getNodeById(NodeId) { + return nodesMap.get(NodeId) + } + + function setCurrentCandle(candle) { + tradingEngine.current.episode.candle.begin.value = candle.begin + tradingEngine.current.episode.candle.end.value = candle.end + tradingEngine.current.episode.candle.open.value = candle.open + tradingEngine.current.episode.candle.close.value = candle.close + tradingEngine.current.episode.candle.min.value = candle.min + tradingEngine.current.episode.candle.max.value = candle.max + } + + function setCurrentCycle(cycle) { + tradingEngine.current.episode.cycle.value = cycle + tradingEngine.current.episode.cycle.lastBegin.value = tradingEngine.current.episode.cycle.begin.value + tradingEngine.current.episode.cycle.lastEnd.value = tradingEngine.current.episode.cycle.end.value + switch (cycle) { + case 'First': { + tradingEngine.current.episode.cycle.begin.value = + tradingEngine.current.episode.candle.begin.value + + sessionParameters.timeFrame.config.value + tradingEngine.current.episode.cycle.end.value = + tradingEngine.current.episode.candle.begin.value + + sessionParameters.timeFrame.config.value + + sessionParameters.timeFrame.config.value / 2 - 1 + break + } + case 'Second': { + tradingEngine.current.episode.cycle.begin.value = + tradingEngine.current.episode.candle.begin.value + + sessionParameters.timeFrame.config.value + + sessionParameters.timeFrame.config.value / 2 + tradingEngine.current.episode.cycle.end.value = + tradingEngine.current.episode.candle.begin.value + + sessionParameters.timeFrame.config.value + + sessionParameters.timeFrame.config.value * 4 / 4 - 1 + break + } + } + } + + function cloneValues(originNode, destinationNode) { + if (originNode === undefined) { return } + if (destinationNode === undefined) { return } + + /* Here copy the node value */ + destinationNode.value = originNode.value + + /* Now we go down through all the children of the origin node, assuming the destination node has the same children structure*/ + let nodeDefinition = global.APP_SCHEMA_MAP.get(originNode.type) + if (nodeDefinition === undefined) { return } + + if (nodeDefinition.properties !== undefined) { + let previousPropertyName // Since there are cases where there are many properties with the same name,because they can hold nodes of different types but only one at the time, we have to avoind counting each property of those as individual children. + for (let i = 0; i < nodeDefinition.properties.length; i++) { + let property = nodeDefinition.properties[i] + + switch (property.type) { + case 'node': { + if (property.name !== previousPropertyName) { + if (originNode[property.name] !== undefined && destinationNode[property.name] !== undefined) { + cloneValues(originNode[property.name], destinationNode[property.name]) + } + previousPropertyName = property.name + } + break + } + case 'array': { + if (originNode[property.name] !== undefined && destinationNode[property.name] !== undefined) { + let originNodePropertyArray = originNode[property.name] + let destinationNodePropertyArray = destinationNode[property.name] + for (let m = 0; m < originNodePropertyArray.length; m++) { + cloneValues(originNodePropertyArray[m], destinationNodePropertyArray[m]) + } + } + break + } + } + } + } + } + + function initializeNodeMap(node) { + if (node === undefined) { return } + /* Add to Node Map */ + nodesMap.set(node.id, node) + + /* Now we go down through all this node children */ + let nodeDefinition = global.APP_SCHEMA_MAP.get(node.type) + if (nodeDefinition === undefined) { return } + + if (nodeDefinition.properties !== undefined) { + let previousPropertyName // Since there are cases where there are many properties with the same name,because they can hold nodes of different types but only one at the time, we have to avoind counting each property of those as individual children. + for (let i = 0; i < nodeDefinition.properties.length; i++) { + let property = nodeDefinition.properties[i] + + switch (property.type) { + case 'node': { + if (property.name !== previousPropertyName) { + if (node[property.name] !== undefined) { + initializeNodeMap(node[property.name]) + } + previousPropertyName = property.name + } + break + } + case 'array': { + if (node[property.name] !== undefined) { + let nodePropertyArray = node[property.name] + for (let m = 0; m < nodePropertyArray.length; m++) { + initializeNodeMap(nodePropertyArray[m]) + } + } + break + } + } + } + } + } + + function initializeNode(node) { + if (node === undefined) { return } + + /* Here we initialize the node value */ + if (node.config !== undefined) { + if (node.config.initialValue !== undefined) { + node.value = node.config.initialValue + } + } + + /* Now we go down through all this node children */ + let nodeDefinition = global.APP_SCHEMA_MAP.get(node.type) + if (nodeDefinition === undefined) { return } + + if (nodeDefinition.properties !== undefined) { + let previousPropertyName // Since there are cases where there are many properties with the same name,because they can hold nodes of different types but only one at the time, we have to avoind counting each property of those as individual children. + for (let i = 0; i < nodeDefinition.properties.length; i++) { + let property = nodeDefinition.properties[i] + + switch (property.type) { + case 'node': { + if (property.name !== previousPropertyName) { + if (node[property.name] !== undefined) { + initializeNode(node[property.name]) + } + previousPropertyName = property.name + } + break + } + case 'array': { + if (node[property.name] !== undefined) { + let nodePropertyArray = node[property.name] + for (let m = 0; m < nodePropertyArray.length; m++) { + initializeNode(nodePropertyArray[m]) + } + } + break + } + } + } + } + } +} + diff --git a/Root.js b/Root.js index 10ac254d5..198f15944 100644 --- a/Root.js +++ b/Root.js @@ -266,6 +266,10 @@ runTradingBot(botConfig, processConfig); break; } + case 'Learning Bot': { + runLearningBot(botConfig, processConfig); + break; + } default: { console.log(logDisplace + "Root : [ERROR] start -> bootingBot -> Unexpected bot type. -> botConfig.type = " + botConfig.type); } @@ -296,6 +300,10 @@ runTradingBot(botConfig, processConfig); break; } + case 'Learning Bot': { + runLearningBot(botConfig, processConfig); + break; + } default: { console.log(logDisplace + "Root : [ERROR] start -> bootingBot -> Unexpected bot type. -> botConfig.type = " + botConfig.type); } @@ -339,6 +347,10 @@ runTradingBot(botConfig, processConfig); break; } + case 'Learning Bot': { + runLearningBot(botConfig, processConfig); + break; + } default: { console.log(logDisplace + "Root : [ERROR] start -> bootingBot -> Unexpected bot type. -> botConfig.type = " + botConfig.type); } @@ -540,6 +552,71 @@ setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) } } + + function runLearningBot(pBotConfig, pProcessConfig) { + + global.TOTAL_PROCESS_INSTANCES_CREATED++ + + try { + const DEBUG_MODULE = require(global.ROOT_DIR + 'DebugLog'); + let logger; + + logger = DEBUG_MODULE.newDebugLog(); + global.LOGGER_MAP.set('runLearningBot', logger) + logger.bot = pBotConfig; + + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runLearningBot -> Entering function."); } + + let learningBot = TRADING_BOT_MODULE.newLearningBot(pBotConfig, logger); + learningBot.initialize(pProcessConfig, onInitializeReady); + + function onInitializeReady(err) { + + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + + learningBot.run(whenRunFinishes); + + function whenRunFinishes(err) { + + pBotConfig.loopCounter = 0; + + let botId; + + botId = pBotConfig.dataMine + "." + pBotConfig.codeName + "." + pBotConfig.process; + + if (err.result === global.DEFAULT_OK_RESPONSE.result) { + + logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runLearningBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished sucessfully."); + logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runLearningBot -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); + + logger.persist(); + + } else { + + logger.write(MODULE_NAME, "[ERROR] start -> bootingBot -> runLearningBot -> onInitializeReady -> whenStartFinishes -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> bootingBot -> runLearningBot -> onInitializeReady -> whenStartFinishes -> Execution will be stopped. "); + logger.write(MODULE_NAME, "[ERROR] start -> bootingBot -> runLearningBot -> onInitializeReady -> whenStartFinishes -> Bye."); + logger.write(MODULE_NAME, "[ERROR] start -> bootingBot -> runLearningBot -> onInitializeReady -> whenStartFinishes -> Bot Id = " + botId); + console.log(logDisplace + "Root : [ERROR] start -> bootingBot -> runLearningBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); + logger.persist(); + } + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) + } + + } else { + logger.write(MODULE_NAME, "[ERROR] start -> bootingBot -> runLearningBot -> onInitializeReady -> err = " + err.message); + logger.write(MODULE_NAME, "[ERROR] start -> bootingBot -> runLearningBot -> onInitializeReady -> Failed to initialize the bot. "); + console.log(logDisplace + "Root : [ERROR] start -> bootingBot -> runLearningBot -> onInitializeReady -> err = " + err.message); + logger.persist(); + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) + } + } + } + catch (err) { + console.log(logDisplace + "Root : [ERROR] start -> bootingBot -> runLearningBot -> err = " + err.stack); + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) + } + } } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> bootingBot -> err = " + err.stack); diff --git a/SensorBot.js b/SensorBot.js index 13e5c251d..e4c1f6a70 100644 --- a/SensorBot.js +++ b/SensorBot.js @@ -63,7 +63,7 @@ function onCommonsDownloaded(err, text) { if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - parentLogger.write(MODULE_NAME, "[WARN] initialize -> onBotDownloaded -> onCommonsDownloaded -> Commons not found: " + err.code || err.message); + parentLogger.write(MODULE_NAME, "[WARN] initialize -> onBotDownloaded -> onCommonsDownloaded -> Commons not found: " + err.message); callBackFunction(global.DEFAULT_OK_RESPONSE); return; } diff --git a/SocialBots.js b/SocialBots.js index 9ae190727..4676a912f 100644 --- a/SocialBots.js +++ b/SocialBots.js @@ -13,10 +13,10 @@ exports.newSocialBots = function newSocialBots(bot, logger) { return thisObject function initialize() { - if (bot.SESSION.socialBots !== undefined) { - if (bot.SESSION.socialBots.bots !== undefined) { - for (let i = 0; i < bot.SESSION.socialBots.bots.length; i++) { - let socialBot = bot.SESSION.socialBots.bots[i] + if (bot.TRADING_SESSION.socialBots !== undefined) { + if (bot.TRADING_SESSION.socialBots.bots !== undefined) { + for (let i = 0; i < bot.TRADING_SESSION.socialBots.bots.length; i++) { + let socialBot = bot.TRADING_SESSION.socialBots.bots[i] if (socialBot.type === "Telegram Bot") { let config = socialBot.config socialBot.botInstance = TELEGRAM_BOT_MODULE.newTelegramBot(bot, logger) @@ -26,9 +26,9 @@ exports.newSocialBots = function newSocialBots(bot, logger) { } function announce(text) { - if (bot.SESSION.socialBots.bots !== undefined) { - for (let i = 0; i < bot.SESSION.socialBots.bots.length; i++) { - let socialBot = bot.SESSION.socialBots.bots[i] + if (bot.TRADING_SESSION.socialBots.bots !== undefined) { + for (let i = 0; i < bot.TRADING_SESSION.socialBots.bots.length; i++) { + let socialBot = bot.TRADING_SESSION.socialBots.bots[i] try { if (socialBot.type === "Telegram Bot") { socialBot.botInstance.telegramAPI.sendMessage(socialBot.botInstance.chatId, text).catch(err => logger.write(MODULE_NAME, "[WARN] initialize -> initializeSocialBots -> announce -> Telegram API error -> err = " + err)) @@ -40,15 +40,15 @@ exports.newSocialBots = function newSocialBots(bot, logger) { } } - bot.SESSION.socialBots.announce = announce + bot.TRADING_SESSION.socialBots.announce = announce } } function finalize() { - if (bot.SESSION.socialBots === undefined) { return } - if (bot.SESSION.socialBots.bots !== undefined) { - for (let i = 0; i < bot.SESSION.socialBots.bots.length; i++) { - let socialBot = bot.SESSION.socialBots.bots[i] + if (bot.TRADING_SESSION.socialBots === undefined) { return } + if (bot.TRADING_SESSION.socialBots.bots !== undefined) { + for (let i = 0; i < bot.TRADING_SESSION.socialBots.bots.length; i++) { + let socialBot = bot.TRADING_SESSION.socialBots.bots[i] if (socialBot.type === "Telegram Bot") { socialBot.botInstance.finalize() } @@ -57,10 +57,10 @@ exports.newSocialBots = function newSocialBots(bot, logger) { } function sendMessage(message) { - if (bot.SESSION.socialBots !== undefined) { - if (bot.SESSION.socialBots.bots !== undefined) { - for (let i = 0; i < bot.SESSION.socialBots.bots.length; i++) { - let socialBot = bot.SESSION.socialBots.bots[i] + if (bot.TRADING_SESSION.socialBots !== undefined) { + if (bot.TRADING_SESSION.socialBots.bots !== undefined) { + for (let i = 0; i < bot.TRADING_SESSION.socialBots.bots.length; i++) { + let socialBot = bot.TRADING_SESSION.socialBots.bots[i] if (socialBot.type === "Telegram Bot") { socialBot.botInstance.sendMessage(message) } diff --git a/StatusReport.js b/StatusReport.js index 21089be6d..2a7699e55 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -100,8 +100,12 @@ } } - if (bot.SESSION !== undefined && statusDependencyNode.bottype === "Trading Bot") { - sessionPath = bot.SESSION.folderName + "/" + if (bot.TRADING_SESSION !== undefined && statusDependencyNode.bottype === "Trading Bot") { + sessionPath = bot.TRADING_SESSION.folderName + "/" + } + + if (bot.LEARNING_SESSION !== undefined && statusDependencyNode.bottype === "Learning Bot") { + sessionPath = bot.LEARNING_SESSION.folderName + "/" } /* Now we will see where do we need to fetch this status report from. */ @@ -264,7 +268,7 @@ On the contraty, if we are running a Sensor bot or an Indicator bot, we might, if necesary, use a previous version of a Status Report since there will be no big impact, just some reprocessing. */ - if (bot.SESSION !== undefined) { + if (bot.TRADING_SESSION !== undefined || bot.LEARNING_SESSION !== undefined) { canUserPrevious = false } else { canUserPrevious = true diff --git a/TelegramBot.js b/TelegramBot.js index 9e2383ed1..79b7c5212 100644 --- a/TelegramBot.js +++ b/TelegramBot.js @@ -23,7 +23,7 @@ exports.newTelegramBot = function newTelegramBot(bot, logger) { thisObject.telegramBot = new Telegraf(botToken) thisObject.telegramBot.start((ctx) => ctx.reply('Hi! I am the Telegram Bot that will tell you what happens with your trading session.')) thisObject.telegramBot.help((ctx) => ctx.reply('I can stop the session if you type STOP.')) - thisObject.telegramBot.hears('STOP', (ctx) => bot.SESSION.stop(' by STOP command at Telegram.')) + thisObject.telegramBot.hears('STOP', (ctx) => bot.TRADING_SESSION.stop(' by STOP command at Telegram.')) thisObject.telegramBot.launch() const Telegram = require('telegraf/telegram') diff --git a/TradingBot.js b/TradingBot.js index 9d4d48ee7..064a3a3c2 100644 --- a/TradingBot.js +++ b/TradingBot.js @@ -5,10 +5,10 @@ const TRADING_PROCESS_MODULE = require(global.ROOT_DIR + '/LowFrequencyTrading/TradingProcess.js'); const FILE_STORAGE = require('./FileStorage.js'); - const SESSION = require(global.ROOT_DIR + 'Session'); + const SESSION = require(global.ROOT_DIR + 'TradingSession'); let fileStorage = FILE_STORAGE.newFileStorage(parentLogger); - let session = SESSION.newSession(bot, parentLogger) + let session = SESSION.newTradingSession(bot, parentLogger) const DEBUG_MODULE = require(global.ROOT_DIR + 'DebugLog'); let logger; // We need this here in order for the loopHealth function to work and be able to rescue the loop when it gets in trouble. @@ -93,23 +93,23 @@ + " Main Loop # " + pad(Number(bot.loopCounter), 8) + " " + bot.processNode.session.type + " " + bot.processNode.session.name) /* Checking if we need to need to emit any event */ - if (bot.SESSION_STATUS === 'Idle' && bot.STOP_SESSION === false) { - bot.SESSION_STATUS = 'Running' + if (bot.TRADING_SESSION_STATUS === 'Idle' && bot.STOP_SESSION === false) { + bot.TRADING_SESSION_STATUS = 'Running' } - if (bot.SESSION_STATUS === 'Running' && bot.STOP_SESSION === true) { - bot.SESSION_STATUS = 'Stopped' + if (bot.TRADING_SESSION_STATUS === 'Running' && bot.STOP_SESSION === true) { + bot.TRADING_SESSION_STATUS = 'Stopped' } - global.EMIT_SESSION_STATUS (bot.SESSION_STATUS, bot.sessionKey) + global.EMIT_SESSION_STATUS (bot.TRADING_SESSION_STATUS, bot.TRADING_SESSIONKey) /* Checking if we should process this loop or not.*/ if (bot.STOP_SESSION === true) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be ran."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be run."); } console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) - + " Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be ran. "); + + " Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be run. "); nextWaitTime = 'Waiting for Session'; loopControl(nextWaitTime); @@ -251,7 +251,6 @@ global.unexpectedError = err.message processStopped() return - return; } default: { logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Unhandled err.result received. -> err.result = " + err.result); @@ -535,7 +534,7 @@ logger.persist(); } - global.EVENT_SERVER_CLIENT.raiseEvent(bot.sessionKey, 'Stopped') + global.EVENT_SERVER_CLIENT.raiseEvent(bot.TRADING_SESSIONKey, 'Stopped') processStopped() return; } @@ -552,7 +551,7 @@ waitingTime = waitTime / 1000 label = 'seconds' } - processHeartBeat(undefined, undefined, "Waiting " + waitingTime + " " + label + " for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be ran. ") + processHeartBeat(undefined, undefined, "Waiting " + waitingTime + " " + label + " for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be run. ") logger.persist(); } break @@ -561,17 +560,17 @@ if (processConfig.waitsForExecutionFinishedEvent === true) { waitTime = 0 } else { - switch (bot.SESSION.type) { + switch (bot.TRADING_SESSION.type) { case 'Live Trading Session': { - waitTime = bot.SESSION.parameters.timeFrame.config.value + waitTime = bot.TRADING_SESSION.tradingParameters.timeFrame.config.value break } case 'Fordward Tessting Session': { - waitTime = bot.SESSION.parameters.timeFrame.config.value + waitTime = bot.TRADING_SESSION.tradingParameters.timeFrame.config.value break } case 'Paper Trading Session': { - waitTime = bot.SESSION.parameters.timeFrame.config.value + waitTime = bot.TRADING_SESSION.tradingParameters.timeFrame.config.value break } case 'Backtesting Session': { @@ -670,9 +669,9 @@ } function sessionStopped() { - if (bot.SESSION_STATUS === 'Running') { - global.EVENT_SERVER_CLIENT.raiseEvent(bot.sessionKey, 'Stopped') - bot.SESSION_STATUS = 'Stopped' + if (bot.TRADING_SESSION_STATUS === 'Running') { + global.EVENT_SERVER_CLIENT.raiseEvent(bot.TRADING_SESSIONKey, 'Stopped') + bot.TRADING_SESSION_STATUS = 'Stopped' } } diff --git a/TradingSession.js b/TradingSession.js new file mode 100644 index 000000000..5999d48dd --- /dev/null +++ b/TradingSession.js @@ -0,0 +1,519 @@ +exports.newTradingSession = function newTradingSession(bot, parentLogger) { + + const MODULE_NAME = "Trading Session" + const FULL_LOG = true; + + let thisObject = { + initialize: initialize + } + + const SOCIAL_BOTS_MODULE = require('./SocialBots.js') + let socialBotsModule = SOCIAL_BOTS_MODULE.newSocialBots(bot, parentLogger) + + return thisObject; + + function initialize(pProcessConfig, callBackFunction) { + try { + /* Initialize this info so that everything is logged propeerly */ + bot.TRADING_SESSION = { + name: bot.processNode.session.name, + id: bot.processNode.session.id + } + + /* Set the folderName for early logging */ + if (bot.processNode.session.config.folderName === undefined) { + bot.TRADING_SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.TRADING_SESSION.id + } else { + bot.TRADING_SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.processNode.session.config.folderName + } + + /* Check if there is a session */ + if (bot.processNode.session === undefined) { + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> Cannot run without a Session."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + + /* Listen to event to start or stop the session. */ + bot.TRADING_SESSIONKey = bot.processNode.session.name + '-' + bot.processNode.session.type + '-' + bot.processNode.session.id + global.SESSION_MAP.set(bot.TRADING_SESSIONKey, bot.TRADING_SESSIONKey) + + global.EVENT_SERVER_CLIENT.listenToEvent(bot.TRADING_SESSIONKey, 'Trading Session Status', undefined, bot.TRADING_SESSIONKey, undefined, onSessionStatus) + global.EVENT_SERVER_CLIENT.listenToEvent(bot.TRADING_SESSIONKey, 'Run Trading Session', undefined, bot.TRADING_SESSIONKey, undefined, onSessionRun) + global.EVENT_SERVER_CLIENT.listenToEvent(bot.TRADING_SESSIONKey, 'Stop Trading Session', undefined, bot.TRADING_SESSIONKey, undefined, onSessionStop) + global.EVENT_SERVER_CLIENT.listenToEvent(bot.TRADING_SESSIONKey, 'Resume Trading Session', undefined, bot.TRADING_SESSIONKey, undefined, onSessionResume) + + /* Connect this here so that it is accesible from other places */ + bot.TRADING_SESSIONError = sessionError + bot.TRADING_SESSIONWarning = sessionWarning + bot.TRADING_SESSIONInfo = sessionInfo + + /* Heartbeats sent to the UI */ + bot.TRADING_SESSIONHeartBeat = sessionHeartBeat + + callBackFunction(global.DEFAULT_OK_RESPONSE) + return + + function onSessionStatus() { + if (bot.TRADING_SESSION_STATUS === 'Running') { + let event = { + status: 'Trading Session Runnning' + } + global.EVENT_SERVER_CLIENT.raiseEvent(bot.TRADING_SESSIONKey, 'Status Response', event) + } else { + let event = { + status: 'Trading Session Not Runnning' + } + global.EVENT_SERVER_CLIENT.raiseEvent(bot.TRADING_SESSIONKey, 'Status Response', event) + } + } + + function onSessionRun(message) { + try { + /* This happens when the UI is reloaded, the session was running and tries to run it again. */ + if (bot.TRADING_SESSION_STATUS === 'Idle' || bot.TRADING_SESSION_STATUS === 'Running') { + parentLogger.write(MODULE_NAME, "[WARN] onSessionRun -> Event received to run the Session while it was already running. ") + return + } + + /* We are going to run the Definition comming at the event. */ + bot.TRADING_SYSTEM = JSON.parse(message.event.tradingSystem) + bot.TRADING_ENGINE = JSON.parse(message.event.tradingEngine) + bot.TRADING_SESSION = JSON.parse(message.event.session) + bot.DEPENDENCY_FILTER = JSON.parse(message.event.dependencyFilter) + bot.RESUME = false + bot.FIRST_EXECUTION = true + bot.TRADING_SESSION.stop = stopSession // stop function + + setUpSessionFolderName() + + /* We validate all parameters received and complete some that might be missing if needed. */ + if (checkParemeters() === false) { return } + + socialBotsModule.initialize() + + let allGood + switch (bot.TRADING_SESSION.type) { + case 'Backtesting Session': { + allGood = startBackTesting(message) + break + } + case 'Live Trading Session': { + allGood = startLiveTrading(message) + break + } + case 'Fordward Testing Session': { + allGood = startFordwardTesting(message) + break + } + case 'Paper Trading Session': { + allGood = startPaperTrading(message) + break + } + } + if (allGood === true) { + bot.TRADING_SESSION_STATUS = 'Idle' + bot.STOP_SESSION = false + } else { + bot.STOP_SESSION = true + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, '[IMPORTANT] onSessionRun -> Stopping the Session now. ') } + } + + socialBotsModule.sendMessage(bot.TRADING_SESSION.type + " '" + bot.TRADING_SESSION.name + "' is starting.") + } catch (err) { + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onSessionRun -> err = " + err.stack); + } + } + + function onSessionStop() { + stopSession('Session Stopped From the User Interface.') + } + + function onSessionResume(message) { + try { + if (bot.TRADING_SESSION.stop === undefined) { + parentLogger.write(MODULE_NAME, "[WARN] onSessionResume -> Event received to resume the Session that have never be ran before. ") + return + } + + /* This happens when the UI is reloaded, the session was running and tries to run it again. */ + if (bot.TRADING_SESSION_STATUS === 'Idle' || bot.TRADING_SESSION_STATUS === 'Running') { + parentLogger.write(MODULE_NAME, "[WARN] onSessionResume -> Event received to resume the Session while it was already running. ") + return + } + + bot.RESUME = true + bot.STOP_SESSION = false + + socialBotsModule.sendMessage(bot.TRADING_SESSION.type + " '" + bot.TRADING_SESSION.name + "' is resuming.") + } catch (err) { + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onSessionResume -> err = " + err.stack); + } + } + + function stopSession(commandOrigin) { + + socialBotsModule.sendMessage(bot.TRADING_SESSION.type + " '" + bot.TRADING_SESSION.name + "' is stopping " + commandOrigin) + socialBotsModule.finalize() + bot.STOP_SESSION = true + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, '[IMPORTANT] stopSession -> Stopping the Session now. ') } + sessionInfo(bot.TRADING_SESSION, commandOrigin) + } + + function setUpSessionFolderName() { + /* + The session object is overwritten when the session is run. For that reason we + need to setup again the folder name at the Session level. + Set the folderName for logging, reports, context and data output + */ + let config + if (bot.TRADING_SESSION.config !== undefined) { + config = bot.TRADING_SESSION.config + if (config.folderName === undefined) { + bot.TRADING_SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.TRADING_SESSION.id + } else { + bot.TRADING_SESSION.folderName = bot.processNode.session.type.replace(' ', '-').replace(' ', '-') + '-' + bot.processNode.session.config.folderName + } + } + } + + function checkParemeters() { + /* + Here we check all the Session Parameters received. If something critical is missing we abort returning false. If something + non critical is missing, we complete it with a default value. + */ + + if (bot.TRADING_SESSION.tradingParameters === undefined) { + let errorMessage = "Session Node with no Parameters." + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) + bot.TRADING_SESSIONError(bot.TRADING_SESSION, errorMessage) + return false + } + + /* Time Range */ + if (bot.TRADING_SESSION.tradingParameters.timeRange === undefined) { // if the Time Range is missing we create a default one. + bot.TRADING_SESSION.tradingParameters.timeRange = { + name: 'Missing Time Range', + type: 'Time Range', + config: { + initialDatetime: (new Date()).valueOf(), + finalDatetime: (new Date()).valueOf() + global.ONE_YEAR_IN_MILISECONDS + } + } + } else { + /* Check that we received valid dates */ + if (bot.TRADING_SESSION.type === 'Backtesting Session') { + if (isNaN(new Date(bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime)).valueOf()) { + let errorMessage = "sessionParameters.timeRange.config.initialDatetime is not a valid date." + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) + bot.TRADING_SESSIONError(bot.TRADING_SESSION.tradingParameters, errorMessage) + return false + } + } + if (isNaN(new Date(bot.TRADING_SESSION.tradingParameters.timeRange.config.finalDatetime)).valueOf()) { + let errorMessage = "sessionParameters.timeRange.config.initialDatetime is not a valid date." + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) + bot.TRADING_SESSIONError(bot.TRADING_SESSION.tradingParameters, errorMessage) + return false + } + } + + /* Session Type Forced Values */ + let today = (new Date()).valueOf() + let aYearAgo = today - global.ONE_YEAR_IN_MILISECONDS + let aYearFromNow = today + global.ONE_YEAR_IN_MILISECONDS + switch (bot.TRADING_SESSION.type) { + case 'Backtesting Session': { + useDefaultDatetimes(aYearAgo, today) + break + } + case 'Live Trading Session': { + useDefaultDatetimes(today, aYearFromNow) + break + } + case 'Fordward Testing Session': { + useDefaultDatetimes(today, aYearFromNow) + break + } + case 'Paper Trading Session': { + useDefaultDatetimes(today, aYearFromNow) + break + } + } + + function useDefaultDatetimes(initialDefault, finalDefault) { + /* + Note that inside the system, we are going to deal with these + dates in their numeric value representation. + */ + + /* Initial Datetime */ + if (bot.TRADING_SESSION.type === 'Backtesting Session') { + if (bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime === undefined) { + bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime = initialDefault + } else { + bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime = (new Date(bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime)).valueOf() + } + } else { + /* Non backtest session can start from the past only if explicitly configured that way */ + if (bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime === undefined || bot.TRADING_SESSION.tradingParameters.timeRange.config.allowStartingFromThePast !== true) { + bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime = initialDefault + } else { + bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime = (new Date(bot.TRADING_SESSION.tradingParameters.timeRange.config.initialDatetime)).valueOf() + } + } + + /* Final Datetime */ + if (bot.TRADING_SESSION.tradingParameters.timeRange.config.finalDatetime === undefined) { + bot.TRADING_SESSION.tradingParameters.timeRange.config.finalDatetime = finalDefault + } else { + bot.TRADING_SESSION.tradingParameters.timeRange.config.finalDatetime = (new Date(bot.TRADING_SESSION.tradingParameters.timeRange.config.finalDatetime)).valueOf() + } + } + + /* Time Frame */ + if (bot.TRADING_SESSION.tradingParameters.timeFrame === undefined) { + let errorMessage = "Session Parameters Node with no Time Frame." + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) + bot.TRADING_SESSIONError(bot.TRADING_SESSION.tradingParameters, errorMessage) + return false + } + if (bot.TRADING_SESSION.tradingParameters.timeFrame.config.label === undefined) { + let errorMessage = "Session Parameters Node with no Time Frame Label configuration." + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) + bot.TRADING_SESSIONError(bot.TRADING_SESSION.tradingParameters.timeFrame, errorMessage) + return false + } + bot.TRADING_SESSION.tradingParameters.timeFrame.config.value = getTimeFrameFromLabel(bot.TRADING_SESSION.tradingParameters.timeFrame.config.label) + if (bot.TRADING_SESSION.tradingParameters.timeFrame.config.value === undefined) { + let errorMessage = "Config error: label value not recognized. Try 01-min or 01-hs for example." + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) + bot.TRADING_SESSIONError(bot.TRADING_SESSION.tradingParameters.timeFrame, errorMessage) + return false + } + + /* Session Base Asset */ + if (bot.TRADING_SESSION.tradingParameters.sessionBaseAsset === undefined) { + let errorMessage = "Session Parameters Node with no Session Base Asset." + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) + bot.TRADING_SESSIONError(bot.TRADING_SESSION.tradingParameters, errorMessage) + return false + } + if (bot.TRADING_SESSION.tradingParameters.sessionBaseAsset.config.initialBalance === undefined) { + let errorMessage = "Session Parameters Session Base Asset with no initialBalance configuration." + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) + bot.TRADING_SESSIONError(bot.TRADING_SESSION.tradingParameters.sessionBaseAsset, errorMessage) + return false + } + + /* Session Quoted Asset */ + if (bot.TRADING_SESSION.tradingParameters.sessionQuotedAsset === undefined) { + let errorMessage = "Session Parameters Node with no Session Quoted Asset." + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) + bot.TRADING_SESSIONError(bot.TRADING_SESSION.tradingParameters, errorMessage) + return false + } + if (bot.TRADING_SESSION.tradingParameters.sessionQuotedAsset.config.initialBalance === undefined) { + let errorMessage = "Session Parameters Session Quoted Asset with no initialBalance configuration." + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> checkParemeters -> " + errorMessage) + bot.TRADING_SESSIONError(bot.TRADING_SESSION.tradingParameters.sessionQuotedAsset, errorMessage) + return false + } + + /* Slippage */ + if (bot.TRADING_SESSION.tradingParameters.slippage === undefined) { // if the Slippage is missing we create a default one. + bot.TRADING_SESSION.tradingParameters.slippage = { + name: 'Missing Slippage', + type: 'Slippage', + config: { + marketOrderRate: 0, + stopLoss: 0, + takeProfit: 0 + } + } + } + if (bot.TRADING_SESSION.tradingParameters.slippage.config.marketOrderRate === undefined) { // if the marketOrderRate is missing we create a default one. + bot.TRADING_SESSION.tradingParameters.slippage.config.marketOrderRate = 0 + } + if (bot.TRADING_SESSION.tradingParameters.slippage.config.stopLoss === undefined) { // if the stopLoss is missing we create a default one. + bot.TRADING_SESSION.tradingParameters.slippage.config.stopLoss = 0 + } + if (bot.TRADING_SESSION.tradingParameters.slippage.config.takeProfit === undefined) { // if the takeProfit is missing we create a default one. + bot.TRADING_SESSION.tradingParameters.slippage.config.takeProfit = 0 + } + + /* Fee Structure */ + if (bot.TRADING_SESSION.tradingParameters.feeStructure === undefined) { // if the Fee Structure is missing we create a default one. + bot.TRADING_SESSION.tradingParameters.feeStructure = { + name: 'Missing Fee Structure', + type: 'Fee Structure', + config: { + maker: 0, + taker: 0 + } + } + } + if (bot.TRADING_SESSION.tradingParameters.feeStructure.config.maker === undefined) { // if the maker is missing we create a default one. + bot.TRADING_SESSION.tradingParameters.feeStructure.config.maker = 0 + } + if (bot.TRADING_SESSION.tradingParameters.feeStructure.config.taker === undefined) { // if the taker is missing we create a default one. + bot.TRADING_SESSION.tradingParameters.feeStructure.config.taker = 0 + } + + return true + } + + function startBackTesting(message) { + return true + } + + function startLiveTrading(message) { + if (bot.KEY === undefined || bot.SECRET === undefined) { + let errorMessage = "Key 'codeName' or 'secret' not provided. Plese check that and try again." + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[ERROR] initialize -> startLiveTrading -> " + errorMessage); } + bot.TRADING_SESSIONError(bot.TRADING_SESSION, errorMessage) + return false + } + return true + } + + function startFordwardTesting(message) { + if (bot.KEY === undefined || bot.SECRET === undefined) { + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[WARN] initialize -> startFordwardTesting -> Key name or Secret not provided, not possible to run the process in Forward Testing mode."); } + console.log("Key 'codeName' or 'secret' not provided. Plese check that and try again.") + return false + } + + /* Reduce the balance */ + let balancePercentage = 1 // This is the default value + + if (bot.TRADING_SESSION.config.balancePercentage !== undefined) { + balancePercentage = bot.TRADING_SESSION.config.balancePercentage + } + + bot.TRADING_SESSION.tradingParameters.sessionBaseAsset.config.initialBalance = bot.TRADING_SESSION.tradingParameters.sessionBaseAsset.config.initialBalance * balancePercentage / 100 + bot.TRADING_SESSION.tradingParameters.sessionQuotedAsset.config.initialBalance = bot.TRADING_SESSION.tradingParameters.sessionQuotedAsset.config.initialBalance * balancePercentage / 100 + + bot.TRADING_SESSION.tradingParameters.sessionBaseAsset.config.minimumBalance = bot.TRADING_SESSION.tradingParameters.sessionBaseAsset.config.minimumBalance * balancePercentage / 100 + bot.TRADING_SESSION.tradingParameters.sessionQuotedAsset.config.minimumBalance = bot.TRADING_SESSION.tradingParameters.sessionQuotedAsset.config.minimumBalance * balancePercentage / 100 + + bot.TRADING_SESSION.tradingParameters.sessionBaseAsset.config.maximumBalance = bot.TRADING_SESSION.tradingParameters.sessionBaseAsset.config.maximumBalance * balancePercentage / 100 + bot.TRADING_SESSION.tradingParameters.sessionQuotedAsset.config.maximumBalance = bot.TRADING_SESSION.tradingParameters.sessionQuotedAsset.config.maximumBalance * balancePercentage / 100 + + pProcessConfig.normalWaitTime = bot.TRADING_SESSION.tradingParameters.timeFrame.config.value + + return true + } + + function startPaperTrading(message) { + return true + } + + function getTimeFrameFromLabel(timeFrameLabel) { + + for (let i = 0; i < global.marketFilesPeriods.length; i++) { + let value = global.marketFilesPeriods[i][0] + let label = global.marketFilesPeriods[i][1] + + if (timeFrameLabel === label) { + return value + } + } + + for (let i = 0; i < global.dailyFilePeriods.length; i++) { + let value = global.dailyFilePeriods[i][0] + let label = global.dailyFilePeriods[i][1] + + if (timeFrameLabel === label) { + return value + } + } + } + + function sessionHeartBeat(processingDate, percentage, status) { + let event = { + seconds: (new Date()).getSeconds(), + processingDate: processingDate, + percentage: percentage, + status: status + } + global.EVENT_SERVER_CLIENT.raiseEvent(bot.TRADING_SESSIONKey, 'Heartbeat', event) + + if (global.STOP_TASK_GRACEFULLY === true) { + bot.STOP_SESSION = true + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, '[IMPORTANT] sessionHeartBeat -> Stopping the Session now. ') } + } + } + + function sessionError(node, errorMessage) { + let event + if (node !== undefined) { + event = { + nodeName: node.name, + nodeType: node.type, + nodeId: node.id, + errorMessage: errorMessage + } + } else { + event = { + errorMessage: errorMessage + } + } + global.EVENT_SERVER_CLIENT.raiseEvent(bot.TRADING_SESSIONKey, 'Error', event) + + if (global.STOP_TASK_GRACEFULLY === true) { + bot.STOP_SESSION = true + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, '[IMPORTANT] sessionError -> Stopping the Session now. ') } + } + } + + function sessionWarning(node, warningMessage) { + let event + if (node !== undefined) { + event = { + nodeName: node.name, + nodeType: node.type, + nodeId: node.id, + warningMessage: warningMessage + } + } else { + event = { + warningMessage: warningMessage + } + } + global.EVENT_SERVER_CLIENT.raiseEvent(bot.TRADING_SESSIONKey, 'Warning', event) + + if (global.STOP_TASK_GRACEFULLY === true) { + bot.STOP_SESSION = true + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, '[IMPORTANT] sessionWarning -> Stopping the Session now. ') } + } + } + + function sessionInfo(node, infoMessage) { + let event + if (node !== undefined) { + event = { + nodeName: node.name, + nodeType: node.type, + nodeId: node.id, + infoMessage: infoMessage + } + } else { + event = { + infoMessage: infoMessage + } + } + global.EVENT_SERVER_CLIENT.raiseEvent(bot.TRADING_SESSIONKey, 'Info', event) + + if (global.STOP_TASK_GRACEFULLY === true) { + bot.STOP_SESSION = true + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, '[IMPORTANT] sessionInfo -> Stopping the Session now. ') } + } + } + + } catch (err) { + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } +}; diff --git a/server.js b/server.js index 3ff27cf3f..e693dcad8 100644 --- a/server.js +++ b/server.js @@ -1,5 +1,4 @@ - /* Callbacks default responses. */ global.DEFAULT_OK_RESPONSE = { result: "Ok", @@ -31,8 +30,8 @@ global.ROOT_DIR = './'; global.ONE_YEAR_IN_MILISECONDS = 365 * 24 * 60 * 60 * 1000 global.ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000 global.ONE_MIN_IN_MILISECONDS = 60 * 1000 -global.LOGGER_MAP = new Map() -global.SESSION_MAP = new Map() +global.LOGGER_MAP = new Map() // We will put all the loggers in a map, so that we can eventually finalize them. +global.SESSION_MAP = new Map() // We will put all the sessions in a map, so that we can eventually finalize them. global.UNIQUE_ID = function () { return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { @@ -316,7 +315,9 @@ function finalizeLoggers() { global.LOGGER_MAP.forEach(forEachLogger) function forEachLogger(logger) { - logger.finalize() + if (logger !== undefined) { + logger.finalize() + } } } @@ -375,12 +376,14 @@ global.EXIT_NODE_PROCESS = function exitProcess() { /* Cleaning Before Exiting. */ clearInterval(global.HEARTBEAT_INTERVAL_HANDLER) - for (let i = 0; i < global.TASK_NODE.bot.processes.length; i++) { - let config = global.TASK_NODE.bot.processes[i].config - let process = global.TASK_NODE.bot.processes[i] - - key = process.name + '-' + process.type + '-' + process.id - global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Stopped') // Meaning Process Stopped + if (global.TASK_NODE !== undefined) { + for (let i = 0; i < global.TASK_NODE.bot.processes.length; i++) { + let config = global.TASK_NODE.bot.processes[i].config + let process = global.TASK_NODE.bot.processes[i] + + let key = process.name + '-' + process.type + '-' + process.id + global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Stopped') // Meaning Process Stopped + } } finalizeLoggers() @@ -426,11 +429,15 @@ function preLoader() { global.EVENT_SERVER_CLIENT.listenToEvent('Task Server - ' + taskId, 'Run Task', undefined, 'Task Server - ' + taskId, undefined, eventReceived) global.EVENT_SERVER_CLIENT.raiseEvent('Task Manager - ' + taskId, 'Nodejs Process Ready for Task') function eventReceived(message) { - global.APP_SCHEMA_ARRAY = JSON.parse(message.event.appSchema) - setUpAppSchema() - global.TASK_NODE = JSON.parse(message.event.taskDefinition) - global.TASK_NETWORK = JSON.parse(message.event.networkDefinition) - bootLoader() + try { + global.APP_SCHEMA_ARRAY = JSON.parse(message.event.appSchema) + setUpAppSchema() + global.TASK_NODE = JSON.parse(message.event.taskDefinition) + global.TASK_NETWORK = JSON.parse(message.event.networkDefinition) + bootLoader() + } catch(err) { + console.log('[ERROR] Task Server -> server -> preLoader -> eventReceived -> ' + err.stack) + } } } catch (err) { console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE -> ' + err.stack) @@ -442,11 +449,16 @@ function preLoader() { //console.log('[INFO] Task Server -> server -> preLoader -> Waiting for event to start debugging...') global.EVENT_SERVER_CLIENT.listenToEvent('Task Server', 'Debug Task Started', undefined, 'Task Server', undefined, startDebugging) function startDebugging(message) { - global.APP_SCHEMA_ARRAY = JSON.parse(message.event.appSchema) - setUpAppSchema() - global.TASK_NODE = JSON.parse(message.event.taskDefinition) - global.TASK_NETWORK = JSON.parse(message.event.networkDefinition) - bootLoader() + try { + global.APP_SCHEMA_ARRAY = JSON.parse(message.event.appSchema) + setUpAppSchema() + global.TASK_NODE = JSON.parse(message.event.taskDefinition) + global.TASK_NETWORK = JSON.parse(message.event.networkDefinition) + bootLoader() + + } catch(err) { + console.log('[ERROR] Task Server -> server -> preLoader -> startDebugging -> ' + err.stack) + } } } catch (err) { console.log('[ERROR] Task Server -> server -> preLoader -> global.TASK_NODE -> ' + err.stack) @@ -502,6 +514,7 @@ function bootLoader() { global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Error', event) } + for (let processIndex = 0; processIndex < global.TASK_NODE.bot.processes.length; processIndex++) { let config = global.TASK_NODE.bot.processes[processIndex].config