From 2116dd35fe9ea8c15b9e1a093d4b9ddf0f65e475 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 20 Aug 2020 17:53:41 +0200 Subject: [PATCH] Working again --- LowFrequencyTrading/TradingOutput.js | 11 +- LowFrequencyTrading/TradingProcess.js | 37 ++++-- LowFrequencyTrading/TradingRecords.js | 1 - LowFrequencyTrading/TradingSimulation.js | 155 ++++++++++++----------- TradingBot.js | 51 +------- 5 files changed, 119 insertions(+), 136 deletions(-) diff --git a/LowFrequencyTrading/TradingOutput.js b/LowFrequencyTrading/TradingOutput.js index ac3a6ca72..47bb6ce6b 100644 --- a/LowFrequencyTrading/TradingOutput.js +++ b/LowFrequencyTrading/TradingOutput.js @@ -1,4 +1,4 @@ -exports.newTradingOutput = function newTradingOutput(bot, logger, UTILITIES, FILE_STORAGE) { +exports.newTradingOutput = function newTradingOutput(bot, logger, tradingEngineModule, UTILITIES, FILE_STORAGE) { /* This module will load if necesary all the data outputs so that they can be appended with new records if needed. After running the simulation, it will save all the data outputs. @@ -38,17 +38,12 @@ exports.newTradingOutput = function newTradingOutput(bot, logger, UTILITIES, FIL /* Preparing everything for the Simulation */ const TRADING_SIMULATION = require('./TradingSimulation.js') - let tradingSimulation = TRADING_SIMULATION.newTradingSimulation(bot, logger, UTILITIES) + let tradingSimulation = TRADING_SIMULATION.newTradingSimulation(bot, logger, tradingEngineModule, UTILITIES) let outputDatasets = bot.processNode.referenceParent.processOutput.outputDatasets let outputDatasetsMap = new Map() if (bot.FIRST_EXECUTION === true && bot.RESUME === false) { - /* - Here is where the Trading Engine and Trading Systems received are moved to the simulation state. - */ - bot.simulationState.tradingEngine = bot.TRADING_ENGINE - bot.simulationState.tradingSystem = bot.TRADING_SYSTEM await initializeOutputs() } else { await readFiles() @@ -59,6 +54,8 @@ exports.newTradingOutput = function newTradingOutput(bot, logger, UTILITIES, FIL outputDatasetsMap, writeFiles ) + + tradingSimulation.finalize() return async function initializeOutputs() { diff --git a/LowFrequencyTrading/TradingProcess.js b/LowFrequencyTrading/TradingProcess.js index 6932a8e56..f0ca70066 100644 --- a/LowFrequencyTrading/TradingProcess.js +++ b/LowFrequencyTrading/TradingProcess.js @@ -1,4 +1,4 @@ -exports.newTradingProcess = function newTradingProcess(bot, logger, UTILITIES, TRADING_OUTPUT_MODULE, USER_BOT_COMMONS) { +exports.newTradingProcess = function newTradingProcess(bot, logger, UTILITIES) { /* This Module will load all the process data dependencies from files and send them downstream. After execution, will save the time range and status report of the process. @@ -20,11 +20,15 @@ let dataFiles = new Map(); let multiPeriodDataFiles = new Map(); - let tradingOutputModule; - const FILE_STORAGE = require('../FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(logger); + const TRADING_ENGINE_MODULE = require('./TradingEngine.js') + let tradingEngineModule = TRADING_ENGINE_MODULE.newTradingEngine(bot, logger) + + let TRADING_OUTPUT_MODULE = require("./TradingOutput") + let tradingOutputModule = TRADING_OUTPUT_MODULE.newTradingOutput(bot, logger, tradingEngineModule, UTILITIES, FILE_STORAGE) + let processConfig; return thisObject; @@ -38,8 +42,6 @@ dataDependenciesModule = pDataDependencies processConfig = pProcessConfig - let TRADING_OUTPUT_MODULE = require("./TradingOutput") - tradingOutputModule = TRADING_OUTPUT_MODULE.newTradingOutput(bot, logger, UTILITIES, FILE_STORAGE) tradingOutputModule.initialize() callBackFunction(global.DEFAULT_OK_RESPONSE) @@ -51,12 +53,17 @@ } function finalize() { + + tradingEngineModule.finalize() + tradingEngineModule = undefined + + tradingOutputModule.finalize() + tradingOutputModule = undefined + dataFiles = undefined multiPeriodDataFiles = undefined statusDependencies = undefined dataDependenciesModule = undefined - tradingOutputModule.finalize() - tradingOutputModule = undefined fileStorage = undefined processConfig = undefined thisObject = undefined @@ -76,6 +83,17 @@ getContextVariables() + if (bot.FIRST_EXECUTION === true && bot.RESUME === false) { + /* + Here is where the Trading Engine and Trading Systems received are moved to the simulation state. + */ + bot.simulationState.tradingEngine = bot.TRADING_ENGINE + bot.simulationState.tradingSystem = bot.TRADING_SYSTEM + } + + /* We set up the Trading Engine Module. */ + tradingEngineModule.initialize() + /* Initializing the Trading Process Date */ if (bot.FIRST_EXECUTION === true && bot.RESUME === false) { /* @@ -117,7 +135,6 @@ } else { /* We are processing Daily Files */ do { - if (checkStopHeadOfTheMarket() === false) {break} if (checkStopTaskGracefully() === false) {break} if (checkStopProcessing() === false) {break} @@ -135,6 +152,8 @@ await generateOutput(chart) await writeProcessFiles() + + if (checkStopHeadOfTheMarket() === false) {break} } while (true) } @@ -434,8 +453,10 @@ again. */ if (bot.simulationState.tradingEngine.current.episode.headOfTheMarket.value === true) { + console.log('YES at the head of the market') return false } + console.log('NOT at the head of the market') } function checkStopTaskGracefully() { diff --git a/LowFrequencyTrading/TradingRecords.js b/LowFrequencyTrading/TradingRecords.js index d5f4fdae3..fd019fbcf 100644 --- a/LowFrequencyTrading/TradingRecords.js +++ b/LowFrequencyTrading/TradingRecords.js @@ -104,7 +104,6 @@ exports.newTradingRecords = function newTradingRecords(bot, logger) { brach of the trading engine data structure it belongs. */ record.push(index) - record.push('"' + tradingEngine.current.episode.cycle.value + '"') persistIndividualRecord(record, product, outputDatasetArray) } } diff --git a/LowFrequencyTrading/TradingSimulation.js b/LowFrequencyTrading/TradingSimulation.js index 023008b31..f52b1a617 100644 --- a/LowFrequencyTrading/TradingSimulation.js +++ b/LowFrequencyTrading/TradingSimulation.js @@ -1,4 +1,4 @@ -exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILITIES) { +exports.newTradingSimulation = function newTradingSimulation(bot, logger, tradingEngineModule, UTILITIES) { /* This Module represents the trading simulacion. Escentially a loop through a set of candles and the execution at each loop cycle of the Trading System Protocol. @@ -23,7 +23,7 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT chart, outputDatasetsMap, writeFiles, - ) { + ) { try { let tradingSystem = bot.simulationState.tradingSystem @@ -34,10 +34,6 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT if (FULL_LOG === true) { logger.write(MODULE_NAME, '[INFO] runSimulation -> finalDatetime = ' + sessionParameters.timeRange.config.finalDatetime) } /* These are the Modules we will need to run the Simulation */ - const TRADING_ENGINE_MODULE = require('./TradingEngine.js') - let tradingEngineModule = TRADING_ENGINE_MODULE.newTradingEngine(bot, logger) - tradingEngineModule.initialize() - const TRADING_RECORDS_MODULE = require('./TradingRecords.js') let tradingRecordsModule = TRADING_RECORDS_MODULE.newTradingRecords(bot, logger) tradingRecordsModule.initialize(outputDatasetsMap) @@ -58,12 +54,11 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT let heartBeatDate let previousHeartBeatDate let firstLoopExecution = true - /* Estimation of the Initial Candle to Process in this Run. */ - let initialCandle - if (bot.FIRST_EXECUTION === true && bot.RESUME === false) { + let initialCandle + if (bot.FIRST_EXECUTION === true && bot.RESUME === false) { /* Estimate Initial Candle based on the timeRage configured for the session. */ let firstEnd = candles[0].end let targetEnd = sessionParameters.timeRange.config.initialDatetime @@ -77,40 +72,38 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT This will happen when the sessionParameters.timeRange.config.initialDatetime is beyond the last candle available, meaning that the dataSet needs to be updated with more up-to-date data. */ - bot.SESSION.stop('Data is not up-to-date enough. Please start the Masters Data Mining Operation.') - return + bot.SESSION.stop('Data is not up-to-date enough. Please start the Masters Data Mining Operation.') + return } } else { - { - /* - In this case we already have last candle index. Now we need to continue - at the next candle, so we add 1. This works well for Market Files, - but for Daily Files we need to consider that we we turn into a new date - a we get a new file with candles statting at index 0. - */ - if (bot.processingDailyFiles === true) { - /* - We are going to test if adding 1 gets us a valid candle, it means that - the candle dataset did not change since the lasst execution, otherwise - it means it changed and we need to start again from 0. - */ - let candle = candles[tradingEngine.current.episode.candle.index.value + 1] - if (candle === undefined) { - initialCandle = 0 - } else { - initialCandle = tradingEngine.current.episode.candle.index.value + 1 - } - } else { - initialCandle = tradingEngine.current.episode.candle.index.value + 1 - } - } + + /* + In this case we already have at the last candle index the next candle to be + processed. We will just continue with this candle. + */ + initialCandle = tradingEngine.current.episode.candle.index.value } /* Main Simulation Loop */ - console.log('initialCandle = ' + initialCandle) - let breakLoop = false - for( let i = initialCandle; i < candles.length; i++) { + console.log('initialCandle = ' + initialCandle) + /* We are going to use this to exit the loop if needed. */ + let breakLoop = false + + /* + We will assume that we are at the head of the market here. We do this + because the loop could be empty and no validation is going to run. If the + loop is not empty, then the lascCandle() check will override this value + depending on if we really are at the head of the market or not. + */ + tradingEngine.current.episode.headOfTheMarket.value = true + /* + This is the main simulation loop. It will go through the initial candle + until one less than the last candle available. We will never process the last + candle available since it is not considered a closed candle, but a candle + that still can change. So effectively will be processing all closed candles. + */ + for (let i = initialCandle; i < candles.length - 1; i++) { tradingEngine.current.episode.candle.index.value = i console.log('index = ' + tradingEngine.current.episode.candle.index.value) @@ -121,7 +114,7 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT if (FULL_LOG === true) { logger.write(MODULE_NAME, '[INFO] runSimulation -> loop -> Candle End @ ' + (new Date(candle.end)).toLocaleString()) } tradingEngineModule.setCurrentCandle(candle) // We move the current candle we are standing at, to the trading engine data structure to make it available to anyone, including conditions and formulas. - + /* We emit a heart beat so that the UI can now where we are at the overal process.*/ heartBeat() @@ -157,7 +150,12 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT the second cycle without spending real time at the order book. */ tradingEngineModule.setCurrentCycle('First') - await runCycle() + await runCycle() + + /* Add new records to the process output */ + tradingRecordsModule.appendRecords() + + if (breakLoop === true) { break } /* Run the second cycle of the Trading System. During this second run some new orders might be created at slots freed up during the first @@ -167,11 +165,22 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT during the second cycle. */ tradingEngineModule.setCurrentCycle('Second') - await runCycle() + await runCycle() + + /* + We check if we need to stop before appending the records so that the stop + reason is also propery recorded. + */ + checkIfWeNeedToStop() + + /* Add new records to the process output */ + tradingRecordsModule.appendRecords() + + if (breakLoop === true) { break } async function runCycle() { - console.log(new Date(), 'Function runCycle', ' Cycle = ' + tradingEngine.current.episode.cycle.value ) + console.log(new Date(), 'Function runCycle', ' Cycle = ' + tradingEngine.current.episode.cycle.value) /* Reset Data Structures */ tradingSystemModule.reset() @@ -179,18 +188,18 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT tradingEngineModule.reset() await tradingSystemModule.run() + } + function checkIfWeNeedToStop() { /* Checking if we should continue processing this loop or not. */ if (bot.STOP_SESSION === true) { if (FULL_LOG === true) { logger.write(MODULE_NAME, '[INFO] runSimulation -> controlLoop -> We are going to stop here bacause we were requested to stop processing this session.') } - console.log('[INFO] runSimulation -> controlLoop -> We are going to stop here bacause we were requested to stop processing this session.') updateEpisode('Session Stopped') breakLoop = true } if (global.STOP_TASK_GRACEFULLY === true) { if (FULL_LOG === true) { logger.write(MODULE_NAME, '[INFO] runSimulation -> controlLoop -> We are going to stop here bacause we were requested to stop processing this task.') } - console.log('[INFO] runSimulation -> controlLoop -> We are going to stop here bacause we were requested to stop processing this task.') updateEpisode('Task Stopped') breakLoop = true } @@ -200,7 +209,7 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT breakLoop = true } - if (checkLastCandle() === false) { + if (checkNextCandle() === false) { updateEpisode('All Available Candles Processed') breakLoop = true } @@ -209,20 +218,13 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT closeEpisode('Min or Max Balance Reached') breakLoop = true } - - /* Add new records to the process output */ - tradingRecordsModule.appendRecords() } - - if (breakLoop === true) {break} } - tradingEngineModule.finalize() tradingSystemModule.finalize() tradingRecordsModule.finalize() tradingEpisodeModule.finalize() - tradingEngineModule = undefined tradingSystemModule = undefined tradingRecordsModule = undefined tradingEpisodeModule = undefined @@ -357,44 +359,55 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT } catch (err) { logger.write(MODULE_NAME, '[ERROR] runSimulation -> getElement -> datasetName = ' + datasetName) logger.write(MODULE_NAME, '[ERROR] runSimulation -> getElement -> err = ' + err.stack) - throw(global.DEFAULT_FAIL_RESPONSE) + throw (global.DEFAULT_FAIL_RESPONSE) } } - function checkLastCandle() { + function checkNextCandle() { /* Before reaching the last candle of the loop, we need to check that the next candle it is not the last candle. We skip the candle at the head of the market because it has not closed yet. For the special case of Daily Files, this means that the last candle of each day will never be processed. */ - if (tradingEngine.current.episode.candle.index.value + 1 === candles.length - 1) { + if (tradingEngine.current.episode.candle.index.value + 1 === candles.length - 1) { /* When processing daily files, we need a mechanism to turn from one day to the next one. - That mechanism is the one implemented here. If we detect that we are at the last candle of - the day, we will advance current process day. By doing so, during the next execution, the + That mechanism is the one implemented here. If we detect that the next candle is the last candle of + the day, we will advance current process day one day. By doing so, during the next execution, the simulation will receive the candles and indicators files of the next day. */ - if (bot.processingDailyFiles) { - let candlesPerDay = global.ONE_DAY_IN_MILISECONDS / sessionParameters.timeFrame.config.value - if (tradingEngine.current.episode.candle.index.value + 1 === candlesPerDay - 1) { - /* - Here we found that the next candle of the Simulation is the last candle of the day. - It is time to move to the next day so as to receive next, the indicator files from - the next day. At the same time we will reset the index to be ready to real next time - the first candle of the new set we receive. - */ - tradingEngine.current.episode.candle.index.value = 0 - tradingEngine.current.episode.processDate.value = - tradingEngine.current.episode.processDate.value + global.ONE_DAY_IN_MILISECONDS - } + let candlesPerDay = global.ONE_DAY_IN_MILISECONDS / sessionParameters.timeFrame.config.value + if ( + bot.processingDailyFiles && + tradingEngine.current.episode.candle.index.value + 1 === candlesPerDay - 1 + ) { + /* + Here we found that the next candle of the Simulation is the last candle of the day. + It is time to move to the next day so as to receive next, the indicator files from + the next day. At the same time we will reset the index to be pointing to + the first candle of the new set we receive. The first candle of the next day starts + at index 0, so we will position the index now at zero. + */ + tradingEngine.current.episode.candle.index.value = 0 + tradingEngine.current.episode.processDate.value = + tradingEngine.current.episode.processDate.value + global.ONE_DAY_IN_MILISECONDS + return false } - /* We reached the head of the market */ + /* + We reached the head of the market but we are not at the last candle of a day during + Daily Files processing. We will advance to the next candle index anyways because in the + next execution it will likely have more candles at the dataset. And if it does not, + it will just wait there until it does. + */ tradingEngine.current.episode.headOfTheMarket.value = true + tradingEngine.current.episode.candle.index.value++ return false } else { + /* Wd did not reach the head of the market */ tradingEngine.current.episode.headOfTheMarket.value = false + tradingEngine.current.episode.candle.index.value++ return true } } @@ -470,7 +483,7 @@ exports.newTradingSimulation = function newTradingSimulation(bot, logger, UTILIT } } catch (err) { logger.write(MODULE_NAME, '[ERROR] runSimulation -> err = ' + err.stack) - throw(global.DEFAULT_FAIL_RESPONSE) + throw (global.DEFAULT_FAIL_RESPONSE) } } } diff --git a/TradingBot.js b/TradingBot.js index 8b16f955e..bab1df523 100644 --- a/TradingBot.js +++ b/TradingBot.js @@ -3,9 +3,6 @@ const MODULE_NAME = "Trading Bot"; const FULL_LOG = true; - let USER_BOT_MODULE; - let USER_BOT_COMMONS; - const TRADING_PROCESS_MODULE = require(global.ROOT_DIR + '/LowFrequencyTrading/TradingProcess.js'); const FILE_STORAGE = require('./FileStorage.js'); const SESSION = require(global.ROOT_DIR + 'Session'); @@ -33,53 +30,9 @@ processConfig = pProcessConfig; if (bot.definedByUI === true) { /* The code of the bot is defined at the UI. No need to load a file with the code. */ - session.initialize(processConfig, onSessionInitialized) - - function onSessionInitialized(err) { - callBackFunction(err); - } - return + session.initialize(processConfig, callBackFunction) } - let filePath = bot.dataMine + "/" + "bots" + "/" + bot.repo + "/" + pProcessConfig.codeName - filePath += "/User.Bot.js" - - fileStorage.getTextFile(filePath, onBotDownloaded); - - function onBotDownloaded(err, text) { - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> err = " + err.message); - parentLogger.write(MODULE_NAME, "[ERROR] initialize -> onInizialized -> onBotDownloaded -> filePath = " + filePath); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - - USER_BOT_MODULE = {} - USER_BOT_MODULE.newUserBot = eval(text); // Use this for production - - filePath = bot.dataMine + "/" + "bots" + "/" + bot.repo; - filePath += "/Commons.js" - - fileStorage.getTextFile(filePath, onCommonsDownloaded); - - function onCommonsDownloaded(err, text) { - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - parentLogger.write(MODULE_NAME, "[WARN] initialize -> onBotDownloaded -> onCommonsDownloaded -> Commons not found: " + err.code || err.message); - callBackFunction(global.DEFAULT_OK_RESPONSE); - return; - } - - USER_BOT_COMMONS = {} - USER_BOT_COMMONS.newCommons = eval(text); // Use this for production - - session.initialize(onSessionInitialized) - - function onSessionInitialized(err) { - callBackFunction(err); - } - } - } } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); @@ -337,7 +290,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> initializeDataDependencies -> onInizialized -> Execution finished well."); } switch (processConfig.framework.name) { case 'Low-Frequency-Trading-Process': { - processFramework = TRADING_PROCESS_MODULE.newTradingProcess(bot, logger, UTILITIES, USER_BOT_MODULE, USER_BOT_COMMONS); + processFramework = TRADING_PROCESS_MODULE.newTradingProcess(bot, logger, UTILITIES); intitializeProcessFramework(); break; }