From 4d0f01e0b57f528f6db1939d8b8a0dbccff8013e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 1 Oct 2019 08:08:23 +0200 Subject: [PATCH 01/19] renaming caused problems --- FileStorage.js | 56 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/FileStorage.js b/FileStorage.js index 8857b87a4..66d570bb4 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -1,11 +1,11 @@ const path = require('path') -const fs = require('fs') exports.newFileStorage = function newFileStorage(logger) { const MODULE_NAME = 'FileStorage' const MAX_RETRY = 30 + const RETRY_TIME_IN_MILISECONDS = 250 let currentRetryGetTextFile = 0 let currentRetryWriteTextFile = 0 @@ -43,6 +43,7 @@ exports.newFileStorage = function newFileStorage(logger) { logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> fileLocation: ' + fileLocation) /* Here we actually write the file. */ + const fs = require('fs') fs.readFile(fileLocation, onFileRead) function onFileRead(err, text) { @@ -62,13 +63,13 @@ exports.newFileStorage = function newFileStorage(logger) { logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> container = ' + container) logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> filePath = ' + filePath) logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> err = ' + err.stack) - setTimeout(retry, 100) + setTimeout(retry, RETRY_TIME_IN_MILISECONDS) return } if (text.toString() === "") { logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Read and Empty File -> filePath = ' + filePath) - setTimeout(retry, 100) + setTimeout(retry, RETRY_TIME_IN_MILISECONDS) return } @@ -117,16 +118,50 @@ exports.newFileStorage = function newFileStorage(logger) { let directoryPath = fileLocation.substring(0, fileLocation.lastIndexOf('/') + 1); mkDirByPathSync(directoryPath) - /* Here we actually write the file. */ - fs.writeFile(fileLocation, fileContent, onFileWriten) + /* + Here we write the file with a temporary name so as to avoid dirty read from other processes. + Then we delete the original file, if exists, and finally we rename the temporary into the original name. + */ + const fs = require('fs') + fs.writeFile(fileLocation + '.tmp', fileContent, onFileWritenn) - function onFileWriten(err) { + function onFileWritenn(err) { if (err) { - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> file = ' + fileLocation) - logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWrite -> Error writing file -> err = ' + err.stack) - setTimeout(retry, 100) + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> Error writing file -> file = ' + fileLocation) + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> Error writing file -> err = ' + err.stack) + setTimeout(retry, RETRY_TIME_IN_MILISECONDS) } else { - callBackFunction(global.DEFAULT_OK_RESPONSE) + + const fs = require('fs') + fs.unlink(fileLocation, onUnlinked) + + function onUnlinked(err) { + let code = '' + if (err) { + code = err.code + } + if (code !== '' && code !== 'ENOENT') { + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error renaming file -> file = ' + fileLocation) + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> Error renaming file -> err = ' + err.stack) + setTimeout(retry, RETRY_TIME_IN_MILISECONDS) + } else { + + const fs = require('fs') + fs.rename(fileLocation + '.tmp', fileLocation, onRenamed) + + function onRenamed(err) { + if (err) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onRenamed -> Error renaming file -> file = ' + fileLocation + '.tmp') + logger.write(MODULE_NAME, '[WARN] FileStorage -> createTextFile -> onFileWriten -> onUnlinked -> onRenamed -> Error renaming file -> err = ' + err.stack) + setTimeout(retry, RETRY_TIME_IN_MILISECONDS) + } else { + + callBackFunction(global.DEFAULT_OK_RESPONSE) + + } + } + } + } } } @@ -160,6 +195,7 @@ exports.newFileStorage = function newFileStorage(logger) { return targetDir.split(sep).reduce((parentDir, childDir) => { const curDir = path.resolve(baseDir, parentDir, childDir); try { + const fs = require('fs') fs.mkdirSync(curDir); } catch (err) { if (err.code === 'EEXIST') { // curDir already exists! From c060308739084d3c77df9f289808d6f9dd6e06c9 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 1 Oct 2019 12:54:09 +0200 Subject: [PATCH 02/19] Horrible Bug Fixed, not stopping listening to events of status reports produced multiple callbacks --- DataDependencies.js | 1 + DebugLog.js | 5 ++++- IndicatorBotProcessMainLoop.js | 16 ++++++++-------- SensorBotProcessMainLoop.js | 8 ++++---- StatusDependencies.js | 1 + StatusReport.js | 13 ++++--------- TradingEngineProcessMainLoop.js | 14 +++++++------- server.js | 1 + 8 files changed, 30 insertions(+), 29 deletions(-) diff --git a/DataDependencies.js b/DataDependencies.js index 61456835a..0be8aa16e 100644 --- a/DataDependencies.js +++ b/DataDependencies.js @@ -74,6 +74,7 @@ if (addCount === thisObject.config.length) { if (alreadyCalledBack === false) { + alreadyCalledBack = true callBackFunction(global.DEFAULT_OK_RESPONSE); return; } diff --git a/DebugLog.js b/DebugLog.js index 616c1b4dd..ac46321e3 100644 --- a/DebugLog.js +++ b/DebugLog.js @@ -94,8 +94,9 @@ exports.newDebugLog = function newDebugLog() { function onFileCreated(err) { if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + console.log("[ERROR] DebugLog -> persist -> onInizialized -> onFileCreated -> err = "+ err.stack); - + setTimeout(writeLog, 10000); // Lets retry until we make it. return; } @@ -134,7 +135,9 @@ exports.newDebugLog = function newDebugLog() { if (thisObject.bot) { key = thisObject.bot.devTeam + '-' + thisObject.bot.codeName + '-' + thisObject.bot.process } + console.log(" "); console.log(message + ' @ ' + key) + console.log(" "); } accumulatedLog = accumulatedLog + logLine; diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index b0bb3508c..6eda2730c 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -172,7 +172,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeStatusDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -276,7 +276,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeDataDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -350,7 +350,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeUserBot -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -450,7 +450,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> startUserBot -> onFinished -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -475,8 +475,8 @@ return; } case "Dependency not ready.": { - logger.write(MODULE_NAME, "[WARN] run -> loop -> startUserBot -> onFinished -> Dependency not ready. This Loop will go to sleep."); - nextWaitTime = 'Sleep'; + logger.write(MODULE_NAME, "[WARN] run -> loop -> startUserBot -> onFinished -> Dependency not ready. Will Retry Later."); + nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; } @@ -579,7 +579,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> intitializeProcessFramework -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -681,7 +681,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> startProcessFramework -> onFinished -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index a839a0d3e..97b205f8d 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -172,7 +172,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeStatusDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -246,7 +246,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeExchangeAPI -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeExchangeAPI -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -320,7 +320,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized > Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeUserBot -> onInizialized > Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -428,7 +428,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished > Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> startUserBot -> onFinished > Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; diff --git a/StatusDependencies.js b/StatusDependencies.js index ae1230e2c..c2bfbba48 100644 --- a/StatusDependencies.js +++ b/StatusDependencies.js @@ -120,6 +120,7 @@ if (loadCount === pStatusDependenciesConfig.length) { if (alreadyCalledBack === false) { + alreadyCalledBack = true callBackFunction(global.DEFAULT_OK_RESPONSE); return; } diff --git a/StatusReport.js b/StatusReport.js index 2725cab2d..aa0d89f87 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -67,26 +67,21 @@ let subscriptionIdStatusReport - subscriptionIdStatusReport = global.SYSTEM_EVENT_HANDLER.listenToEvent(key, 'Status Report Updated', undefined, callerId, responseCallBack, eventsCallBack) + global.SYSTEM_EVENT_HANDLER.listenToEvent(key, 'Status Report Updated', undefined, callerId, responseCallBack, eventsCallBack) function responseCallBack(message) { if (message.result !== global.DEFAULT_OK_RESPONSE.result) { callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } else { + subscriptionIdStatusReport = message.eventSubscriptionId } } function eventsCallBack() { /* We continue the normal flow after we learn the dependent process has updated its status report. */ - stopListening() + global.SYSTEM_EVENT_HANDLER.stopListening(key, 'Status Report Updated', subscriptionIdStatusReport) callBackFunction(global.DEFAULT_OK_RESPONSE); } - - function stopListening() { - if (subscriptionIdStatusReport) { - stopListening(key, 'Status Report Updated', subscriptionIdStatusReport) - } - } - } else { /* In this case, the Status Report does not depends on a process which needs to wait for. */ callBackFunction(global.DEFAULT_OK_RESPONSE); diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index e971714e1..bd7f4cdaf 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -328,7 +328,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeStatusDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -403,7 +403,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeDataDependencies -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -476,7 +476,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeContext -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeContext -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -550,7 +550,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeExchangeAPI -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeExchangeAPI -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -657,7 +657,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeAssistant -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> initializeAssistant -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -729,7 +729,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> intitializeProcessFramework -> onInizialized -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; @@ -831,7 +831,7 @@ return; } case global.DEFAULT_RETRY_RESPONSE.result: { // Something bad happened, but if we retry in a while it might go through the next time. - logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Retry Later. Requesting Execution Retry."); + logger.write(MODULE_NAME, "[WARN] run -> loop -> startProcessFramework -> onFinished -> Retry Later. Requesting Execution Retry."); nextWaitTime = 'Retry'; loopControl(nextWaitTime); return; diff --git a/server.js b/server.js index 3fcdece97..23ba18dba 100644 --- a/server.js +++ b/server.js @@ -74,6 +74,7 @@ else { // I use this section to debug in standalone mode. let argument = ' {"type":"Task","name":"Runs Backtests, Fordwardtests & Live Trades ","bot":{"type":"Trading Engine","processes":[{"type":"Process","name":"Multi Period","code":{"team":"AAMasters","bot":"AAJason","process":"Multi-Period","repo":"AAJason-Trading-Engine-Bot"},"id":"4748c8c4-4d19-4076-96b2-e9c06524fbb3"}]},"id":"561bac18-fc78-464a-90c5-79fd821fc633"}' // charly argument = '{"type":"Task","name":"Brings Trades Records from the Exchange","bot":{"type":"Sensor","name":"Charly","processes":[{"type":"Process","name":"Live Trades","code":{"team":"AAMasters","bot":"AACharly","process":"Live-Trades"},"id":"5846ebc7-1979-4a80-9cc7-94bb4b8659dc"},{"type":"Process","name":"Hole Fixing","code":{"team":"AAMasters","bot":"AACharly","process":"Hole-Fixing"},"id":"31fc4f05-75d4-419a-9fb0-73e25e856f15"}]},"id":"5bfef4dc-54c1-44db-ace6-1ab27a86746e"}' // olivia argument = '{"type":"Task","name":"Generates 1 min to 24 hs Candles & Volumes","bot":{"type":"Indicator","name":"Olivia","processes":[{"type":"Process","name":"Daily","code":{"team":"AAMasters","bot":"AAOlivia","process":"Multi-Period-Daily"},"id":"68cc8e1b-e94a-477e-82d0-15ecc9f1b9e2"},{"type":"Process","name":"Market","code":{"team":"AAMasters","bot":"AAOlivia","process":"Multi-Period-Market"},"id":"e1ac5e3d-d491-4e90-baf4-d4e5b71a8b1a"}]},"id":"efe36e05-75ea-41b5-8d92-c6b27635c834"}' + argument = ' {"type":"Task","name":"Converts Trades into 1 min Candles & Volumes","bot":{"type":"Indicator","name":"Bruce","processes":[{"type":"Process","name":"New Process","code":{"team":"AAMasters","bot":"AABruce","process":"Single-Period-Daily"},"id":"e184744f-5de0-41c1-ad4c-36960f4ced90"}]},"id":"42758590-a7bd-4712-a5c9-80a3a820c5b3"}' try { global.TASK_NODE = JSON.parse(argument) } catch (err) { From c9037dff2a5e79c4eca75efb761fddc8f882d49e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 1 Oct 2019 14:27:41 +0200 Subject: [PATCH 03/19] Bug Fixed, race condition when starting processes --- Root.js | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Root.js b/Root.js index 49e504b88..6dd38245d 100644 --- a/Root.js +++ b/Root.js @@ -37,6 +37,7 @@ const ROOT_DIR = './'; const MODULE_NAME = "Root"; + const WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START = 10000 // This avoid a race condition that could happen if one process finished before all the other even started. let thisObject = { initialize: initialize, @@ -456,7 +457,7 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> whenStartFinishes -> Bot execution was aborted."); logger.persist(); } - exitProcessInstance() + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) } } else { @@ -465,13 +466,13 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> onInitializeReady -> err = " + err.message); logger.persist(); - exitProcessInstance() + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) } } } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runSensorBot -> err = " + err.stack); - exitProcessInstance() + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) } } @@ -528,7 +529,7 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); logger.persist(); } - exitProcessInstance() + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) } } else { @@ -536,13 +537,13 @@ logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> Failed to initialize the bot. "); console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> onInitializeReady -> err = " + err.message); logger.persist(); - exitProcessInstance() + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) } } } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runIndicatorBot -> err = " + err.stack); - exitProcessInstance() + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) } } @@ -594,7 +595,7 @@ console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> whenStartFinishes -> Bot execution finished with errors. Please check the logs."); logger.persist(); } - exitProcessInstance() + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) } } else { @@ -602,19 +603,19 @@ logger.write(MODULE_NAME, "[ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> Failed to initialize the bot. "); console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> onInitializeReady -> err = " + err.message); logger.persist(); - exitProcessInstance() + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) } } } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> runTradingEngine -> err = " + err.stack); - exitProcessInstance() + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) } } } catch (err) { console.log(logDisplace + "Root : [ERROR] start -> findProcess -> err = " + err.stack); - exitProcessInstance() + setTimeout(exitProcessInstance, WAIT_TIME_FOR_ALL_PROCESS_INSTANCES_TO_START) } } From 0df0857ec98a113edaf333aa0a9783fdfd21864e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 1 Oct 2019 22:00:21 +0200 Subject: [PATCH 04/19] Multiple bug fixes and improvements --- Context.js | 6 ++--- DebugLog.js | 4 +--- FileStorage.js | 42 +++++++++++++++++++++++++-------- StatusDependencies.js | 22 +++++++++++++---- StatusReport.js | 7 ++++++ TradingEngineProcessMainLoop.js | 16 ++++++++----- server.js | 4 ++-- 7 files changed, 72 insertions(+), 29 deletions(-) diff --git a/Context.js b/Context.js index 6b0c32b0e..0392beed2 100644 --- a/Context.js +++ b/Context.js @@ -304,10 +304,10 @@ thisObject.executionHistory = []; - if (process.env.INITIAL_BALANCE_ASSET_A === undefined || process.env.INITIAL_BALANCE_ASSET_B === undefined) { throw new Error("Environment Variables INITIAL_BALANCE_A and INITIAL_BALANCE_B cannot be undefined.") } + if (global.INITIAL_BALANCE_ASSET_A === undefined || global.INITIAL_BALANCE_ASSET_B === undefined) { throw new Error("Global Variables INITIAL_BALANCE_A and INITIAL_BALANCE_B cannot be undefined.") } - const INITIAL_BALANCE_A = Number(process.env.INITIAL_BALANCE_ASSET_A); - const INITIAL_BALANCE_B = Number(process.env.INITIAL_BALANCE_ASSET_B); + const INITIAL_BALANCE_A = Number(global.INITIAL_BALANCE_ASSET_A); + const INITIAL_BALANCE_B = Number(global.INITIAL_BALANCE_ASSET_B); thisObject.executionContext = { diff --git a/DebugLog.js b/DebugLog.js index ac46321e3..07da825e5 100644 --- a/DebugLog.js +++ b/DebugLog.js @@ -135,9 +135,7 @@ exports.newDebugLog = function newDebugLog() { if (thisObject.bot) { key = thisObject.bot.devTeam + '-' + thisObject.bot.codeName + '-' + thisObject.bot.process } - console.log(" "); - console.log(message + ' @ ' + key) - console.log(" "); + console.log('*********** ' + message + ' @ ' + key) } accumulatedLog = accumulatedLog + logLine; diff --git a/FileStorage.js b/FileStorage.js index 66d570bb4..5f5534c3d 100644 --- a/FileStorage.js +++ b/FileStorage.js @@ -24,10 +24,12 @@ exports.newFileStorage = function newFileStorage(logger) { return thisObject - function getTextFile(container, filePath, callBackFunction) { + function getTextFile(container, filePath, callBackFunction, noRetry) { logger.write(MODULE_NAME, '[INFO] FileStorage -> getTextFile -> Entering Function.') + let fileDoesNotExist = false + container = container.toLowerCase() /* Choose path for either bots or data */ @@ -50,13 +52,21 @@ exports.newFileStorage = function newFileStorage(logger) { if (err) { - if (err.code === 'ENOENT') { - let customResponse = { - result: global.CUSTOM_FAIL_RESPONSE.result, - message: 'File does not exist.' + if (err.code === 'ENOENT') { // since files are deleted before being replaced, it can happen that it does not exist and after a retry it does. + if (noRetry === true) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> File does not exist. Not Retrying. ') + let customResponse = { + result: global.CUSTOM_FAIL_RESPONSE.result, + message: 'File does not exist.' + } + callBackFunction(customResponse) + return + } else { + fileDoesNotExist = true + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> File does not exist. Retrying. ') + setTimeout(retry, RETRY_TIME_IN_MILISECONDS) + return } - callBackFunction(customResponse) - return } logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> onFileRead -> Error reading file -> err = ' + err.stack) @@ -89,9 +99,21 @@ exports.newFileStorage = function newFileStorage(logger) { getTextFile(container, filePath, callBackFunction) } else { currentRetryGetTextFile = 0 - logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> Max retries reached reading a file. Giving up.') - logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> file = ' + fileLocation) - callBackFunction(global.DEFAULT_FAIL_RESPONSE) + + if (fileDoesNotExist === true) { + logger.write(MODULE_NAME, '[WARN] FileStorage -> getTextFile -> retry -> Max retries reached reading a file. File Not Found.') + let customResponse = { + result: global.CUSTOM_FAIL_RESPONSE.result, + message: 'File does not exist.' + } + callBackFunction(customResponse) + return + } else { + logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> Max retries reached reading a file. Giving up.') + logger.write(MODULE_NAME, '[ERROR] FileStorage -> getTextFile -> retry -> file = ' + fileLocation) + callBackFunction(global.DEFAULT_FAIL_RESPONSE) + return + } } } } diff --git a/StatusDependencies.js b/StatusDependencies.js index c2bfbba48..a0eaa4374 100644 --- a/StatusDependencies.js +++ b/StatusDependencies.js @@ -34,25 +34,33 @@ let statusReportModule = STATUS_REPORT.newStatusReport(BOT, logger, UTILITIES); + logger.write(MODULE_NAME, "[INFO] initialize -> onInitilized -> Initializing Status Report # " + (i + 1)); statusReportModule.initialize(pStatusDependenciesConfig[i], pMonth, pYear, onInitilized); function onInitilized(err) { - if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] initialize -> onInitilized -> err = "+ err.stack); + logger.write(MODULE_NAME, "[INFO] initialize -> onInitilized -> Initialized Status Report # " + (i + 1)); - alreadyCalledBack = true; - callBackFunction(err); + if (err.result !== global.DEFAULT_OK_RESPONSE.result) { + if (alreadyCalledBack === false) { + logger.write(MODULE_NAME, "[ERROR] initialize -> onInitilized -> err = " + err.stack); + logger.write(MODULE_NAME, "[ERROR] initialize -> onInitilized -> err.message = " + err.message); + alreadyCalledBack = true; + callBackFunction(err); + } else { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[WARN] initialize -> Can not call back because I already did."); } + } return; } + logger.write(MODULE_NAME, "[INFO] initialize -> onInitilized -> Loading Status Report # " + (i + 1)); statusReportModule.load(onLoad); } function onLoad(err) { try { - + logger.write(MODULE_NAME, "[INFO] initialize -> onLoad -> Loaded Status Report # " + (i + 1)); statusReportModule.status = err.message; switch (err.message) { @@ -102,8 +110,10 @@ function addReport() { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] initialize -> addReport -> Entering function."); } + logger.write(MODULE_NAME, "[INFO] initialize -> addReport -> Adding Status Report # " + (i + 1)); loadCount++; + logger.write(MODULE_NAME, "[INFO] initialize -> addReport -> Total Added = " + loadCount); let key; @@ -123,6 +133,8 @@ alreadyCalledBack = true callBackFunction(global.DEFAULT_OK_RESPONSE); return; + } else { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[WARN] initialize -> addReport -> Can not call back because I already did."); } } } } diff --git a/StatusReport.js b/StatusReport.js index aa0d89f87..1b6eb7446 100644 --- a/StatusReport.js +++ b/StatusReport.js @@ -41,6 +41,7 @@ function initialize(pOwner, pMonth, pYear, callBackFunction) { try { + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } logger.fileName = MODULE_NAME + "." + pOwner.bot + "." + pOwner.process; @@ -56,6 +57,9 @@ } if (owner.waitUntilNextUpdate === true) { + + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> waitUntilNextUpdate -> Entering Code Block."); } + /* This forces this process to wait until the process that this one depends on, updates its status report. */ let extraCallerId = '' @@ -71,6 +75,7 @@ function responseCallBack(message) { if (message.result !== global.DEFAULT_OK_RESPONSE.result) { + logger.write(MODULE_NAME, "[ERROR] initialize -> responseCallBack -> message = " + JSON.stringify(message)) callBackFunction(global.DEFAULT_FAIL_RESPONSE); } else { subscriptionIdStatusReport = message.eventSubscriptionId @@ -78,6 +83,8 @@ } function eventsCallBack() { + if (global.LOG_CONTROL[MODULE_NAME].logInfo === true) { logger.write(MODULE_NAME, "[INFO] initialize -> eventsCallBack -> Entering function."); } + /* We continue the normal flow after we learn the dependent process has updated its status report. */ global.SYSTEM_EVENT_HANDLER.stopListening(key, 'Status Report Updated', subscriptionIdStatusReport) callBackFunction(global.DEFAULT_OK_RESPONSE); diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index bd7f4cdaf..ae56003d5 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -200,6 +200,10 @@ if (tradingSystem.parameters.baseAsset !== undefined) { let code try { + /* Default Value */ + global.INITIAL_BALANCE_ASSET_B = 0.001; + global.INITIAL_BALANCE_ASSET_A = 0 + code = JSON.parse(tradingSystem.parameters.baseAsset.code); if (code.name !== undefined) { @@ -212,20 +216,20 @@ if (baseAsset === 'BTC') { // NOTE: POLONIEX, the only exchange working so far, has Asset A and B inverted. We need to fix this. if (code.initialBalance !== undefined) { - process.env.INITIAL_BALANCE_ASSET_B = code.initialBalance; - process.env.INITIAL_BALANCE_ASSET_A = 0 + global.INITIAL_BALANCE_ASSET_B = code.initialBalance; + global.INITIAL_BALANCE_ASSET_A = 0 } } else { if (code.initialBalance !== undefined) { - process.env.INITIAL_BALANCE_ASSET_A = code.initialBalance; - process.env.INITIAL_BALANCE_ASSET_B = 0 + global.INITIAL_BALANCE_ASSET_A = code.initialBalance; + global.INITIAL_BALANCE_ASSET_B = 0 } } } catch (err) { global.DEFINITION.tradingSystem.parameters.baseAsset.error = err.message - process.env.INITIAL_BALANCE_ASSET_A = 0 // default - process.env.INITIAL_BALANCE_ASSET_B = 0.001 // default + global.INITIAL_BALANCE_ASSET_A = 0 // default + global.INITIAL_BALANCE_ASSET_B = 0.001 // default } } diff --git a/server.js b/server.js index 23ba18dba..0b7f12a40 100644 --- a/server.js +++ b/server.js @@ -72,9 +72,9 @@ if (global.TASK_NODE !== undefined) { } else { // I use this section to debug in standalone mode. let argument = ' {"type":"Task","name":"Runs Backtests, Fordwardtests & Live Trades ","bot":{"type":"Trading Engine","processes":[{"type":"Process","name":"Multi Period","code":{"team":"AAMasters","bot":"AAJason","process":"Multi-Period","repo":"AAJason-Trading-Engine-Bot"},"id":"4748c8c4-4d19-4076-96b2-e9c06524fbb3"}]},"id":"561bac18-fc78-464a-90c5-79fd821fc633"}' - // charly argument = '{"type":"Task","name":"Brings Trades Records from the Exchange","bot":{"type":"Sensor","name":"Charly","processes":[{"type":"Process","name":"Live Trades","code":{"team":"AAMasters","bot":"AACharly","process":"Live-Trades"},"id":"5846ebc7-1979-4a80-9cc7-94bb4b8659dc"},{"type":"Process","name":"Hole Fixing","code":{"team":"AAMasters","bot":"AACharly","process":"Hole-Fixing"},"id":"31fc4f05-75d4-419a-9fb0-73e25e856f15"}]},"id":"5bfef4dc-54c1-44db-ace6-1ab27a86746e"}' + /* charly */ argument = '{"type":"Task","name":"Brings Trades Records from the Exchange","bot":{"type":"Sensor","name":"Charly","processes":[{"type":"Process","name":"Live Trades","code":{"team":"AAMasters","bot":"AACharly","process":"Live-Trades"},"id":"5846ebc7-1979-4a80-9cc7-94bb4b8659dc"},{"type":"Process","name":"Hole Fixing","code":{"team":"AAMasters","bot":"AACharly","process":"Hole-Fixing"},"id":"31fc4f05-75d4-419a-9fb0-73e25e856f15"}]},"id":"5bfef4dc-54c1-44db-ace6-1ab27a86746e"}' // olivia argument = '{"type":"Task","name":"Generates 1 min to 24 hs Candles & Volumes","bot":{"type":"Indicator","name":"Olivia","processes":[{"type":"Process","name":"Daily","code":{"team":"AAMasters","bot":"AAOlivia","process":"Multi-Period-Daily"},"id":"68cc8e1b-e94a-477e-82d0-15ecc9f1b9e2"},{"type":"Process","name":"Market","code":{"team":"AAMasters","bot":"AAOlivia","process":"Multi-Period-Market"},"id":"e1ac5e3d-d491-4e90-baf4-d4e5b71a8b1a"}]},"id":"efe36e05-75ea-41b5-8d92-c6b27635c834"}' - argument = ' {"type":"Task","name":"Converts Trades into 1 min Candles & Volumes","bot":{"type":"Indicator","name":"Bruce","processes":[{"type":"Process","name":"New Process","code":{"team":"AAMasters","bot":"AABruce","process":"Single-Period-Daily"},"id":"e184744f-5de0-41c1-ad4c-36960f4ced90"}]},"id":"42758590-a7bd-4712-a5c9-80a3a820c5b3"}' + // bruce argument = ' {"type":"Task","name":"Converts Trades into 1 min Candles & Volumes","bot":{"type":"Indicator","name":"Bruce","processes":[{"type":"Process","name":"New Process","code":{"team":"AAMasters","bot":"AABruce","process":"Single-Period-Daily"},"id":"e184744f-5de0-41c1-ad4c-36960f4ced90"}]},"id":"42758590-a7bd-4712-a5c9-80a3a820c5b3"}' try { global.TASK_NODE = JSON.parse(argument) } catch (err) { From 62b5d5c9ea1a05922e211c9b84ff4395e6ce9043 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 2 Oct 2019 10:34:45 +0200 Subject: [PATCH 05/19] Time Period taken from Time Period Object --- TradingEngineProcessMainLoop.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index ae56003d5..83798826a 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -107,7 +107,7 @@ processConfig.framework.startDate.resumeExecution = false; global.STOP_PROCESSING = false bot.hasTheBotJustStarted = true - setInitialBalance() + setValuesToUse() const ONE_YEAR_IN_MILISECONDS = 365 * 24 * 60 * 60 * 1000 /* If we can not get for any reason the initial date of the backtest, we will start at present time. */ @@ -182,21 +182,21 @@ bot.multiPeriodDailyProcessDatetime = processConfig.framework.startDate.fixedDate global.STOP_PROCESSING = false bot.hasTheBotJustStarted = true - setInitialBalance() + setValuesToUse() } function stopRequested() { global.STOP_PROCESSING = true } - function setInitialBalance() { + function setValuesToUse() { if (global.DEFINITION !== undefined) { - - /* Get the initial balance from the global.DEFINITION */ let tradingSystem = global.DEFINITION.tradingSystem if (tradingSystem) { if (tradingSystem.parameters !== undefined) { + + /* Decide which Initial Balance we are going to use. */ if (tradingSystem.parameters.baseAsset !== undefined) { let code try { @@ -233,6 +233,12 @@ } } + + /* Decide which Time Period we are going to use. If a TimePeriod is explicity defined we replace the one which was sent on simulation params.*/ + + if (tradingSystem.parameters.timePeriod !== undefined) { + global.DEFINITION.simulationParams.timePeriod = tradingSystem.parameters.timePeriod.code + } } } } From 3ae0c647f42c8ef8c2f52053bbb7daf6bef5854a Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 3 Oct 2019 16:53:59 +0200 Subject: [PATCH 06/19] Processing Events to Run Sessions --- Context.js | 6 +-- MultiPeriod.js | 4 +- Root.js | 1 + TradingEngineProcessMainLoop.js | 88 ++++++++++++++++++++++++--------- server.js | 7 ++- 5 files changed, 75 insertions(+), 31 deletions(-) diff --git a/Context.js b/Context.js index 0392beed2..6bb50a361 100644 --- a/Context.js +++ b/Context.js @@ -304,10 +304,10 @@ thisObject.executionHistory = []; - if (global.INITIAL_BALANCE_ASSET_A === undefined || global.INITIAL_BALANCE_ASSET_B === undefined) { throw new Error("Global Variables INITIAL_BALANCE_A and INITIAL_BALANCE_B cannot be undefined.") } + if (bot.VALUES_TO_USE.initialBalanceAssetA === undefined || bot.VALUES_TO_USE.initialBalanceAssetB === undefined) { throw new Error("Global Variables INITIAL_BALANCE_A and INITIAL_BALANCE_B cannot be undefined.") } - const INITIAL_BALANCE_A = Number(global.INITIAL_BALANCE_ASSET_A); - const INITIAL_BALANCE_B = Number(global.INITIAL_BALANCE_ASSET_B); + const INITIAL_BALANCE_A = Number(bot.VALUES_TO_USE.initialBalanceAssetA); + const INITIAL_BALANCE_B = Number(bot.VALUES_TO_USE.initialBalanceAssetB); thisObject.executionContext = { diff --git a/MultiPeriod.js b/MultiPeriod.js index e6e5eb155..a31e3870e 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -284,7 +284,7 @@ let dependencyIndex = 0; dataFiles = []; - if (global.DEFINITION.simulationParams.timePeriod === outputPeriodLabel) { + if (bot.VALUES_TO_USE.timePeriod === outputPeriodLabel) { currentTimePeriod = global.marketFilesPeriods[n][0]; currentOutputPeriodLabel = global.marketFilesPeriods[n][1]; } @@ -582,7 +582,7 @@ } } - if (global.DEFINITION.simulationParams.timePeriod === outputPeriodLabel) { + if (bot.VALUES_TO_USE.timePeriod === outputPeriodLabel) { currentTimePeriod = global.dailyFilePeriods[n][0]; currentOutputPeriodLabel = global.dailyFilePeriods[n][1]; } diff --git a/Root.js b/Root.js index 6dd38245d..cc459e242 100644 --- a/Root.js +++ b/Root.js @@ -174,6 +174,7 @@ botConfig.process = global.TASK_NODE.bot.processes[processIndex].code.process; botConfig.debug = {}; botConfig.processIndex = processIndex + botConfig.processNode = global.TASK_NODE.bot.processes[processIndex] /* Loop Counter */ diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 83798826a..0bad0c65f 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -89,14 +89,41 @@ if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] run -> Entering function."); } - bot.enableCheckLoopHealth = true; + /* Check if there is a session */ + if (bot.processNode.session === undefined) { + parentLogger.write(MODULE_NAME, "[ERROR] run -> Cannot run without a Session."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + /* Some initial values*/ + bot.enableCheckLoopHealth = true; let fixedTimeLoopIntervalHandle; global.STOP_PROCESSING = true; - global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Backstesting Started', undefined, undefined, undefined, startBackTesting) - global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Live-Trading Started', undefined, undefined, undefined, startLiveTrading) - global.SYSTEM_EVENT_HANDLER.listenToEvent('Cockpit-Restart-Button', 'Stop Requested', undefined, undefined, undefined, stopRequested) + /* Listen to event to start or stop the session. */ + + let key = 'Session' + '-' + bot.processNode.session.id + '-' + 'Events' + + global.SYSTEM_EVENT_HANDLER.listenToEvent(key, 'Run Session', undefined, undefined, undefined, runSession) + global.SYSTEM_EVENT_HANDLER.listenToEvent(key, 'Stop Session', undefined, undefined, undefined, stopSession) + + function runSession(message) { + switch (bot.processNode.session.type) { + case 'Backtesting Session': { + startBackTesting(message) + break + } + case 'Live Trading Session': { + startLiveTrading(message) + break + } + } + } + + function stopSession(message) { + global.STOP_PROCESSING = true + } function startBackTesting(message) { @@ -107,7 +134,7 @@ processConfig.framework.startDate.resumeExecution = false; global.STOP_PROCESSING = false bot.hasTheBotJustStarted = true - setValuesToUse() + setValuesToUse(message) const ONE_YEAR_IN_MILISECONDS = 365 * 24 * 60 * 60 * 1000 /* If we can not get for any reason the initial date of the backtest, we will start at present time. */ @@ -182,14 +209,18 @@ bot.multiPeriodDailyProcessDatetime = processConfig.framework.startDate.fixedDate global.STOP_PROCESSING = false bot.hasTheBotJustStarted = true - setValuesToUse() + setValuesToUse(message) } - function stopRequested() { - global.STOP_PROCESSING = true - } + function setValuesToUse(message) { + /* + Base on the information received we will determined which values ultimatelly are going to be used, + and once we do, they will become constants across the multiple loops executions. + */ + + bot.SESSION = JSON.parse(message.event.session) + bot.VALUES_TO_USE = {} - function setValuesToUse() { if (global.DEFINITION !== undefined) { let tradingSystem = global.DEFINITION.tradingSystem @@ -201,8 +232,8 @@ let code try { /* Default Value */ - global.INITIAL_BALANCE_ASSET_B = 0.001; - global.INITIAL_BALANCE_ASSET_A = 0 + bot.VALUES_TO_USE.initialBalanceAssetB = 0.001; + bot.VALUES_TO_USE.initialBalanceAssetA = 0 code = JSON.parse(tradingSystem.parameters.baseAsset.code); @@ -216,28 +247,41 @@ if (baseAsset === 'BTC') { // NOTE: POLONIEX, the only exchange working so far, has Asset A and B inverted. We need to fix this. if (code.initialBalance !== undefined) { - global.INITIAL_BALANCE_ASSET_B = code.initialBalance; - global.INITIAL_BALANCE_ASSET_A = 0 + bot.VALUES_TO_USE.initialBalanceAssetB = code.initialBalance; + bot.VALUES_TO_USE.initialBalanceAssetA = 0 } } else { if (code.initialBalance !== undefined) { - global.INITIAL_BALANCE_ASSET_A = code.initialBalance; - global.INITIAL_BALANCE_ASSET_B = 0 + bot.VALUES_TO_USE.initialBalanceAssetA = code.initialBalance; + bot.VALUES_TO_USE.initialBalanceAssetB = 0 } } } catch (err) { global.DEFINITION.tradingSystem.parameters.baseAsset.error = err.message - global.INITIAL_BALANCE_ASSET_A = 0 // default - global.INITIAL_BALANCE_ASSET_B = 0.001 // default + bot.VALUES_TO_USE.initialBalanceAssetA = 0 // default + bot.VALUES_TO_USE.initialBalanceAssetB = 0.001 // default } } - /* Decide which Time Period we are going to use. If a TimePeriod is explicity defined we replace the one which was sent on simulation params.*/ + /* Decide which Time Period we are going to use.*/ + /* This is the last fallback */ + bot.VALUES_TO_USE.timePeriod = global.DEFINITION.simulationParams.timePeriod + + /* Secondary option, if exists. */ if (tradingSystem.parameters.timePeriod !== undefined) { - global.DEFINITION.simulationParams.timePeriod = tradingSystem.parameters.timePeriod.code + bot.VALUES_TO_USE.timePeriod = tradingSystem.parameters.timePeriod.code + } + + /* Primary source of information, if it exists */ + if (bot.SESSION !== undefined) { + if (bot.SESSION.parameters !== undefined) { + if (bot.SESSION.parameters.timePeriod !== undefined) { + bot.VALUES_TO_USE.timePeriod = bot.SESSION.parameters.timePeriod.code + } + } } } } @@ -294,10 +338,10 @@ /* Checking if we should process this loop or not.*/ if (global.STOP_PROCESSING === true) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for Backtesting or Live Trade to be started."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be started."); } console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) - + " Waiting for Backtesting or Live Trade to be started. "); + + " Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be started. "); global.SYSTEM_EVENT_HANDLER.raiseEvent('Jason-Multi-Period', 'Process Stopped') diff --git a/server.js b/server.js index 0b7f12a40..bdf6269dc 100644 --- a/server.js +++ b/server.js @@ -71,10 +71,11 @@ if (global.TASK_NODE !== undefined) { } else { // I use this section to debug in standalone mode. - let argument = ' {"type":"Task","name":"Runs Backtests, Fordwardtests & Live Trades ","bot":{"type":"Trading Engine","processes":[{"type":"Process","name":"Multi Period","code":{"team":"AAMasters","bot":"AAJason","process":"Multi-Period","repo":"AAJason-Trading-Engine-Bot"},"id":"4748c8c4-4d19-4076-96b2-e9c06524fbb3"}]},"id":"561bac18-fc78-464a-90c5-79fd821fc633"}' - /* charly */ argument = '{"type":"Task","name":"Brings Trades Records from the Exchange","bot":{"type":"Sensor","name":"Charly","processes":[{"type":"Process","name":"Live Trades","code":{"team":"AAMasters","bot":"AACharly","process":"Live-Trades"},"id":"5846ebc7-1979-4a80-9cc7-94bb4b8659dc"},{"type":"Process","name":"Hole Fixing","code":{"team":"AAMasters","bot":"AACharly","process":"Hole-Fixing"},"id":"31fc4f05-75d4-419a-9fb0-73e25e856f15"}]},"id":"5bfef4dc-54c1-44db-ace6-1ab27a86746e"}' + let argument = '{"type":"Task","name":"Runs Backtests, Fordwardtests & Live Trades ","bot":{"type":"Trading Engine","processes":[{"type":"Process","subType":"Trading Engine Process","name":"Multi Period","code":{"team":"AAMasters","bot":"AAJason","process":"Multi-Period","repo":"AAJason-Trading-Engine-Bot"},"session":{"type":"Backtesting Session","name":"New Backtesting Session","parameters":{"type":"Parameters","name":"Parameters","baseAsset":{"type":"Base Asset","name":"Base Asset","code":"{ \n\"name\": \"BTC\",\n\"initialBalance\": 0.001,\n\"minimumBalance\": 0.0001,\n\"maximumBalance\": 1\n}"},"timeRange":{"type":"Time Range","name":"Time Range","code":"{\n\"initialDatetime\": \"2019-09-15T00:00:00.000Z\",\n\"finalDatetime\": \"2019-09-30T00:00:00.000Z\"\n}"},"timePeriod":{"type":"Time Period","name":"Time Period","code":"01-hs"},"slippage":{"type":"Slippage","name":"Slippage","code":"{\n\"positionRate\": 0.1,\n\"stopLoss\": 0.1,\n\"takeProfit\": 0.1\n}"},"feeStructure":{"type":"Fee Structure","name":"Fee Structure","code":"{\n\"maker\": 0.15,\n\"taker\": 0.25\n}"}},"id":"1381b048-7795-4e60-84a4-cf02e7385ba1"},"id":"072065fc-a57c-439c-94cf-f3f9c901ed37"}]},"id":"cddc97dd-ca2d-4b14-95ce-f15b6149996f"}' + /* charly argument = '{"type":"Task","name":"Brings Trades Records from the Exchange","bot":{"type":"Sensor","name":"Charly","processes":[{"type":"Process","name":"Live Trades","code":{"team":"AAMasters","bot":"AACharly","process":"Live-Trades"},"id":"5846ebc7-1979-4a80-9cc7-94bb4b8659dc"},{"type":"Process","name":"Hole Fixing","code":{"team":"AAMasters","bot":"AACharly","process":"Hole-Fixing"},"id":"31fc4f05-75d4-419a-9fb0-73e25e856f15"}]},"id":"5bfef4dc-54c1-44db-ace6-1ab27a86746e"}' // olivia argument = '{"type":"Task","name":"Generates 1 min to 24 hs Candles & Volumes","bot":{"type":"Indicator","name":"Olivia","processes":[{"type":"Process","name":"Daily","code":{"team":"AAMasters","bot":"AAOlivia","process":"Multi-Period-Daily"},"id":"68cc8e1b-e94a-477e-82d0-15ecc9f1b9e2"},{"type":"Process","name":"Market","code":{"team":"AAMasters","bot":"AAOlivia","process":"Multi-Period-Market"},"id":"e1ac5e3d-d491-4e90-baf4-d4e5b71a8b1a"}]},"id":"efe36e05-75ea-41b5-8d92-c6b27635c834"}' // bruce argument = ' {"type":"Task","name":"Converts Trades into 1 min Candles & Volumes","bot":{"type":"Indicator","name":"Bruce","processes":[{"type":"Process","name":"New Process","code":{"team":"AAMasters","bot":"AABruce","process":"Single-Period-Daily"},"id":"e184744f-5de0-41c1-ad4c-36960f4ced90"}]},"id":"42758590-a7bd-4712-a5c9-80a3a820c5b3"}' + */ try { global.TASK_NODE = JSON.parse(argument) } catch (err) { @@ -141,8 +142,6 @@ global.EXIT_NODE_PROCESS = function exitProcess() { setTimeout(process.exit, 10000) // We will give 10 seconds to logs be written on file } -let notFirstSequence = false; - /* Setting up the global Event Handler */ const EVENT_HANDLER_MODULE = require('./SystemEventHandler.js'); From 6875cc2bc05832dec8ec4feccc264c57af79362b Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 3 Oct 2019 17:10:39 +0200 Subject: [PATCH 07/19] Refactoring --- TradingEngineProcessMainLoop.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 0bad0c65f..8d910a862 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -147,9 +147,9 @@ processConfig.framework.endDate.fixedDate = processConfig.framework.endDate.fixedDate.toISOString() /* If we received simulation params we use them instead. */ - if (global.DEFINITION.simulationParams) { - if (global.DEFINITION.simulationParams.beginDatetime) { - processConfig.framework.startDate.fixedDate = global.DEFINITION.simulationParams.beginDatetime + if (global.DEFINITION.uiCurrentValues) { + if (global.DEFINITION.uiCurrentValues.beginDatetime) { + processConfig.framework.startDate.fixedDate = global.DEFINITION.uiCurrentValues.beginDatetime } } @@ -268,7 +268,7 @@ /* Decide which Time Period we are going to use.*/ /* This is the last fallback */ - bot.VALUES_TO_USE.timePeriod = global.DEFINITION.simulationParams.timePeriod + bot.VALUES_TO_USE.timePeriod = global.DEFINITION.uiCurrentValues.timePeriod /* Secondary option, if exists. */ if (tradingSystem.parameters.timePeriod !== undefined) { From e84da7e5c68cd09d064d6ca58e3024fd14798d59 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 3 Oct 2019 22:07:23 +0200 Subject: [PATCH 08/19] Session Running and Stopping from menu and at task server --- MultiPeriod.js | 6 ++-- TradingEngineProcessMainLoop.js | 52 +++++++++++++++++++++------------ server.js | 2 +- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/MultiPeriod.js b/MultiPeriod.js index a31e3870e..3f95dadb0 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -465,7 +465,7 @@ const logText = "User Defined End Datetime reached @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> advanceTime -> " + logText); } - global.STOP_PROCESSING = true + global.STOP_SESSION = true callBackFunction(global.DEFAULT_OK_RESPONSE); return; @@ -522,7 +522,7 @@ /* Validation that we dont need to stop. */ - if (global.STOP_PROCESSING === true) { + if (global.STOP_SESSION === true) { callBackFunction(global.DEFAULT_OK_RESPONSE); return; @@ -907,7 +907,7 @@ const logText = "User Defined End Datetime reached @ " + now.getUTCFullYear() + "/" + (now.getUTCMonth() + 1) + "/" + now.getUTCDate() + "."; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> " + logText); } - global.STOP_PROCESSING = true + global.STOP_SESSION = true callBackFunction(global.DEFAULT_OK_RESPONSE); return; diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 8d910a862..9edee5595 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -99,14 +99,13 @@ /* Some initial values*/ bot.enableCheckLoopHealth = true; let fixedTimeLoopIntervalHandle; - global.STOP_PROCESSING = true; + global.STOP_SESSION = true; /* Listen to event to start or stop the session. */ + bot.sessionKey = bot.processNode.session.name + '-' + bot.processNode.session.type + '-' + bot.processNode.session.id - let key = 'Session' + '-' + bot.processNode.session.id + '-' + 'Events' - - global.SYSTEM_EVENT_HANDLER.listenToEvent(key, 'Run Session', undefined, undefined, undefined, runSession) - global.SYSTEM_EVENT_HANDLER.listenToEvent(key, 'Stop Session', undefined, undefined, undefined, stopSession) + global.SYSTEM_EVENT_HANDLER.listenToEvent(bot.sessionKey, 'Run Session', undefined, undefined, undefined, runSession) + global.SYSTEM_EVENT_HANDLER.listenToEvent(bot.sessionKey, 'Stop Session', undefined, undefined, undefined, stopSession) function runSession(message) { switch (bot.processNode.session.type) { @@ -119,10 +118,12 @@ break } } + bot.sessionStatus = 'Idle' + global.STOP_SESSION = false } function stopSession(message) { - global.STOP_PROCESSING = true + global.STOP_SESSION = true } function startBackTesting(message) { @@ -132,7 +133,6 @@ bot.startMode = "Backtest" processConfig.framework.startDate.resumeExecution = false; - global.STOP_PROCESSING = false bot.hasTheBotJustStarted = true setValuesToUse(message) @@ -207,7 +207,6 @@ processConfig.framework.startDate.fixedDate = new Date() processConfig.framework.startDate.resumeExecution = false; bot.multiPeriodDailyProcessDatetime = processConfig.framework.startDate.fixedDate - global.STOP_PROCESSING = false bot.hasTheBotJustStarted = true setValuesToUse(message) } @@ -312,9 +311,9 @@ let nextWaitTime; - /* Loop Heartbeat sent to the UI */ - bot.hearBeat = hearBeat // allow the bot itself to produce a heartbeat - hearBeat() + /* Heartbeats sent to the UI */ + bot.sessionHeartBeat = sessionHeartBeat + processHeartBeat() /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ @@ -336,15 +335,26 @@ console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " Entered into Main Loop # " + pad(Number(bot.loopCounter), 8)); + /* Checking if we need to need to emit any event */ + + if (bot.sessionStatus === 'Idle' && global.STOP_SESSION === false) { + global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.sessionKey, 'Running') + bot.sessionStatus = 'Running' + } + + if (bot.sessionStatus === 'Running' && global.STOP_SESSION === true) { + global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.sessionKey, 'Stopped') + bot.sessionStatus = 'Stopped' + } + /* Checking if we should process this loop or not.*/ - if (global.STOP_PROCESSING === true) { + if (global.STOP_SESSION === true) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be started."); } console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be started. "); - global.SYSTEM_EVENT_HANDLER.raiseEvent('Jason-Multi-Period', 'Process Stopped') - nextWaitTime = 'Normal'; loopControl(nextWaitTime); return @@ -1077,7 +1087,7 @@ /* We show we reached the end of the loop. */ - hearBeat() + processHeartBeat() /* Here we check if we must stop the loop gracefully. */ @@ -1093,8 +1103,7 @@ logger.persist(); } - global.SYSTEM_EVENT_HANDLER.raiseEvent('Jason-Multi-Period', 'Process Stopped') - + global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.sessionKey, 'Stopped') callBackFunction(global.DEFAULT_OK_RESPONSE); return; @@ -1215,7 +1224,7 @@ } } - function hearBeat() { + function processHeartBeat() { let key = global.TASK_NODE.bot.processes[bot.processIndex].name + '-' + global.TASK_NODE.bot.processes[bot.processIndex].type + '-' + global.TASK_NODE.bot.processes[bot.processIndex].id let event = { @@ -1224,6 +1233,13 @@ global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) } + function sessionHeartBeat() { + let event = { + seconds: (new Date()).getSeconds() + } + global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.sessionKey, 'Heartbeat', event) + } + } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] run -> err = "+ err.stack); clearInterval(fixedTimeLoopIntervalHandle); diff --git a/server.js b/server.js index bdf6269dc..80441ad81 100644 --- a/server.js +++ b/server.js @@ -106,7 +106,7 @@ process.on('message', message => { if (message === 'Stop this Task') { global.STOP_TASK_GRACEFULLY = true; - global.STOP_PROCESSING = true + global.STOP_SESSION = true /* There are some process that might no be able to end grafully, for example the ones schedulle to process information in a future day or month. From 60db94703e6911c76202466e2dbb0a92eb618208 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 4 Oct 2019 08:16:03 +0200 Subject: [PATCH 09/19] details --- TradingEngineProcessMainLoop.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 9edee5595..673a8f628 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -350,10 +350,10 @@ /* Checking if we should process this loop or not.*/ if (global.STOP_SESSION === true) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be started."); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be ran."); } console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) - + " Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be started. "); + + " Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be ran. "); nextWaitTime = 'Normal'; loopControl(nextWaitTime); From 71739c0c27943373a885fefd81728cc14e789083 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 4 Oct 2019 09:58:47 +0200 Subject: [PATCH 10/19] Values to Use greatly improved --- Context.js | 6 +- TradingEngineProcessMainLoop.js | 175 +++++++++++++++++++++++++------- 2 files changed, 143 insertions(+), 38 deletions(-) diff --git a/Context.js b/Context.js index 6bb50a361..64c11744c 100644 --- a/Context.js +++ b/Context.js @@ -304,10 +304,10 @@ thisObject.executionHistory = []; - if (bot.VALUES_TO_USE.initialBalanceAssetA === undefined || bot.VALUES_TO_USE.initialBalanceAssetB === undefined) { throw new Error("Global Variables INITIAL_BALANCE_A and INITIAL_BALANCE_B cannot be undefined.") } + if (bot.VALUES_TO_USE.initialBalanceA === undefined || bot.VALUES_TO_USE.initialBalanceB === undefined) { throw new Error("Global Variables INITIAL_BALANCE_A and INITIAL_BALANCE_B cannot be undefined.") } - const INITIAL_BALANCE_A = Number(bot.VALUES_TO_USE.initialBalanceAssetA); - const INITIAL_BALANCE_B = Number(bot.VALUES_TO_USE.initialBalanceAssetB); + const INITIAL_BALANCE_A = Number(bot.VALUES_TO_USE.initialBalanceA); + const INITIAL_BALANCE_B = Number(bot.VALUES_TO_USE.initialBalanceB); thisObject.executionContext = { diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 673a8f628..7340f9d90 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -218,67 +218,172 @@ */ bot.SESSION = JSON.parse(message.event.session) - bot.VALUES_TO_USE = {} - if (global.DEFINITION !== undefined) { - let tradingSystem = global.DEFINITION.tradingSystem + /* Set all default values */ + bot.VALUES_TO_USE = { + baseAsset: "BTC", + initialBalanceA: 0, + initialBalanceB: 0.001, + minimumBalanceA: 0, + minimumBalanceB: 0.0005, + maximumBalanceA: 0, + maximumBalanceB: 0.002, + timePeriod: global.DEFINITION.uiCurrentValues.timePeriod, + slippage: { + positionRate: 0, + stopLoss: 0, + takeProfit: 0 + }, + feeStructure: { + maker: 0, + taker: 0 + } + } - if (tradingSystem) { - if (tradingSystem.parameters !== undefined) { + let tradingSystem = global.DEFINITION.tradingSystem - /* Decide which Initial Balance we are going to use. */ + if (tradingSystem !== undefined) { + + /* Applying Trading System Level Parameters */ + if (tradingSystem.parameters !== undefined) { + + /* Base Asset and Initial Balances. */ + { if (tradingSystem.parameters.baseAsset !== undefined) { let code try { - /* Default Value */ - bot.VALUES_TO_USE.initialBalanceAssetB = 0.001; - bot.VALUES_TO_USE.initialBalanceAssetA = 0 - code = JSON.parse(tradingSystem.parameters.baseAsset.code); if (code.name !== undefined) { - baseAsset = code.name; - if (baseAsset !== 'BTC' && baseAsset !== 'USDT') { - /* using BTC as default */ - baseAsset = 'BTC' - } + bot.VALUES_TO_USE.baseAsset = code.name; } if (baseAsset === 'BTC') { // NOTE: POLONIEX, the only exchange working so far, has Asset A and B inverted. We need to fix this. if (code.initialBalance !== undefined) { - bot.VALUES_TO_USE.initialBalanceAssetB = code.initialBalance; - bot.VALUES_TO_USE.initialBalanceAssetA = 0 + bot.VALUES_TO_USE.initialBalanceB = code.initialBalance; + bot.VALUES_TO_USE.initialBalanceA = 0 + } + if (code.minimumBalance !== undefined) { + bot.VALUES_TO_USE.minimumBalanceA = code.minimumBalance; + bot.VALUES_TO_USE.minimumBalanceB = 0 + } + if (code.maximumBalance !== undefined) { + bot.VALUES_TO_USE.maximumBalanceA = code.maximumBalance; + bot.VALUES_TO_USE.maximumBalanceB = 0 } } else { if (code.initialBalance !== undefined) { - bot.VALUES_TO_USE.initialBalanceAssetA = code.initialBalance; - bot.VALUES_TO_USE.initialBalanceAssetB = 0 + bot.VALUES_TO_USE.initialBalanceA = code.initialBalance; + bot.VALUES_TO_USE.initialBalanceB = 0 + } + if (code.minimumBalance !== undefined) { + bot.VALUES_TO_USE.minimumBalanceB = code.minimumBalance; + bot.VALUES_TO_USE.minimumBalanceA = 0 + } + if (code.maximumBalance !== undefined) { + bot.VALUES_TO_USE.maximumBalanceB = code.maximumBalance; + bot.VALUES_TO_USE.maximumBalanceA = 0 } } } catch (err) { - global.DEFINITION.tradingSystem.parameters.baseAsset.error = err.message + tradingSystem.parameters.baseAsset.error = err.message + } + } + } + + /* Time Period */ + if (tradingSystem.parameters.timePeriod !== undefined) { + bot.VALUES_TO_USE.timePeriod = tradingSystem.parameters.timePeriod.code + } - bot.VALUES_TO_USE.initialBalanceAssetA = 0 // default - bot.VALUES_TO_USE.initialBalanceAssetB = 0.001 // default + /* Slippage */ + if (tradingSystem.parameters.slippage !== undefined) { + if (tradingSystem.parameters.slippage.code !== undefined) { + try { + let code = JSON.parse(tradingSystem.parameters.slippage.code) + if (code.positionRate !== undefined) { + bot.VALUES_TO_USE.slippage.positionRate = code.positionRate + } + if (code.stopLoss !== undefined) { + bot.VALUES_TO_USE.slippage.stopLoss = code.stopLoss + } + if (code.takeProfit !== undefined) { + bot.VALUES_TO_USE.slippage.takeProfit = code.takeProfit + } + + } catch (err) { + tradingSystem.parameters.slippage.error = err.message + } + } + } + + /* Fee Structure */ + if (tradingSystem.parameters.feeStructure !== undefined) { + if (tradingSystem.parameters.feeStructure.code !== undefined) { + try { + let code = JSON.parse(tradingSystem.parameters.feeStructure.code) + + if (code.maker !== undefined) { + bot.VALUES_TO_USE.feeStructure.maker = code.maker + } + if (code.taker !== undefined) { + bot.VALUES_TO_USE.feeStructure.taker = code.taker + } + + } catch (err) { + tradingSystem.parameters.feeStructure.error = err.message + } + } + } + } + + /* Applying Session Level Parameters */ + if (bot.SESSION !== undefined) { + if (bot.SESSION.parameters !== undefined) { + + /* Time Period */ + if (bot.SESSION.parameters.timePeriod !== undefined) { + bot.VALUES_TO_USE.timePeriod = bot.SESSION.parameters.timePeriod.code + } + + /* Slippage */ + if (bot.SESSION.parameters.slippage !== undefined) { + if (bot.SESSION.parameters.slippage.code !== undefined) { + try { + let code = JSON.parse(bot.SESSION.parameters.slippage.code) + + if (code.positionRate !== undefined) { + bot.VALUES_TO_USE.slippage.positionRate = code.positionRate + } + if (code.stopLoss !== undefined) { + bot.VALUES_TO_USE.slippage.stopLoss = code.stopLoss + } + if (code.takeProfit !== undefined) { + bot.VALUES_TO_USE.slippage.takeProfit = code.takeProfit + } + + } catch (err) { + parentLogger.write(MODULE_NAME, "[WARN] run -> startLiveTrading -> Invalid Slippage Value -> err = " + err.stack); + } } } - /* Decide which Time Period we are going to use.*/ + /* Fee Structure */ + if (bot.SESSION.parameters.feeStructure !== undefined) { + if (bot.SESSION.parameters.feeStructure.code !== undefined) { + try { + let code = JSON.parse(bot.SESSION.parameters.feeStructure.code) - /* This is the last fallback */ - bot.VALUES_TO_USE.timePeriod = global.DEFINITION.uiCurrentValues.timePeriod + if (code.maker !== undefined) { + bot.VALUES_TO_USE.feeStructure.maker = code.maker + } + if (code.taker !== undefined) { + bot.VALUES_TO_USE.feeStructure.taker = code.taker + } - /* Secondary option, if exists. */ - if (tradingSystem.parameters.timePeriod !== undefined) { - bot.VALUES_TO_USE.timePeriod = tradingSystem.parameters.timePeriod.code - } - - /* Primary source of information, if it exists */ - if (bot.SESSION !== undefined) { - if (bot.SESSION.parameters !== undefined) { - if (bot.SESSION.parameters.timePeriod !== undefined) { - bot.VALUES_TO_USE.timePeriod = bot.SESSION.parameters.timePeriod.code + } catch (err) { + parentLogger.write(MODULE_NAME, "[WARN] run -> startLiveTrading -> Invalid Fee Structure Value -> err = " + err.stack); } } } From 8c44f53f80cda551ea4a4ee703caeb2be34487e6 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 4 Oct 2019 10:48:55 +0200 Subject: [PATCH 11/19] Better Handling of timeRange --- MultiPeriod.js | 10 +-- TradingEngineProcessMainLoop.js | 126 ++++++++++++++++++-------------- 2 files changed, 73 insertions(+), 63 deletions(-) diff --git a/MultiPeriod.js b/MultiPeriod.js index 3f95dadb0..eb287180e 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -119,7 +119,7 @@ let statusReport; /* We are going to use the start date as beging of market date. */ - contextVariables.dateBeginOfMarket = new Date(processConfig.framework.startDate.fixedDate) + contextVariables.dateBeginOfMarket = bot.VALUES_TO_USE.timeRange.initialDatetime /* Here we get the status report from the bot who knows which is the end of the market. @@ -459,8 +459,7 @@ /* Validation that we are not going past the user defined end date. */ - const userDefinedEndDatetime = new Date(processConfig.framework.endDate.fixedDate); - if (bot.multiPeriodDailyProcessDatetime.valueOf() >= userDefinedEndDatetime.valueOf()) { + if (bot.multiPeriodDailyProcessDatetime.valueOf() >= bot.VALUES_TO_USE.timeRange.finalDatetime.valueOf()) { const logText = "User Defined End Datetime reached @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> advanceTime -> " + logText); } @@ -834,8 +833,6 @@ currentTimePeriod, currentOutputPeriodLabel, bot.multiPeriodDailyProcessDatetime, - processConfig.framework.startDate.fixedDate, - processConfig.framework.endDate.fixedDate, interExecutionMemoryArray[n], onBotFinished); @@ -900,9 +897,8 @@ /* This is where we check if we need reached the user defined end datetime. */ - const userDefinedEndDatetime = new Date(processConfig.framework.endDate.fixedDate); const now = new Date() - if (now.valueOf() >= userDefinedEndDatetime.valueOf()) { + if (now.valueOf() >= bot.VALUES_TO_USE.timeRange.finalDatetime.valueOf()) { const logText = "User Defined End Datetime reached @ " + now.getUTCFullYear() + "/" + (now.getUTCMonth() + 1) + "/" + now.getUTCDate() + "."; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> " + logText); } diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 7340f9d90..8cf375d52 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -108,7 +108,14 @@ global.SYSTEM_EVENT_HANDLER.listenToEvent(bot.sessionKey, 'Stop Session', undefined, undefined, undefined, stopSession) function runSession(message) { - switch (bot.processNode.session.type) { + + /* We are going to run the Definition comming at the event. */ + global.DEFINITION = JSON.parse(message.event.definition) + bot.SESSION = JSON.parse(message.event.session) + + setValuesToUse(message) + + switch (bot.SESSION.type) { case 'Backtesting Session': { startBackTesting(message) break @@ -127,60 +134,14 @@ } function startBackTesting(message) { - - /* We are going to run the Definition comming at the event. */ - global.DEFINITION = JSON.parse(message.event.definition) - bot.startMode = "Backtest" processConfig.framework.startDate.resumeExecution = false; - bot.hasTheBotJustStarted = true - setValuesToUse(message) - - const ONE_YEAR_IN_MILISECONDS = 365 * 24 * 60 * 60 * 1000 - /* If we can not get for any reason the initial date of the backtest, we will start at present time. */ - processConfig.framework.startDate.fixedDate = new Date() - /* If we can not get for any reason the final date of the backtest, we will stop one year after the start date. */ - processConfig.framework.endDate.fixedDate = new Date(processConfig.framework.startDate.fixedDate.valueOf() + ONE_YEAR_IN_MILISECONDS) - - /* Convert this dates to string */ - processConfig.framework.startDate.fixedDate = processConfig.framework.startDate.fixedDate.toISOString() - processConfig.framework.endDate.fixedDate = processConfig.framework.endDate.fixedDate.toISOString() - - /* If we received simulation params we use them instead. */ - if (global.DEFINITION.uiCurrentValues) { - if (global.DEFINITION.uiCurrentValues.beginDatetime) { - processConfig.framework.startDate.fixedDate = global.DEFINITION.uiCurrentValues.beginDatetime - } - } - - /* We finally will try to exctact the initial and final date for the backtest from the Definition */ - if (global.DEFINITION.tradingSystem !== undefined) { - if (global.DEFINITION.tradingSystem.parameters !== undefined) { - if (global.DEFINITION.tradingSystem.parameters.timeRange !== undefined) { - if (global.DEFINITION.tradingSystem.parameters.timeRange.code !== undefined) { - try { - let code = JSON.parse(global.DEFINITION.tradingSystem.parameters.timeRange.code) - if (code.initialDatetime !== undefined) { - processConfig.framework.startDate.fixedDate = code.initialDatetime /* The second override occurs here, with the date explicitelly defined by the user */ - } - if (code.finalDatetime !== undefined) { - processConfig.framework.endDate.fixedDate = code.finalDatetime - } - } catch (err) { - global.DEFINITION.tradingSystem.parameters.timeRange.error = err.message - } - } - } - } - } - bot.multiPeriodDailyProcessDatetime = processConfig.framework.startDate.fixedDate + bot.hasTheBotJustStarted = true + bot.multiPeriodDailyProcessDatetime = bot.VALUES_TO_USE.timeRange.initialDatetime } function startLiveTrading(message) { - /* We are going to run the Definition comming at the event. */ - global.DEFINITION = JSON.parse(message.event.definition) - if (global.DEFINITION.personalData) { if (global.DEFINITION.personalData.exchangeAccounts) { if (global.DEFINITION.personalData.exchangeAccounts.length > 0) { @@ -204,11 +165,13 @@ } bot.startMode = "Live" - processConfig.framework.startDate.fixedDate = new Date() + if (bot.VALUES_TO_USE.timeRange.initialDatetime.valueOf() < (new Date()).valueOf) { + bot.VALUES_TO_USE.timeRange.initialDatetime = new Date() + } processConfig.framework.startDate.resumeExecution = false; - bot.multiPeriodDailyProcessDatetime = processConfig.framework.startDate.fixedDate + bot.multiPeriodDailyProcessDatetime = bot.VALUES_TO_USE.timeRange.initialDatetime bot.hasTheBotJustStarted = true - setValuesToUse(message) + } function setValuesToUse(message) { @@ -216,9 +179,7 @@ Base on the information received we will determined which values ultimatelly are going to be used, and once we do, they will become constants across the multiple loops executions. */ - - bot.SESSION = JSON.parse(message.event.session) - + /* Set all default values */ bot.VALUES_TO_USE = { baseAsset: "BTC", @@ -237,6 +198,25 @@ feeStructure: { maker: 0, taker: 0 + }, + timeRange: { + initialDatetime: new Date(), + finalDatetime: new Date() + } + } + + /* Session Type Dependant Default Values */ + const ONE_YEAR_IN_MILISECONDS = 365 * 24 * 60 * 60 * 1000 + switch (bot.SESSION.type) { + case 'Backtesting Session': { + bot.VALUES_TO_USE.timeRange.initialDatetime = new Date(global.DEFINITION.uiCurrentValues.initialDatetime) + bot.VALUES_TO_USE.timeRange.finalDatetime = new Date() + break + } + case 'Live Trading Session': { + bot.VALUES_TO_USE.timeRange.initialDatetime = new Date() + bot.VALUES_TO_USE.timeRange.finalDatetime = new Date((new Date()).valueOf() + ONE_YEAR_IN_MILISECONDS) + break } } @@ -335,7 +315,24 @@ tradingSystem.parameters.feeStructure.error = err.message } } - } + } + + /* Time Range */ + if (tradingSystem.parameters.timeRange !== undefined) { + if (tradingSystem.parameters.timeRange.code !== undefined) { + try { + let code = JSON.parse(tradingSystem.parameters.timeRange.code) + if (code.initialDatetime !== undefined) { + bot.VALUES_TO_USE.timeRange.initialDatetime = new Date(code.initialDatetime) + } + if (code.finalDatetime !== undefined) { + bot.VALUES_TO_USE.timeRange.finalDatetime = new Date(code.finalDatetime) + } + } catch (err) { + tradingSystem.parameters.timeRange.error = err.message + } + } + } } /* Applying Session Level Parameters */ @@ -387,6 +384,23 @@ } } } + + /* Time Range */ + if (bot.SESSION.parameters.timeRange !== undefined) { + if (bot.SESSION.parameters.timeRange.code !== undefined) { + try { + let code = JSON.parse(bot.SESSION.parameters.timeRange.code) + if (code.initialDatetime !== undefined) { + bot.VALUES_TO_USE.timeRange.initialDatetime = new Date(code.initialDatetime) + } + if (code.finalDatetime !== undefined) { + bot.VALUES_TO_USE.timeRange.finalDatetime = new Date(code.finalDatetime) + } + } catch (err) { + parentLogger.write(MODULE_NAME, "[WARN] run -> startLiveTrading -> Invalid Time Range Value -> err = " + err.stack); + } + } + } } } } From d4a08b839c51f326552f82ee9886f3e30cd2e0b8 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 4 Oct 2019 12:03:55 +0200 Subject: [PATCH 12/19] New Session Module Implemented --- MultiPeriod.js | 8 +- Project.njsproj | 3 + Session.js | 335 ++++++++++++++++++++++++++++++++ TradingEngineProcessMainLoop.js | 333 ++----------------------------- server.js | 1 - 5 files changed, 356 insertions(+), 324 deletions(-) create mode 100644 Session.js diff --git a/MultiPeriod.js b/MultiPeriod.js index eb287180e..d9ccbfac2 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -184,7 +184,7 @@ if (thisReport.lastFile !== undefined) { - if (bot.hasTheBotJustStarted === true && processConfig.framework.startDate.resumeExecution === false) { + if (bot.hasTheBotJustStarted === true && bot.resumeExecution === false) { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> getContextVariables -> Starting from the begining because bot has just started and resume execution was true."); } startFromBegining(); @@ -464,7 +464,7 @@ const logText = "User Defined End Datetime reached @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> advanceTime -> " + logText); } - global.STOP_SESSION = true + bot.STOP_SESSION = true callBackFunction(global.DEFAULT_OK_RESPONSE); return; @@ -521,7 +521,7 @@ /* Validation that we dont need to stop. */ - if (global.STOP_SESSION === true) { + if (bot.STOP_SESSION === true) { callBackFunction(global.DEFAULT_OK_RESPONSE); return; @@ -903,7 +903,7 @@ const logText = "User Defined End Datetime reached @ " + now.getUTCFullYear() + "/" + (now.getUTCMonth() + 1) + "/" + now.getUTCDate() + "."; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> callTheBot -> onBotFinished -> onMarketStatusReport -> " + logText); } - global.STOP_SESSION = true + bot.STOP_SESSION = true callBackFunction(global.DEFAULT_OK_RESPONSE); return; diff --git a/Project.njsproj b/Project.njsproj index 207b21485..63fc32b8b 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -36,6 +36,9 @@ Code + + Code + Code diff --git a/Session.js b/Session.js new file mode 100644 index 000000000..fd319381a --- /dev/null +++ b/Session.js @@ -0,0 +1,335 @@ +exports.newSession = function newSession(bot, parentLogger) { + + const MODULE_NAME = "Session" + const FULL_LOG = true; + + let thisObject = { + initialize: initialize + } + + return thisObject; + + function initialize(callBackFunction) { + try { + if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } + + /* Check if there is a session */ + if (bot.processNode.session === undefined) { + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> Cannot run without a Session."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; + } + + /* Listen to event to start or stop the session. */ + bot.sessionKey = bot.processNode.session.name + '-' + bot.processNode.session.type + '-' + bot.processNode.session.id + + global.SYSTEM_EVENT_HANDLER.listenToEvent(bot.sessionKey, 'Run Session', undefined, undefined, undefined, runSession) + global.SYSTEM_EVENT_HANDLER.listenToEvent(bot.sessionKey, 'Stop Session', undefined, undefined, undefined, stopSession) + + function runSession(message) { + + /* We are going to run the Definition comming at the event. */ + bot.DEFINITION = JSON.parse(message.event.definition) + bot.SESSION = JSON.parse(message.event.session) + + setValuesToUse(message) + + switch (bot.SESSION.type) { + case 'Backtesting Session': { + startBackTesting(message) + break + } + case 'Live Trading Session': { + startLiveTrading(message) + break + } + } + bot.SESSION_STATUS = 'Idle' + bot.STOP_SESSION = false + } + + function stopSession(message) { + bot.STOP_SESSION = true + } + + function startBackTesting(message) { + bot.startMode = "Backtest" + bot.resumeExecution = false; + bot.hasTheBotJustStarted = true + bot.multiPeriodDailyProcessDatetime = bot.VALUES_TO_USE.timeRange.initialDatetime + } + + function startLiveTrading(message) { + + if (bot.DEFINITION.personalData) { + if (bot.DEFINITION.personalData.exchangeAccounts) { + if (bot.DEFINITION.personalData.exchangeAccounts.length > 0) { + let exchangeAccount = bot.DEFINITION.personalData.exchangeAccounts[0] + if (exchangeAccount.keys) { + if (exchangeAccount.keys.length > 0) { + let key = exchangeAccount.keys[0] + + process.env.KEY = key.name + process.env.SECRET = key.code + + } + } + } + } + } + + if (process.env.KEY === undefined || process.env.SECRET === undefined) { + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[WARN] initialize -> startLiveTrading -> Key name or Secret not provided, not possible to run the process in Live mode."); } + return + } + + bot.startMode = "Live" + if (bot.VALUES_TO_USE.timeRange.initialDatetime.valueOf() < (new Date()).valueOf) { + bot.VALUES_TO_USE.timeRange.initialDatetime = new Date() + } + bot.resumeExecution = false; + bot.multiPeriodDailyProcessDatetime = bot.VALUES_TO_USE.timeRange.initialDatetime + bot.hasTheBotJustStarted = true + + } + + function setValuesToUse(message) { + /* + Base on the information received we will determined which values ultimatelly are going to be used, + and once we do, they will become constants across the multiple loops executions. + */ + + /* Set all default values */ + bot.VALUES_TO_USE = { + baseAsset: "BTC", + initialBalanceA: 0, + initialBalanceB: 0.001, + minimumBalanceA: 0, + minimumBalanceB: 0.0005, + maximumBalanceA: 0, + maximumBalanceB: 0.002, + timePeriod: bot.DEFINITION.uiCurrentValues.timePeriod, + slippage: { + positionRate: 0, + stopLoss: 0, + takeProfit: 0 + }, + feeStructure: { + maker: 0, + taker: 0 + }, + timeRange: { + initialDatetime: new Date(), + finalDatetime: new Date() + } + } + + /* Session Type Dependant Default Values */ + const ONE_YEAR_IN_MILISECONDS = 365 * 24 * 60 * 60 * 1000 + switch (bot.SESSION.type) { + case 'Backtesting Session': { + bot.VALUES_TO_USE.timeRange.initialDatetime = new Date(bot.DEFINITION.uiCurrentValues.initialDatetime) + bot.VALUES_TO_USE.timeRange.finalDatetime = new Date() + break + } + case 'Live Trading Session': { + bot.VALUES_TO_USE.timeRange.initialDatetime = new Date() + bot.VALUES_TO_USE.timeRange.finalDatetime = new Date((new Date()).valueOf() + ONE_YEAR_IN_MILISECONDS) + break + } + } + + let tradingSystem = bot.DEFINITION.tradingSystem + + if (tradingSystem !== undefined) { + + /* Applying Trading System Level Parameters */ + if (tradingSystem.parameters !== undefined) { + + /* Base Asset and Initial Balances. */ + { + if (tradingSystem.parameters.baseAsset !== undefined) { + let code + try { + code = JSON.parse(tradingSystem.parameters.baseAsset.code); + + if (code.name !== undefined) { + bot.VALUES_TO_USE.baseAsset = code.name; + } + + if (baseAsset === 'BTC') { // NOTE: POLONIEX, the only exchange working so far, has Asset A and B inverted. We need to fix this. + if (code.initialBalance !== undefined) { + bot.VALUES_TO_USE.initialBalanceB = code.initialBalance; + bot.VALUES_TO_USE.initialBalanceA = 0 + } + if (code.minimumBalance !== undefined) { + bot.VALUES_TO_USE.minimumBalanceA = code.minimumBalance; + bot.VALUES_TO_USE.minimumBalanceB = 0 + } + if (code.maximumBalance !== undefined) { + bot.VALUES_TO_USE.maximumBalanceA = code.maximumBalance; + bot.VALUES_TO_USE.maximumBalanceB = 0 + } + } else { + if (code.initialBalance !== undefined) { + bot.VALUES_TO_USE.initialBalanceA = code.initialBalance; + bot.VALUES_TO_USE.initialBalanceB = 0 + } + if (code.minimumBalance !== undefined) { + bot.VALUES_TO_USE.minimumBalanceB = code.minimumBalance; + bot.VALUES_TO_USE.minimumBalanceA = 0 + } + if (code.maximumBalance !== undefined) { + bot.VALUES_TO_USE.maximumBalanceB = code.maximumBalance; + bot.VALUES_TO_USE.maximumBalanceA = 0 + } + } + } catch (err) { + tradingSystem.parameters.baseAsset.error = err.message + } + } + } + + /* Time Period */ + if (tradingSystem.parameters.timePeriod !== undefined) { + bot.VALUES_TO_USE.timePeriod = tradingSystem.parameters.timePeriod.code + } + + /* Slippage */ + if (tradingSystem.parameters.slippage !== undefined) { + if (tradingSystem.parameters.slippage.code !== undefined) { + try { + let code = JSON.parse(tradingSystem.parameters.slippage.code) + + if (code.positionRate !== undefined) { + bot.VALUES_TO_USE.slippage.positionRate = code.positionRate + } + if (code.stopLoss !== undefined) { + bot.VALUES_TO_USE.slippage.stopLoss = code.stopLoss + } + if (code.takeProfit !== undefined) { + bot.VALUES_TO_USE.slippage.takeProfit = code.takeProfit + } + + } catch (err) { + tradingSystem.parameters.slippage.error = err.message + } + } + } + + /* Fee Structure */ + if (tradingSystem.parameters.feeStructure !== undefined) { + if (tradingSystem.parameters.feeStructure.code !== undefined) { + try { + let code = JSON.parse(tradingSystem.parameters.feeStructure.code) + + if (code.maker !== undefined) { + bot.VALUES_TO_USE.feeStructure.maker = code.maker + } + if (code.taker !== undefined) { + bot.VALUES_TO_USE.feeStructure.taker = code.taker + } + + } catch (err) { + tradingSystem.parameters.feeStructure.error = err.message + } + } + } + + /* Time Range */ + if (tradingSystem.parameters.timeRange !== undefined) { + if (tradingSystem.parameters.timeRange.code !== undefined) { + try { + let code = JSON.parse(tradingSystem.parameters.timeRange.code) + if (code.initialDatetime !== undefined) { + bot.VALUES_TO_USE.timeRange.initialDatetime = new Date(code.initialDatetime) + } + if (code.finalDatetime !== undefined) { + bot.VALUES_TO_USE.timeRange.finalDatetime = new Date(code.finalDatetime) + } + } catch (err) { + tradingSystem.parameters.timeRange.error = err.message + } + } + } + } + + /* Applying Session Level Parameters */ + if (bot.SESSION !== undefined) { + if (bot.SESSION.parameters !== undefined) { + + /* Time Period */ + if (bot.SESSION.parameters.timePeriod !== undefined) { + bot.VALUES_TO_USE.timePeriod = bot.SESSION.parameters.timePeriod.code + } + + /* Slippage */ + if (bot.SESSION.parameters.slippage !== undefined) { + if (bot.SESSION.parameters.slippage.code !== undefined) { + try { + let code = JSON.parse(bot.SESSION.parameters.slippage.code) + + if (code.positionRate !== undefined) { + bot.VALUES_TO_USE.slippage.positionRate = code.positionRate + } + if (code.stopLoss !== undefined) { + bot.VALUES_TO_USE.slippage.stopLoss = code.stopLoss + } + if (code.takeProfit !== undefined) { + bot.VALUES_TO_USE.slippage.takeProfit = code.takeProfit + } + + } catch (err) { + parentLogger.write(MODULE_NAME, "[WARN] initialize -> startLiveTrading -> Invalid Slippage Value -> err = " + err.stack); + } + } + } + + /* Fee Structure */ + if (bot.SESSION.parameters.feeStructure !== undefined) { + if (bot.SESSION.parameters.feeStructure.code !== undefined) { + try { + let code = JSON.parse(bot.SESSION.parameters.feeStructure.code) + + if (code.maker !== undefined) { + bot.VALUES_TO_USE.feeStructure.maker = code.maker + } + if (code.taker !== undefined) { + bot.VALUES_TO_USE.feeStructure.taker = code.taker + } + + } catch (err) { + parentLogger.write(MODULE_NAME, "[WARN] initialize -> startLiveTrading -> Invalid Fee Structure Value -> err = " + err.stack); + } + } + } + + /* Time Range */ + if (bot.SESSION.parameters.timeRange !== undefined) { + if (bot.SESSION.parameters.timeRange.code !== undefined) { + try { + let code = JSON.parse(bot.SESSION.parameters.timeRange.code) + if (code.initialDatetime !== undefined) { + bot.VALUES_TO_USE.timeRange.initialDatetime = new Date(code.initialDatetime) + } + if (code.finalDatetime !== undefined) { + bot.VALUES_TO_USE.timeRange.finalDatetime = new Date(code.finalDatetime) + } + } catch (err) { + parentLogger.write(MODULE_NAME, "[WARN] initialize -> startLiveTrading -> Invalid Time Range Value -> err = " + err.stack); + } + } + } + } + } + } + } + + callBackFunction(global.DEFAULT_OK_RESPONSE) + + } catch (err) { + parentLogger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.stack); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + } + } +}; diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 8cf375d52..ed8f4b9d4 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -12,8 +12,10 @@ const MULTI_PERIOD_DAILY = require(ROOT_DIR + 'MultiPeriodDaily'); const MULTI_PERIOD = require(ROOT_DIR + 'MultiPeriod'); const FILE_STORAGE = require('./FileStorage.js'); + const SESSION = require(ROOT_DIR + 'Session'); let fileStorage = FILE_STORAGE.newFileStorage(parentLogger); + let session = SESSION.newSession(bot, parentLogger) const DEBUG_MODULE = require(ROOT_DIR + 'DebugLog'); let logger; // We need this here in order for the loopHealth function to work and be able to rescue the loop when it gets in trouble. @@ -38,6 +40,7 @@ if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } processConfig = pProcessConfig; + bot.resumeExecution = processConfig.framework.startDate.resumeExecution // We are inherating this from root, but from here we need it at bot let filePath = bot.devTeam + "/" + "bots" + "/" + bot.repo + "/" + pProcessConfig.name filePath += "/User.Bot.js" @@ -74,7 +77,11 @@ //COMMONS_MODULE.newCommons = require(process.env.BOTS_PATH + '/aamasters/AAMasters/bots/AAJason-Trading-Engine-Bot/Commons').newCommons // Use this for a better debugging experience. You need to bring this js module to this folder in order to work. COMMONS_MODULE.newCommons = eval(text); // Use this for production - callBackFunction(global.DEFAULT_OK_RESPONSE); + session.initialize(onSessionInitialized) + + function onSessionInitialized(err) { + callBackFunction(err); + } } } } catch (err) { @@ -89,322 +96,10 @@ if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] run -> Entering function."); } - /* Check if there is a session */ - if (bot.processNode.session === undefined) { - parentLogger.write(MODULE_NAME, "[ERROR] run -> Cannot run without a Session."); - callBackFunction(global.DEFAULT_FAIL_RESPONSE); - return; - } - /* Some initial values*/ bot.enableCheckLoopHealth = true; let fixedTimeLoopIntervalHandle; - global.STOP_SESSION = true; - - /* Listen to event to start or stop the session. */ - bot.sessionKey = bot.processNode.session.name + '-' + bot.processNode.session.type + '-' + bot.processNode.session.id - - global.SYSTEM_EVENT_HANDLER.listenToEvent(bot.sessionKey, 'Run Session', undefined, undefined, undefined, runSession) - global.SYSTEM_EVENT_HANDLER.listenToEvent(bot.sessionKey, 'Stop Session', undefined, undefined, undefined, stopSession) - - function runSession(message) { - - /* We are going to run the Definition comming at the event. */ - global.DEFINITION = JSON.parse(message.event.definition) - bot.SESSION = JSON.parse(message.event.session) - - setValuesToUse(message) - - switch (bot.SESSION.type) { - case 'Backtesting Session': { - startBackTesting(message) - break - } - case 'Live Trading Session': { - startLiveTrading(message) - break - } - } - bot.sessionStatus = 'Idle' - global.STOP_SESSION = false - } - - function stopSession(message) { - global.STOP_SESSION = true - } - - function startBackTesting(message) { - bot.startMode = "Backtest" - processConfig.framework.startDate.resumeExecution = false; - bot.hasTheBotJustStarted = true - bot.multiPeriodDailyProcessDatetime = bot.VALUES_TO_USE.timeRange.initialDatetime - } - - function startLiveTrading(message) { - - if (global.DEFINITION.personalData) { - if (global.DEFINITION.personalData.exchangeAccounts) { - if (global.DEFINITION.personalData.exchangeAccounts.length > 0) { - let exchangeAccount = global.DEFINITION.personalData.exchangeAccounts[0] - if (exchangeAccount.keys) { - if (exchangeAccount.keys.length > 0) { - let key = exchangeAccount.keys[0] - - process.env.KEY = key.name - process.env.SECRET = key.code - - } - } - } - } - } - - if (process.env.KEY === undefined || process.env.SECRET === undefined) { - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[WARN] run -> startLiveTrading -> Key name or Secret not provided, not possible to run the process in Live mode."); } - return - } - - bot.startMode = "Live" - if (bot.VALUES_TO_USE.timeRange.initialDatetime.valueOf() < (new Date()).valueOf) { - bot.VALUES_TO_USE.timeRange.initialDatetime = new Date() - } - processConfig.framework.startDate.resumeExecution = false; - bot.multiPeriodDailyProcessDatetime = bot.VALUES_TO_USE.timeRange.initialDatetime - bot.hasTheBotJustStarted = true - - } - - function setValuesToUse(message) { - /* - Base on the information received we will determined which values ultimatelly are going to be used, - and once we do, they will become constants across the multiple loops executions. - */ - - /* Set all default values */ - bot.VALUES_TO_USE = { - baseAsset: "BTC", - initialBalanceA: 0, - initialBalanceB: 0.001, - minimumBalanceA: 0, - minimumBalanceB: 0.0005, - maximumBalanceA: 0, - maximumBalanceB: 0.002, - timePeriod: global.DEFINITION.uiCurrentValues.timePeriod, - slippage: { - positionRate: 0, - stopLoss: 0, - takeProfit: 0 - }, - feeStructure: { - maker: 0, - taker: 0 - }, - timeRange: { - initialDatetime: new Date(), - finalDatetime: new Date() - } - } - - /* Session Type Dependant Default Values */ - const ONE_YEAR_IN_MILISECONDS = 365 * 24 * 60 * 60 * 1000 - switch (bot.SESSION.type) { - case 'Backtesting Session': { - bot.VALUES_TO_USE.timeRange.initialDatetime = new Date(global.DEFINITION.uiCurrentValues.initialDatetime) - bot.VALUES_TO_USE.timeRange.finalDatetime = new Date() - break - } - case 'Live Trading Session': { - bot.VALUES_TO_USE.timeRange.initialDatetime = new Date() - bot.VALUES_TO_USE.timeRange.finalDatetime = new Date((new Date()).valueOf() + ONE_YEAR_IN_MILISECONDS) - break - } - } - - let tradingSystem = global.DEFINITION.tradingSystem - - if (tradingSystem !== undefined) { - - /* Applying Trading System Level Parameters */ - if (tradingSystem.parameters !== undefined) { - - /* Base Asset and Initial Balances. */ - { - if (tradingSystem.parameters.baseAsset !== undefined) { - let code - try { - code = JSON.parse(tradingSystem.parameters.baseAsset.code); - - if (code.name !== undefined) { - bot.VALUES_TO_USE.baseAsset = code.name; - } - - if (baseAsset === 'BTC') { // NOTE: POLONIEX, the only exchange working so far, has Asset A and B inverted. We need to fix this. - if (code.initialBalance !== undefined) { - bot.VALUES_TO_USE.initialBalanceB = code.initialBalance; - bot.VALUES_TO_USE.initialBalanceA = 0 - } - if (code.minimumBalance !== undefined) { - bot.VALUES_TO_USE.minimumBalanceA = code.minimumBalance; - bot.VALUES_TO_USE.minimumBalanceB = 0 - } - if (code.maximumBalance !== undefined) { - bot.VALUES_TO_USE.maximumBalanceA = code.maximumBalance; - bot.VALUES_TO_USE.maximumBalanceB = 0 - } - } else { - if (code.initialBalance !== undefined) { - bot.VALUES_TO_USE.initialBalanceA = code.initialBalance; - bot.VALUES_TO_USE.initialBalanceB = 0 - } - if (code.minimumBalance !== undefined) { - bot.VALUES_TO_USE.minimumBalanceB = code.minimumBalance; - bot.VALUES_TO_USE.minimumBalanceA = 0 - } - if (code.maximumBalance !== undefined) { - bot.VALUES_TO_USE.maximumBalanceB = code.maximumBalance; - bot.VALUES_TO_USE.maximumBalanceA = 0 - } - } - } catch (err) { - tradingSystem.parameters.baseAsset.error = err.message - } - } - } - - /* Time Period */ - if (tradingSystem.parameters.timePeriod !== undefined) { - bot.VALUES_TO_USE.timePeriod = tradingSystem.parameters.timePeriod.code - } - - /* Slippage */ - if (tradingSystem.parameters.slippage !== undefined) { - if (tradingSystem.parameters.slippage.code !== undefined) { - try { - let code = JSON.parse(tradingSystem.parameters.slippage.code) - - if (code.positionRate !== undefined) { - bot.VALUES_TO_USE.slippage.positionRate = code.positionRate - } - if (code.stopLoss !== undefined) { - bot.VALUES_TO_USE.slippage.stopLoss = code.stopLoss - } - if (code.takeProfit !== undefined) { - bot.VALUES_TO_USE.slippage.takeProfit = code.takeProfit - } - - } catch (err) { - tradingSystem.parameters.slippage.error = err.message - } - } - } - - /* Fee Structure */ - if (tradingSystem.parameters.feeStructure !== undefined) { - if (tradingSystem.parameters.feeStructure.code !== undefined) { - try { - let code = JSON.parse(tradingSystem.parameters.feeStructure.code) - - if (code.maker !== undefined) { - bot.VALUES_TO_USE.feeStructure.maker = code.maker - } - if (code.taker !== undefined) { - bot.VALUES_TO_USE.feeStructure.taker = code.taker - } - - } catch (err) { - tradingSystem.parameters.feeStructure.error = err.message - } - } - } - - /* Time Range */ - if (tradingSystem.parameters.timeRange !== undefined) { - if (tradingSystem.parameters.timeRange.code !== undefined) { - try { - let code = JSON.parse(tradingSystem.parameters.timeRange.code) - if (code.initialDatetime !== undefined) { - bot.VALUES_TO_USE.timeRange.initialDatetime = new Date(code.initialDatetime) - } - if (code.finalDatetime !== undefined) { - bot.VALUES_TO_USE.timeRange.finalDatetime = new Date(code.finalDatetime) - } - } catch (err) { - tradingSystem.parameters.timeRange.error = err.message - } - } - } - } - - /* Applying Session Level Parameters */ - if (bot.SESSION !== undefined) { - if (bot.SESSION.parameters !== undefined) { - - /* Time Period */ - if (bot.SESSION.parameters.timePeriod !== undefined) { - bot.VALUES_TO_USE.timePeriod = bot.SESSION.parameters.timePeriod.code - } - - /* Slippage */ - if (bot.SESSION.parameters.slippage !== undefined) { - if (bot.SESSION.parameters.slippage.code !== undefined) { - try { - let code = JSON.parse(bot.SESSION.parameters.slippage.code) - - if (code.positionRate !== undefined) { - bot.VALUES_TO_USE.slippage.positionRate = code.positionRate - } - if (code.stopLoss !== undefined) { - bot.VALUES_TO_USE.slippage.stopLoss = code.stopLoss - } - if (code.takeProfit !== undefined) { - bot.VALUES_TO_USE.slippage.takeProfit = code.takeProfit - } - - } catch (err) { - parentLogger.write(MODULE_NAME, "[WARN] run -> startLiveTrading -> Invalid Slippage Value -> err = " + err.stack); - } - } - } - - /* Fee Structure */ - if (bot.SESSION.parameters.feeStructure !== undefined) { - if (bot.SESSION.parameters.feeStructure.code !== undefined) { - try { - let code = JSON.parse(bot.SESSION.parameters.feeStructure.code) - - if (code.maker !== undefined) { - bot.VALUES_TO_USE.feeStructure.maker = code.maker - } - if (code.taker !== undefined) { - bot.VALUES_TO_USE.feeStructure.taker = code.taker - } - - } catch (err) { - parentLogger.write(MODULE_NAME, "[WARN] run -> startLiveTrading -> Invalid Fee Structure Value -> err = " + err.stack); - } - } - } - - /* Time Range */ - if (bot.SESSION.parameters.timeRange !== undefined) { - if (bot.SESSION.parameters.timeRange.code !== undefined) { - try { - let code = JSON.parse(bot.SESSION.parameters.timeRange.code) - if (code.initialDatetime !== undefined) { - bot.VALUES_TO_USE.timeRange.initialDatetime = new Date(code.initialDatetime) - } - if (code.finalDatetime !== undefined) { - bot.VALUES_TO_USE.timeRange.finalDatetime = new Date(code.finalDatetime) - } - } catch (err) { - parentLogger.write(MODULE_NAME, "[WARN] run -> startLiveTrading -> Invalid Time Range Value -> err = " + err.stack); - } - } - } - } - } - } - } + bot.STOP_SESSION = true; loop(); @@ -456,18 +151,18 @@ /* Checking if we need to need to emit any event */ - if (bot.sessionStatus === 'Idle' && global.STOP_SESSION === false) { + if (bot.SESSION_STATUS === 'Idle' && bot.STOP_SESSION === false) { global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.sessionKey, 'Running') - bot.sessionStatus = 'Running' + bot.SESSION_STATUS = 'Running' } - if (bot.sessionStatus === 'Running' && global.STOP_SESSION === true) { + if (bot.SESSION_STATUS === 'Running' && bot.STOP_SESSION === true) { global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.sessionKey, 'Stopped') - bot.sessionStatus = 'Stopped' + bot.SESSION_STATUS = 'Stopped' } /* Checking if we should process this loop or not.*/ - if (global.STOP_SESSION === true) { + if (bot.STOP_SESSION === true) { if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Waiting for " + bot.processNode.session.type + " " + bot.processNode.session.name + " to be ran."); } diff --git a/server.js b/server.js index 80441ad81..62870d075 100644 --- a/server.js +++ b/server.js @@ -106,7 +106,6 @@ process.on('message', message => { if (message === 'Stop this Task') { global.STOP_TASK_GRACEFULLY = true; - global.STOP_SESSION = true /* There are some process that might no be able to end grafully, for example the ones schedulle to process information in a future day or month. From 0617c83207e5c6fffe8469b2b0cde6a247d70d3d Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 4 Oct 2019 18:40:43 +0200 Subject: [PATCH 13/19] Stopping Session when task stops --- TradingEngineProcessMainLoop.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index ed8f4b9d4..3a88751b6 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -146,8 +146,8 @@ /* High level log entry */ - console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) - + " Entered into Main Loop # " + pad(Number(bot.loopCounter), 8)); + console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " " + bot.sessionKey + + " Loop # " + pad(Number(bot.loopCounter), 8)); /* Checking if we need to need to emit any event */ @@ -1052,6 +1052,10 @@ seconds: (new Date()).getSeconds() } global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.sessionKey, 'Heartbeat', event) + + if (global.STOP_TASK_GRACEFULLY === true) { + bot.STOP_SESSION = true + } } } catch (err) { From 92883afa17a0ebd9b058a9d119adfd9f046af045 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 4 Oct 2019 19:43:34 +0200 Subject: [PATCH 14/19] Bug Fixed with Initial Balance --- Context.js | 4 ++-- Session.js | 22 +++++++++++----------- server.js | 5 +++-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Context.js b/Context.js index 64c11744c..6765c725a 100644 --- a/Context.js +++ b/Context.js @@ -306,8 +306,8 @@ if (bot.VALUES_TO_USE.initialBalanceA === undefined || bot.VALUES_TO_USE.initialBalanceB === undefined) { throw new Error("Global Variables INITIAL_BALANCE_A and INITIAL_BALANCE_B cannot be undefined.") } - const INITIAL_BALANCE_A = Number(bot.VALUES_TO_USE.initialBalanceA); - const INITIAL_BALANCE_B = Number(bot.VALUES_TO_USE.initialBalanceB); + const INITIAL_BALANCE_A = Number(bot.VALUES_TO_USE.initialBalanceB); // NOTE THAT THIS IS INVERTED BECAUSE OF POLONIEX THAT IS THE ONLY EXCHANGE SUPPORTED RIGHT NOW + const INITIAL_BALANCE_B = Number(bot.VALUES_TO_USE.initialBalanceA); thisObject.executionContext = { diff --git a/Session.js b/Session.js index fd319381a..135db5723 100644 --- a/Session.js +++ b/Session.js @@ -102,12 +102,12 @@ /* Set all default values */ bot.VALUES_TO_USE = { baseAsset: "BTC", - initialBalanceA: 0, - initialBalanceB: 0.001, - minimumBalanceA: 0, - minimumBalanceB: 0.0005, - maximumBalanceA: 0, - maximumBalanceB: 0.002, + initialBalanceA: 0.001, + initialBalanceB: 0, + minimumBalanceA: 0.0005, + minimumBalanceB: 0, + maximumBalanceA: 0.002, + maximumBalanceB: 0, timePeriod: bot.DEFINITION.uiCurrentValues.timePeriod, slippage: { positionRate: 0, @@ -157,10 +157,10 @@ bot.VALUES_TO_USE.baseAsset = code.name; } - if (baseAsset === 'BTC') { // NOTE: POLONIEX, the only exchange working so far, has Asset A and B inverted. We need to fix this. + if (baseAsset === 'BTC') { if (code.initialBalance !== undefined) { - bot.VALUES_TO_USE.initialBalanceB = code.initialBalance; - bot.VALUES_TO_USE.initialBalanceA = 0 + bot.VALUES_TO_USE.initialBalanceA = code.initialBalance; + bot.VALUES_TO_USE.initialBalanceB = 0 } if (code.minimumBalance !== undefined) { bot.VALUES_TO_USE.minimumBalanceA = code.minimumBalance; @@ -172,8 +172,8 @@ } } else { if (code.initialBalance !== undefined) { - bot.VALUES_TO_USE.initialBalanceA = code.initialBalance; - bot.VALUES_TO_USE.initialBalanceB = 0 + bot.VALUES_TO_USE.initialBalanceB = code.initialBalance; + bot.VALUES_TO_USE.initialBalanceA = 0 } if (code.minimumBalance !== undefined) { bot.VALUES_TO_USE.minimumBalanceB = code.minimumBalance; diff --git a/server.js b/server.js index 62870d075..3261fddf1 100644 --- a/server.js +++ b/server.js @@ -62,16 +62,17 @@ of this Task and know exactly what to run within this server instance. global.TASK_NODE = process.argv[2] if (global.TASK_NODE !== undefined) { - console.log('[INFO] Task Server -> server -> global.TASK_NODE = ' + global.TASK_NODE) + try { global.TASK_NODE = JSON.parse(global.TASK_NODE) + console.log('[INFO] Task Server -> server -> global.TASK_NODE = ' + JSON.stringify(global.TASK_NODE)) } catch (err) { console.log('[ERROR] Task Server -> server -> global.TASK_NODE -> ' + err.stack) } } else { // I use this section to debug in standalone mode. - let argument = '{"type":"Task","name":"Runs Backtests, Fordwardtests & Live Trades ","bot":{"type":"Trading Engine","processes":[{"type":"Process","subType":"Trading Engine Process","name":"Multi Period","code":{"team":"AAMasters","bot":"AAJason","process":"Multi-Period","repo":"AAJason-Trading-Engine-Bot"},"session":{"type":"Backtesting Session","name":"New Backtesting Session","parameters":{"type":"Parameters","name":"Parameters","baseAsset":{"type":"Base Asset","name":"Base Asset","code":"{ \n\"name\": \"BTC\",\n\"initialBalance\": 0.001,\n\"minimumBalance\": 0.0001,\n\"maximumBalance\": 1\n}"},"timeRange":{"type":"Time Range","name":"Time Range","code":"{\n\"initialDatetime\": \"2019-09-15T00:00:00.000Z\",\n\"finalDatetime\": \"2019-09-30T00:00:00.000Z\"\n}"},"timePeriod":{"type":"Time Period","name":"Time Period","code":"01-hs"},"slippage":{"type":"Slippage","name":"Slippage","code":"{\n\"positionRate\": 0.1,\n\"stopLoss\": 0.1,\n\"takeProfit\": 0.1\n}"},"feeStructure":{"type":"Fee Structure","name":"Fee Structure","code":"{\n\"maker\": 0.15,\n\"taker\": 0.25\n}"}},"id":"1381b048-7795-4e60-84a4-cf02e7385ba1"},"id":"072065fc-a57c-439c-94cf-f3f9c901ed37"}]},"id":"cddc97dd-ca2d-4b14-95ce-f15b6149996f"}' + let argument = ' {"type":"Task","name":"Runs Backtests, Fordwardtests & Live Trades ","bot":{"type":"Trading Engine","processes":[{"type":"Process","subType":"Trading Engine Process","name":"Multi Period","code":{"team":"AAMasters","bot":"AAJason","process":"Multi-Period","repo":"AAJason-Trading-Engine-Bot"},"session":{"type":"Backtesting Session","name":"1 Hour","id":"1381b048-7795-4e60-84a4-cf02e7385ba1"},"id":"072065fc-a57c-439c-94cf-f3f9c901ed37"}]},"id":"cddc97dd-ca2d-4b14-95ce-f15b6149996f"}' /* charly argument = '{"type":"Task","name":"Brings Trades Records from the Exchange","bot":{"type":"Sensor","name":"Charly","processes":[{"type":"Process","name":"Live Trades","code":{"team":"AAMasters","bot":"AACharly","process":"Live-Trades"},"id":"5846ebc7-1979-4a80-9cc7-94bb4b8659dc"},{"type":"Process","name":"Hole Fixing","code":{"team":"AAMasters","bot":"AACharly","process":"Hole-Fixing"},"id":"31fc4f05-75d4-419a-9fb0-73e25e856f15"}]},"id":"5bfef4dc-54c1-44db-ace6-1ab27a86746e"}' // olivia argument = '{"type":"Task","name":"Generates 1 min to 24 hs Candles & Volumes","bot":{"type":"Indicator","name":"Olivia","processes":[{"type":"Process","name":"Daily","code":{"team":"AAMasters","bot":"AAOlivia","process":"Multi-Period-Daily"},"id":"68cc8e1b-e94a-477e-82d0-15ecc9f1b9e2"},{"type":"Process","name":"Market","code":{"team":"AAMasters","bot":"AAOlivia","process":"Multi-Period-Market"},"id":"e1ac5e3d-d491-4e90-baf4-d4e5b71a8b1a"}]},"id":"efe36e05-75ea-41b5-8d92-c6b27635c834"}' // bruce argument = ' {"type":"Task","name":"Converts Trades into 1 min Candles & Volumes","bot":{"type":"Indicator","name":"Bruce","processes":[{"type":"Process","name":"New Process","code":{"team":"AAMasters","bot":"AABruce","process":"Single-Period-Daily"},"id":"e184744f-5de0-41c1-ad4c-36960f4ced90"}]},"id":"42758590-a7bd-4712-a5c9-80a3a820c5b3"}' From 0b999ed1fbb4182691b53eb4f2da012d55b1ab06 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 4 Oct 2019 21:21:13 +0200 Subject: [PATCH 15/19] sending processing date --- TradingEngineProcessMainLoop.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 3a88751b6..43eb7bacf 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -1047,9 +1047,10 @@ global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) } - function sessionHeartBeat() { + function sessionHeartBeat(processingDate) { let event = { - seconds: (new Date()).getSeconds() + seconds: (new Date()).getSeconds(), + processingDate: processingDate } global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.sessionKey, 'Heartbeat', event) From a0843d22812c04a4590d792735a59754b8a90d8c Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 5 Oct 2019 10:26:09 +0200 Subject: [PATCH 16/19] Multi Session Logs + Improvements in handlling Fails --- DebugLog.js | 8 +++++++- IndicatorBotProcessMainLoop.js | 13 ++++++++++++- SensorBotProcessMainLoop.js | 9 +++++++++ Session.js | 5 +++++ TradingEngineProcessMainLoop.js | 29 ++++++++++++++++++++++++++++- server.js | 12 ++++++------ 6 files changed, 67 insertions(+), 9 deletions(-) diff --git a/DebugLog.js b/DebugLog.js index 07da825e5..d2f34a057 100644 --- a/DebugLog.js +++ b/DebugLog.js @@ -66,7 +66,13 @@ exports.newDebugLog = function newDebugLog() { const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(); - let filePath = thisObject.bot.filePathRoot + "/Logs/" + thisObject.bot.process + "/" + executionDatetime; + let filePath = thisObject.bot.filePathRoot + "/Logs/" + thisObject.bot.process + "/" + + if (thisObject.bot.SESSION !== undefined) { + filePath = filePath + thisObject.bot.SESSION.name + "/" + executionDatetime; + } else { + filePath = filePath + executionDatetime; + } if (thisObject.bot.debug.year !== undefined) { diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 6eda2730c..717729ecb 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -178,8 +178,8 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Operation Failed. Aborting the process."); - logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -282,6 +282,7 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -356,6 +357,7 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -456,6 +458,7 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -585,6 +588,7 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -687,6 +691,7 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -932,6 +937,12 @@ } global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) } + + function processStopped() { + let key = global.TASK_NODE.bot.processes[bot.processIndex].name + '-' + global.TASK_NODE.bot.processes[bot.processIndex].type + '-' + global.TASK_NODE.bot.processes[bot.processIndex].id + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Stopped') + } + } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] run -> err = "+ err.stack); clearInterval(fixedTimeLoopIntervalHandle); diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index 97b205f8d..d3f18e837 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -178,6 +178,7 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -252,6 +253,7 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeExchangeAPI -> onInizialized -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -326,6 +328,7 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeUserBot -> onInizialized > Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -434,6 +437,7 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> startUserBot -> onFinished > Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -705,6 +709,11 @@ } global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) } + + function processStopped() { + let key = global.TASK_NODE.bot.processes[bot.processIndex].name + '-' + global.TASK_NODE.bot.processes[bot.processIndex].type + '-' + global.TASK_NODE.bot.processes[bot.processIndex].id + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Stopped') + } } catch (err) { diff --git a/Session.js b/Session.js index 135db5723..62f51a569 100644 --- a/Session.js +++ b/Session.js @@ -13,6 +13,11 @@ try { if (FULL_LOG === true) { parentLogger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); } + /* Initialize this info so that everything is logged propeerly */ + bot.SESSION = { + name: bot.processNode.session.name + } + /* Check if there is a session */ if (bot.processNode.session === undefined) { parentLogger.write(MODULE_NAME, "[ERROR] initialize -> Cannot run without a Session."); diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index 43eb7bacf..f77c01b7a 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -212,8 +212,9 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + sessionStopped() + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeStatusDependencies -> onInizialized -> Operation Failed. Aborting the process."); - logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); clearTimeout(nextLoopTimeoutHandle); @@ -287,6 +288,8 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + sessionStopped() + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeDataDependencies -> onInizialized -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -360,6 +363,8 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + sessionStopped() + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeContext -> onInizialized -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -434,6 +439,8 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + sessionStopped() + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeExchangeAPI -> onInizialized -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -541,6 +548,8 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + sessionStopped() + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> initializeAssistant -> onInizialized -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -613,6 +622,8 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + sessionStopped() + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> intitializeProcessFramework -> onInizialized -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -715,6 +726,8 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + sessionStopped() + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> startProcessFramework -> onFinished -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -849,6 +862,8 @@ return; } case global.DEFAULT_FAIL_RESPONSE.result: { // This is an unexpected exception that we do not know how to handle. + sessionStopped() + processStopped() logger.write(MODULE_NAME, "[ERROR] run -> loop -> saveContext -> onFinished -> Operation Failed. Aborting the process."); logger.persist(); clearInterval(fixedTimeLoopIntervalHandle); @@ -1059,6 +1074,18 @@ } } + function processStopped() { + let key = global.TASK_NODE.bot.processes[bot.processIndex].name + '-' + global.TASK_NODE.bot.processes[bot.processIndex].type + '-' + global.TASK_NODE.bot.processes[bot.processIndex].id + global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Stopped') + } + + function sessionStopped() { + if (bot.SESSION_STATUS === 'Running') { + global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.sessionKey, 'Stopped') + bot.SESSION_STATUS = 'Stopped' + } + } + } catch (err) { parentLogger.write(MODULE_NAME, "[ERROR] run -> err = "+ err.stack); clearInterval(fixedTimeLoopIntervalHandle); diff --git a/server.js b/server.js index 3261fddf1..87cb567bd 100644 --- a/server.js +++ b/server.js @@ -28,14 +28,14 @@ global.CUSTOM_FAIL_RESPONSE = { /* Process Events */ process.on('uncaughtException', function (err) { - console.log('[INFO] Task Server -> server -> uncaughtException -> err.message = ' + err.message) - console.log('[INFO] Task Server -> server -> uncaughtException -> err.stack = ' + err.stack) + console.log('[ERROR] Task Server -> server -> uncaughtException -> err.message = ' + err.message) + console.log('[ERROR] Task Server -> server -> uncaughtException -> err.stack = ' + err.stack) process.exit(1) }) process.on('unhandledRejection', (reason, p) => { - console.log('[INFO] Task Server -> server -> unhandledRejection -> reason = ' + JSON.stringify(reason)) - console.log('[INFO] Task Server -> server -> unhandledRejection -> p = ' + JSON.stringify(p)) + console.log('[ERROR] Task Server -> server -> unhandledRejection -> reason = ' + JSON.stringify(reason)) + console.log('[ERROR] Task Server -> server -> unhandledRejection -> p = ' + JSON.stringify(p)) process.exit(1) }) @@ -158,9 +158,9 @@ function bootLoader() { global.SYSTEM_EVENT_HANDLER.createEventHandler(key) global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Running') // Meaning Task Running - global.HEARTBEAT_INTERVAL_HANDLER = setInterval(hearBeat, 1000) + global.HEARTBEAT_INTERVAL_HANDLER = setInterval(taskHearBeat, 1000) - function hearBeat() { + function taskHearBeat() { /* The heartbeat event is raised at the event handler of the instance of this task, created at the UI. */ let event = { From 606b183257abbe03666b782c69631afff4c9b3ec Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 5 Oct 2019 14:30:03 +0200 Subject: [PATCH 17/19] Fixed Bug when Reports are missing --- IndicatorBotProcessMainLoop.js | 13 +++++-------- MultiPeriodDaily.js | 25 ++++++++++++++++++++++--- Root.js | 4 ++++ SensorBotProcessMainLoop.js | 13 +++++-------- TradingEngineProcessMainLoop.js | 7 ++----- 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 717729ecb..5d742c75e 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -119,7 +119,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Entering function."); } /* Loop Heartbeat sent to the UI */ - hearBeat() + processHearBeat() /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ @@ -801,7 +801,7 @@ /* We show we reached the end of the loop. */ - hearBeat() + processHearBeat() /* Here we check if we must stop the loop gracefully. */ @@ -929,18 +929,15 @@ } } - function hearBeat() { - let key = global.TASK_NODE.bot.processes[bot.processIndex].name + '-' + global.TASK_NODE.bot.processes[bot.processIndex].type + '-' + global.TASK_NODE.bot.processes[bot.processIndex].id - + function processHearBeat() { let event = { seconds: (new Date()).getSeconds() } - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) + global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.processKey, 'Heartbeat', event) } function processStopped() { - let key = global.TASK_NODE.bot.processes[bot.processIndex].name + '-' + global.TASK_NODE.bot.processes[bot.processIndex].type + '-' + global.TASK_NODE.bot.processes[bot.processIndex].id - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Stopped') + global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.processKey, 'Stopped') } } catch (err) { diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index 7b4d9f14d..f7732a42b 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -24,6 +24,8 @@ const FILE_STORAGE = require('./FileStorage.js'); let fileStorage = FILE_STORAGE.newFileStorage(logger); + let bootstrappingTheProcess = false + let processConfig; return thisObject; @@ -268,6 +270,13 @@ */ interExecutionMemoryArray = []; + /* + Also, we will remember that we are bootstrapping the process, this will allow us, to advanceTime if necesary. + It can happen that a process depends on data from another process that does not produce a file during the first day, or even more days, + in that situation we want the process not to just wait for that data that will never arrive, but to advance time until it finds valid data. + */ + + bootstrappingTheProcess = true for (let i = 0; i < global.dailyFilePeriods.length; i++) { let emptyObject = {}; @@ -449,7 +458,7 @@ if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getPreviousFile -> onFileReceived -> err = "+ err.stack); callBackFunction(global.DEFAULT_RETRY_RESPONSE); return; } @@ -504,6 +513,16 @@ if ((err.result === "Fail Because" && err.message === "File does not exist.") || err.code === 'The specified key does not exist.') { + if (bootstrappingTheProcess === true) { + /* + In the special situation where we are running this process for the first time (no status report found), + we will consider the possibility that a depanant process does not produce a file during the first day of the market, + or even more than one day. That is why we are going to advance time here. + */ + advanceTime() + return + } + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> filePath = " + filePath); logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> fileName = " + fileName); @@ -513,7 +532,7 @@ if (err.result !== global.DEFAULT_OK_RESPONSE.result) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = " + err.code); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Not OK -> onFileReceived -> err = " + err.code); callBackFunction(err); return; } @@ -527,7 +546,7 @@ } catch (err) { - logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> onFileReceived -> err = "+ err.stack); + logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Catch Error -> onFileReceived -> err = "+ err.stack); callBackFunction(global.DEFAULT_FAIL_RESPONSE); } } diff --git a/Root.js b/Root.js index cc459e242..a6c16f8bf 100644 --- a/Root.js +++ b/Root.js @@ -184,6 +184,10 @@ botConfig.filePathRoot = botConfig.devTeam + "/" + botConfig.codeName + "." + botConfig.version.major + "." + botConfig.version.minor + "/" + global.CLONE_EXECUTOR.codeName + "." + global.CLONE_EXECUTOR.version + "/" + global.EXCHANGE_NAME + "/" + botConfig.dataSetVersion; + /* Process Key */ + + botConfig.processKey = global.TASK_NODE.bot.processes[processIndex].name + '-' + global.TASK_NODE.bot.processes[processIndex].type + '-' + global.TASK_NODE.bot.processes[processIndex].id + if (FULL_LOG === true) { console.log(logDisplace + "Root : [INFO] start -> findProcess -> filePathRoot = " + botConfig.filePathRoot); } /* Now we loop throug all the configured processes at each bots configuration until we find the one we are supposed to run at this Node.js process. */ diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index d3f18e837..b37e8a9ee 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -121,7 +121,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Entering function."); } /* Loop Heartbeat sent to the UI */ - hearBeat() + processHearBeat() /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ @@ -547,7 +547,7 @@ /* We show we reached the end of the loop. */ - hearBeat() + processHearBeat() /* Here we check if we must stop the loop gracefully. */ @@ -701,18 +701,15 @@ } } - function hearBeat() { - let key = global.TASK_NODE.bot.processes[bot.processIndex].name + '-' + global.TASK_NODE.bot.processes[bot.processIndex].type + '-' + global.TASK_NODE.bot.processes[bot.processIndex].id - + function processHearBeat() { let event = { seconds: (new Date()).getSeconds() } - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) + global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.processKey, 'Heartbeat', event) } function processStopped() { - let key = global.TASK_NODE.bot.processes[bot.processIndex].name + '-' + global.TASK_NODE.bot.processes[bot.processIndex].type + '-' + global.TASK_NODE.bot.processes[bot.processIndex].id - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Stopped') + global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.processKey, 'Stopped') } } diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index f77c01b7a..d6d7a6b8c 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -1054,12 +1054,10 @@ } function processHeartBeat() { - let key = global.TASK_NODE.bot.processes[bot.processIndex].name + '-' + global.TASK_NODE.bot.processes[bot.processIndex].type + '-' + global.TASK_NODE.bot.processes[bot.processIndex].id - let event = { seconds: (new Date()).getSeconds() } - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Heartbeat', event) + global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.processKey, 'Heartbeat', event) } function sessionHeartBeat(processingDate) { @@ -1075,8 +1073,7 @@ } function processStopped() { - let key = global.TASK_NODE.bot.processes[bot.processIndex].name + '-' + global.TASK_NODE.bot.processes[bot.processIndex].type + '-' + global.TASK_NODE.bot.processes[bot.processIndex].id - global.SYSTEM_EVENT_HANDLER.raiseEvent(key, 'Stopped') + global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.processKey, 'Stopped') } function sessionStopped() { From b945ecdfbbdf4780f085dba5eb5775ff0d9a7f2d Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 5 Oct 2019 14:51:46 +0200 Subject: [PATCH 18/19] Dates in process heartbeat --- IndicatorBotProcessMainLoop.js | 10 ++++++---- MultiPeriod.js | 4 ++++ MultiPeriodDaily.js | 26 +++++++++++++++----------- SensorBotProcessMainLoop.js | 10 ++++++---- Session.js | 4 ++-- TradingEngineProcessMainLoop.js | 6 ++++-- 6 files changed, 37 insertions(+), 23 deletions(-) diff --git a/IndicatorBotProcessMainLoop.js b/IndicatorBotProcessMainLoop.js index 5d742c75e..dd451faec 100644 --- a/IndicatorBotProcessMainLoop.js +++ b/IndicatorBotProcessMainLoop.js @@ -119,7 +119,8 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Entering function."); } /* Loop Heartbeat sent to the UI */ - processHearBeat() + bot.processHeartBeat = processHeartBeat + processHeartBeat() /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ @@ -801,7 +802,7 @@ /* We show we reached the end of the loop. */ - processHearBeat() + processHeartBeat() /* Here we check if we must stop the loop gracefully. */ @@ -929,9 +930,10 @@ } } - function processHearBeat() { + function processHeartBeat(processingDate) { let event = { - seconds: (new Date()).getSeconds() + seconds: (new Date()).getSeconds(), + processingDate: processingDate } global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.processKey, 'Heartbeat', event) } diff --git a/MultiPeriod.js b/MultiPeriod.js index d9ccbfac2..42aed721b 100644 --- a/MultiPeriod.js +++ b/MultiPeriod.js @@ -542,6 +542,10 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriodsDailyFiles -> periodsLoop -> Entering function."); } + /* Telling the world we are alive and doing well */ + let processingDate = bot.multiPeriodDailyProcessDatetime.getUTCFullYear() + '-' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCMonth() + 1, 2) + '-' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCDate(), 2); + bot.processHeartBeat(processingDate) + /* We will iterate through all posible periods. diff --git a/MultiPeriodDaily.js b/MultiPeriodDaily.js index f7732a42b..0ca72c491 100644 --- a/MultiPeriodDaily.js +++ b/MultiPeriodDaily.js @@ -306,7 +306,7 @@ let n; let botNeverRan = true; - bot.multiPeriodDailyProcessDatetime = new Date(contextVariables.lastFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. + bot.multiPeriodProcessDatetime = new Date(contextVariables.lastFile.valueOf() - ONE_DAY_IN_MILISECONDS); // Go back one day to start well when we advance time at the begining of the loop. advanceTime(); @@ -316,21 +316,21 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> Entering function."); } - bot.multiPeriodDailyProcessDatetime = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() + ONE_DAY_IN_MILISECONDS); - previousDay = new Date(bot.multiPeriodDailyProcessDatetime.valueOf() - ONE_DAY_IN_MILISECONDS); + bot.multiPeriodProcessDatetime = new Date(bot.multiPeriodProcessDatetime.valueOf() + ONE_DAY_IN_MILISECONDS); + previousDay = new Date(bot.multiPeriodProcessDatetime.valueOf() - ONE_DAY_IN_MILISECONDS); - console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " " + "bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime.toISOString()); + console.log(new Date().toISOString() + " " + pad(bot.codeName, 20) + " " + pad(bot.process, 30) + " " + "bot.multiPeriodProcessDatetime = " + bot.multiPeriodProcessDatetime.toISOString()); if (global.WRITE_LOGS_TO_FILES === 'true') { logger.newInternalLoop(bot.codeName, bot.process); } - if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> bot.multiPeriodDailyProcessDatetime = " + bot.multiPeriodDailyProcessDatetime); } + if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> bot.multiPeriodProcessDatetime = " + bot.multiPeriodProcessDatetime); } if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> previousDay = " + previousDay); } /* Validation that we are not going past the head of the market. */ - if (bot.multiPeriodDailyProcessDatetime.valueOf() > contextVariables.dateEndOfMarket.valueOf()) { + if (bot.multiPeriodProcessDatetime.valueOf() > contextVariables.dateEndOfMarket.valueOf()) { const logText = "Head of the market found @ " + previousDay.getUTCFullYear() + "/" + (previousDay.getUTCMonth() + 1) + "/" + previousDay.getUTCDate() + "."; if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> advanceTime -> " + logText); } @@ -354,6 +354,10 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoop -> Entering function."); } + /* Telling the world we are alive and doing well */ + let processingDate = bot.multiPeriodProcessDatetime.getUTCFullYear() + '-' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCMonth() + 1, 2) + '-' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCDate(), 2); + bot.processHeartBeat(processingDate) + /* We will iterate through all posible periods. @@ -410,7 +414,7 @@ let previousFile; let currentFile; - if (bot.multiPeriodDailyProcessDatetime.valueOf() > contextVariables.dateBeginOfMarket.valueOf()) { + if (bot.multiPeriodProcessDatetime.valueOf() > contextVariables.dateBeginOfMarket.valueOf()) { getPreviousFile(); } else { previousFile = []; @@ -493,7 +497,7 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> processTimePeriods -> periodsLoopBody -> dependencyLoopBody -> getCurrentFile -> Entering function."); } - let dateForPath = bot.multiPeriodDailyProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodDailyProcessDatetime.getUTCDate(), 2); + let dateForPath = bot.multiPeriodProcessDatetime.getUTCFullYear() + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.multiPeriodProcessDatetime.getUTCDate(), 2); let filePath if (dependency.dataSet === "Multi-Period-Daily") { filePath = dependency.product + '/' + dependency.dataSet + "/" + outputPeriodLabel + "/" + dateForPath; @@ -600,7 +604,7 @@ dataFiles, timePeriod, outputPeriodLabel, - bot.multiPeriodDailyProcessDatetime, + bot.multiPeriodProcessDatetime, contextVariables.dateBeginOfMarket, contextVariables.dateEndOfMarket, interExecutionMemoryArray[n], @@ -669,7 +673,7 @@ return; } - writeStatusReport(bot.multiPeriodDailyProcessDatetime, advanceTime); + writeStatusReport(bot.multiPeriodProcessDatetime, advanceTime); } catch (err) { logger.write(MODULE_NAME, "[ERROR] start -> processTimePeriods -> controlLoop -> onWritten -> err = "+ err.stack); @@ -704,7 +708,7 @@ let folderName = bot.products[productIndex].codeName; - writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodDailyProcessDatetime, folderName, controlLoop); + writeDataRange(contextVariables.dateBeginOfMarket, bot.multiPeriodProcessDatetime, folderName, controlLoop); } function controlLoop() { diff --git a/SensorBotProcessMainLoop.js b/SensorBotProcessMainLoop.js index b37e8a9ee..3d20a878d 100644 --- a/SensorBotProcessMainLoop.js +++ b/SensorBotProcessMainLoop.js @@ -121,7 +121,8 @@ if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] run -> loop -> Entering function."); } /* Loop Heartbeat sent to the UI */ - processHearBeat() + bot.processHeartBeat = processHeartBeat + processHeartBeat() /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ @@ -547,7 +548,7 @@ /* We show we reached the end of the loop. */ - processHearBeat() + processHeartBeat() /* Here we check if we must stop the loop gracefully. */ @@ -701,9 +702,10 @@ } } - function processHearBeat() { + function processHeartBeat(processingDate) { let event = { - seconds: (new Date()).getSeconds() + seconds: (new Date()).getSeconds(), + processingDate: processingDate } global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.processKey, 'Heartbeat', event) } diff --git a/Session.js b/Session.js index 62f51a569..07e526339 100644 --- a/Session.js +++ b/Session.js @@ -61,7 +61,7 @@ bot.startMode = "Backtest" bot.resumeExecution = false; bot.hasTheBotJustStarted = true - bot.multiPeriodDailyProcessDatetime = bot.VALUES_TO_USE.timeRange.initialDatetime + bot.multiPeriodProcessDatetime = bot.VALUES_TO_USE.timeRange.initialDatetime } function startLiveTrading(message) { @@ -93,7 +93,7 @@ bot.VALUES_TO_USE.timeRange.initialDatetime = new Date() } bot.resumeExecution = false; - bot.multiPeriodDailyProcessDatetime = bot.VALUES_TO_USE.timeRange.initialDatetime + bot.multiPeriodProcessDatetime = bot.VALUES_TO_USE.timeRange.initialDatetime bot.hasTheBotJustStarted = true } diff --git a/TradingEngineProcessMainLoop.js b/TradingEngineProcessMainLoop.js index d6d7a6b8c..efc4c8143 100644 --- a/TradingEngineProcessMainLoop.js +++ b/TradingEngineProcessMainLoop.js @@ -127,6 +127,7 @@ /* Heartbeats sent to the UI */ bot.sessionHeartBeat = sessionHeartBeat + bot.processHeartBeat = processHeartBeat processHeartBeat() /* We define here all the modules that the rest of the infraestructure, including the bots themselves can consume. */ @@ -1053,9 +1054,10 @@ } } - function processHeartBeat() { + function processHeartBeat(processingDate) { let event = { - seconds: (new Date()).getSeconds() + seconds: (new Date()).getSeconds(), + processingDate: processingDate } global.SYSTEM_EVENT_HANDLER.raiseEvent(bot.processKey, 'Heartbeat', event) } From 4d3c9ce4abda94b7d7e773eb5d68ad73bf11a454 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 6 Oct 2019 11:39:52 +0200 Subject: [PATCH 19/19] Tyding up --- Session.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Session.js b/Session.js index 07e526339..9236dc6e3 100644 --- a/Session.js +++ b/Session.js @@ -36,6 +36,7 @@ /* We are going to run the Definition comming at the event. */ bot.DEFINITION = JSON.parse(message.event.definition) bot.SESSION = JSON.parse(message.event.session) + bot.UI_CURRENT_VALUES = message.event.uiCurrentValues setValuesToUse(message) @@ -113,7 +114,7 @@ minimumBalanceB: 0, maximumBalanceA: 0.002, maximumBalanceB: 0, - timePeriod: bot.DEFINITION.uiCurrentValues.timePeriod, + timePeriod: bot.UI_CURRENT_VALUES.timePeriod, slippage: { positionRate: 0, stopLoss: 0, @@ -133,7 +134,7 @@ const ONE_YEAR_IN_MILISECONDS = 365 * 24 * 60 * 60 * 1000 switch (bot.SESSION.type) { case 'Backtesting Session': { - bot.VALUES_TO_USE.timeRange.initialDatetime = new Date(bot.DEFINITION.uiCurrentValues.initialDatetime) + bot.VALUES_TO_USE.timeRange.initialDatetime = new Date(bot.UI_CURRENT_VALUES.initialDatetime) bot.VALUES_TO_USE.timeRange.finalDatetime = new Date() break }