From 9d24120b9b4c38bfee9ede94c65e3a53670fe73b Mon Sep 17 00:00:00 2001 From: Ferrox85 <64791738+Ferrox85@users.noreply.github.com> Date: Tue, 14 Sep 2021 21:21:37 +0200 Subject: [PATCH] Changes to CSV Snapshot and Filestorage foor CSV handling Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/Snapshots.js changes on this file: adds only the first run a header to the to be saved content next runs it fetches the CSV file adds the new content to it saves the file this way you have a snapshot of all trades for referencing Projects/Foundations/TS/Task-Modules/FileStorage.js changes on this file: -skips JSON validation (because it is no json file) and returns the text --- .../Low-Frequency-Trading/Snapshots.js | 219 +++++++++++------- .../TS/Task-Modules/FileStorage.js | 32 ++- 2 files changed, 159 insertions(+), 92 deletions(-) diff --git a/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/Snapshots.js b/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/Snapshots.js index 35f3bf21b..4de4e1bb5 100644 --- a/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/Snapshots.js +++ b/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/Snapshots.js @@ -1,7 +1,7 @@ -exports.newFoundationsBotModulesSnapshots = function (processIndex) { +exports.newFoundationsBotModulesSnapshots = function(processIndex) { /* - This module encapsulates the snapshots functionality. - */ + This module encapsulates the snapshots functionality. + */ const MODULE_NAME = 'Snapshots' let thisObject = { @@ -28,20 +28,21 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) { positions: [] } - let positionKeys = new Map() // This is to prevent adding the same indicator / variable to the output collection. + let positionKeys = new Map() // This is to prevent adding the same indicator / variable to the output collection. let strategyKeys = new Map() - let createHeaders = true // This is going to be true only once - let createCloseHeaders = true // This is going to be true only once + let createHeaders = true // This is going to be true only once + let createCloseHeaders = true // This is going to be true only once let closeValues let strategyValues let addToStrategyValues let positionValues let addToPositionValues + let snapshotAvailable = false /* - These 3 are the main data structures available to users - when writing conditions and formulas. - */ + These 3 are the main data structures available to users + when writing conditions and formulas. + */ let chart let exchange let market @@ -52,6 +53,12 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) { tradingEngine = TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.tradingEngine tradingSystem = TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.tradingSystem sessionParameters = TS.projects.foundations.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters + // first trade forces overwrite of the existing file + if (tradingEngine.tradingCurrent.tradingEpisode.tradingEpisodeCounters.positions.value > 1) { + snapshotAvailable = true + createHeaders = false // This is going to be true only once + createCloseHeaders = false + } } function finalize() { @@ -70,9 +77,9 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) { function updateChart(pChart, pExchange, pMarket) { /* - We need these 3 data structures to be a local objects - accessible while evaluating conditions and formulas. - */ + We need these 3 data structures to be a local objects + accessible while evaluating conditions and formulas. + */ chart = pChart exchange = pExchange market = pMarket @@ -80,9 +87,9 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) { function strategyEntry() { /* - We will record here a snapshot of all values used in conditions and formulas within the Trading System. - If the strategy exists with a trade, this information will be added to the snapshots file. - */ + We will record here a snapshot of all values used in conditions and formulas within the Trading System. + If the strategy exists with a trade, this information will be added to the snapshots file. + */ strategyKeys = new Map() strategyValues = [] @@ -147,24 +154,26 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) { let property = schemaDocument.childrenNodesProperties[i] switch (property.type) { - case 'node': { - if (property.name !== previousPropertyName) { + case 'node': + { + if (property.name !== previousPropertyName) { + if (node[property.name] !== undefined) { + evalNode(node[property.name]) + } + previousPropertyName = property.name + } + break + } + case 'array': + { if (node[property.name] !== undefined) { - evalNode(node[property.name]) + let nodePropertyArray = node[property.name] + for (let m = 0; m < nodePropertyArray.length; m++) { + evalNode(nodePropertyArray[m]) + } } - previousPropertyName = property.name + break } - break - } - case 'array': { - if (node[property.name] !== undefined) { - let nodePropertyArray = node[property.name] - for (let m = 0; m < nodePropertyArray.length; m++) { - evalNode(nodePropertyArray[m]) - } - } - break - } } } } @@ -173,10 +182,10 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) { function evalCondition(node) { /* - The code can be at the condition node if it was done with the Conditions Editor, or it can also be - at a Javascript Code node. If there is a Javascript object we will give it preference and take the code - from there. Otherwise we will take the code from the Condition node. - */ + The code can be at the condition node if it was done with the Conditions Editor, or it can also be + at a Javascript Code node. If there is a Javascript object we will give it preference and take the code + from there. Otherwise we will take the code from the Condition node. + */ let nodeWithCode = node if (node.javascriptCode !== undefined) { @@ -211,14 +220,15 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) { instruction = instruction.replace(/!==/g, '') instruction = instruction.replace(/==/g, '') instruction = instruction.replace(/===/g, '') - instruction = instruction.replace(/{/g, '') - instruction = instruction.replace(/}/g, '') + instruction = instruction.replace(/{/g, '') //this wrecks something in JS markup? + instruction = instruction.replace(/}/g, '') //this wrecks something in JS markup? if (instruction.indexOf('chart') >= 0) { let parts = instruction.split('.') let timeFrame = parts[1] let product = parts[2] let property checkPrevious(3) + function checkPrevious(index) { property = parts[index] if (property === 'previous') { @@ -295,17 +305,17 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) { let closeHeaders = ['Trade Number', 'Open Datetime', 'Close Datetime', 'Strategy Name', 'Trigger On Situation', 'Take Position Situation', 'Result In Base Asset', 'Result In Quoted Asset', 'ROI in Base Asset', 'ROI in Quoted Asset', 'Exit Type'] closeValues = [ - tradingEngine.tradingCurrent.tradingEpisode.tradingEpisodeCounters.positions.value, // Position Number - (new Date(openDatetime)).toISOString(), // Open Datetime - (new Date(closeDatetime)).toISOString(), // Open Datetime - tradingEngine.tradingCurrent.strategy.strategyName.value, // Strategy Name - tradingEngine.tradingCurrent.strategy.situationName.value, // Trigger On Situation - tradingEngine.tradingCurrent.position.situationName.value, // Take Position Situation - tradingEngine.tradingCurrent.position.positionBaseAsset.hitFail.value, // Result in Base Asset - tradingEngine.tradingCurrent.position.positionQuotedAsset.hitFail.value, // Result in Base Asset - tradingEngine.tradingCurrent.position.positionBaseAsset.ROI.value, // ROI in Base Asseet - tradingEngine.tradingCurrent.position.positionQuotedAsset.ROI.value, // ROI in Quoted Asset - tradingEngine.tradingCurrent.position.exitType.value // Exit Type + tradingEngine.tradingCurrent.tradingEpisode.tradingEpisodeCounters.positions.value, // Position Number + (new Date(openDatetime)).toISOString(), // Open Datetime + (new Date(closeDatetime)).toISOString(), // Open Datetime + tradingEngine.tradingCurrent.strategy.strategyName.value, // Strategy Name + tradingEngine.tradingCurrent.strategy.situationName.value, // Trigger On Situation + tradingEngine.tradingCurrent.position.situationName.value, // Take Position Situation + tradingEngine.tradingCurrent.position.positionBaseAsset.hitFail.value, // Result in Base Asset + tradingEngine.tradingCurrent.position.positionQuotedAsset.hitFail.value, // Result in Base Asset + tradingEngine.tradingCurrent.position.positionBaseAsset.ROI.value, // ROI in Base Asseet + tradingEngine.tradingCurrent.position.positionQuotedAsset.ROI.value, // ROI in Quoted Asset + tradingEngine.tradingCurrent.position.exitType.value // Exit Type ] if (createCloseHeaders === true) { @@ -333,46 +343,95 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) { try { let fileStorage = TS.projects.foundations.taskModules.fileStorage.newFileStorage(processIndex); - let fileContent = ""; - let separator = "\r\n"; + let fileName = pFileName + '.csv' - parseRecord(snapshots.headers) - - for (let i = 0; i < snapshotArray.length; i++) { - let record = snapshotArray[i]; - parseRecord(record) - } - - function parseRecord(record) { - for (let j = 0; j < record.length; j++) { - let property = record[j] - - fileContent = fileContent + '' + property - if (j !== record.length - 1) { - fileContent = fileContent + "," - } - } - fileContent = fileContent + separator - } - - fileContent = "" + fileContent + ""; - - let fileName = pFileName + '.csv'; - let filePath = TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + "/" + SNAPSHOTS_FOLDER_NAME; + let filePath = TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + SNAPSHOTS_FOLDER_NAME; filePath += '/' + fileName - fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated); + //Get contents of current file if available, otherwise start empty file + if (snapshotAvailable) { + getSnapShotFile(filePath) + } else { + writeOutput('') + } - function onFileCreated(err) { - if (err.result !== TS.projects.foundations.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { - TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] writeSnapshotFile -> onFileCreated -> err = " + err.stack); - TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] writeSnapshotFile -> onFileCreated -> filePath = " + filePath); - TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] writeSnapshotFile -> onFileCreated -> market = " + TS.projects.foundations.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "_" + TS.projects.foundations.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName); + function writeOutput(fileContent) { + let separator = '\r\n'; + + parseRecord(snapshots.headers) + + for (let i = 0; i < snapshotArray.length; i++) { + let record = snapshotArray[i]; + parseRecord(record) + } + + function parseRecord(record) { + for (let j = 0; j < record.length; j++) { + let property = record[j] + + fileContent = fileContent + '' + property + if (j !== record.length - 1) { + fileContent = fileContent + "," + } + } + fileContent = fileContent + separator + } + + fileContent = "" + fileContent + ""; + + fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated) + + function onFileCreated(err) { + if (err.result !== TS.projects.foundations.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] writeSnapshotFile -> onFileCreated -> err = ' + err.stack); + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] writeSnapshotFile -> onFileCreated -> filePath = ' + filePath); + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] writeSnapshotFile -> onFileCreated -> market = ' + TS.projects.foundations.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + '_' + TS.projects.foundations.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName); + } } } + + } catch (err) { + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get( + processIndex + ).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write( + MODULE_NAME, + '[ERROR] writeSnapshotFile -> err = ' + err.stack + ) } - catch (err) { - TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] writeSnapshotFile -> err = " + err.stack); + //get file contents + function getSnapShotFile(filePath) { + + try { + + let fileStorage = TS.projects.foundations.taskModules.fileStorage.newFileStorage(processIndex); + + fileStorage.getTextFile(filePath, onFileReceived) + + function onFileReceived(err, text) { + if (err.result === TS.projects.foundations.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { + try { + writeOutput(text) + } catch (err) { + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> fileName = ' + filePath); + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> filePath = ' + filePath); + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> err = ' + err.stack); + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.'); + callBackFunction(TS.projects.foundations.globals.standardResponses.DEFAULT_RETRY_RESPONSE) + } + } else { + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> err = ' + err.stack); + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> filePath = ' + filePath); + TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err; + callBackFunction(err) + } + } + + } catch (error) { + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> err = ' + err.stack); + TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> filePath = ' + filePath); + TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err; + callBackFunction(err); + } } } } \ No newline at end of file diff --git a/Projects/Foundations/TS/Task-Modules/FileStorage.js b/Projects/Foundations/TS/Task-Modules/FileStorage.js index 97cd5a7f6..a680fd6ab 100644 --- a/Projects/Foundations/TS/Task-Modules/FileStorage.js +++ b/Projects/Foundations/TS/Task-Modules/FileStorage.js @@ -8,16 +8,17 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) { let thisObject = { asyncGetTextFile: asyncGetTextFile, - asyncCreateTextFile: asyncCreateTextFile, + asyncCreateTextFile: asyncCreateTextFile, getTextFile: getTextFile, createTextFile: createTextFile, deleteTextFile: deleteTextFile } - + let logger if (TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT === undefined) { // Dummy logger logger = {} + function write() { } @@ -31,16 +32,17 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) { /* This function allows its caller to work with async / await instead of callbacks */ let promise = new Promise((resolve, reject) => { - getTextFile(filePath, onFileRead, noRetry, canUsePrevious) + getTextFile(filePath, onFileRead, noRetry, canUsePrevious) + function onFileRead(err, text) { - + let response = { err: err, text: text } resolve(response) } - }) + }) return promise } @@ -49,15 +51,16 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) { /* This function allows its caller to work with async / await instead of callbacks */ let promise = new Promise((resolve, reject) => { - createTextFile(filePath, fileContent, onFileWriten, keepPrevious, noTemp) + createTextFile(filePath, fileContent, onFileWriten, keepPrevious, noTemp) + function onFileWriten(err) { - + let response = { err: err } resolve(response) } - }) + }) return promise } @@ -78,8 +81,13 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) { fileLocation = global.env.PATH_TO_PROJECTS + '/' + filePath mustBeJason = false } else { + if (filePath.indexOf('.csv') > 0) { + mustBeJason = false + } else { + mustBeJason = true + } fileLocation = global.env.PATH_TO_DATA_STORAGE + '/' + filePath - mustBeJason = true + } try { @@ -145,7 +153,7 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) { } /* - We are going to check if the file is a valir JSON object + We are going to check if the file is a valid JSON object */ try { let jsonCheck = JSON.parse(text.toString()) @@ -457,7 +465,7 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) { let request = http.get(url, onResponse); - request.on('error', function (err) { + request.on('error', function(err) { logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onError -> err = " + err.stack); logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onError -> Failed to fetch file via HTTP. Will retry later. "); callBackFunction(TS.projects.foundations.globals.standardResponses.DEFAULT_RETRY_RESPONSE); @@ -498,4 +506,4 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) { callBackFunction(TS.projects.foundations.globals.standardResponses.DEFAULT_RETRY_RESPONSE); } } -} +} \ No newline at end of file