fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1763 from Ferrox85/develop

Browse Source 
Changes to CSV Snapshot and Filestorage foor CSV handling
This commit is contained in:
Luis Fernando Molina authored and GitHub committed 2021-09-14 15:41:52 -04:00
2 parents bf854b4419 9d24120b9b
commit 29601079c8
2 files changed
+159 -92
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/Snapshots.js
+139 -80
View File
@@ -1,7 +1,7 @@
exports.newFoundationsBotModulesSnapshots = function (processIndex) {
exports.newFoundationsBotModulesSnapshots = function(processIndex) {
/*
This module encapsulates the snapshots functionality.
*/
This module encapsulates the snapshots functionality.
*/
const MODULE_NAME = 'Snapshots'
let thisObject = {
@@ -28,20 +28,21 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) {
positions: []
}
let positionKeys = new Map() // This is to prevent adding the same indicator / variable to the output collection.
let positionKeys = new Map() // This is to prevent adding the same indicator / variable to the output collection.
let strategyKeys = new Map()
let createHeaders = true // This is going to be true only once
let createCloseHeaders = true // This is going to be true only once
let createHeaders = true // This is going to be true only once
let createCloseHeaders = true // This is going to be true only once
let closeValues
let strategyValues
let addToStrategyValues
let positionValues
let addToPositionValues
let snapshotAvailable = false
/*
These 3 are the main data structures available to users
when writing conditions and formulas.
*/
These 3 are the main data structures available to users
when writing conditions and formulas.
*/
let chart
let exchange
let market
@@ -52,6 +53,12 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) {
tradingEngine = TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.tradingEngine
tradingSystem = TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.tradingSystem
sessionParameters = TS.projects.foundations.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters
// first trade forces overwrite of the existing file
if (tradingEngine.tradingCurrent.tradingEpisode.tradingEpisodeCounters.positions.value > 1) {
snapshotAvailable = true
createHeaders = false // This is going to be true only once
createCloseHeaders = false
}
}
function finalize() {
@@ -70,9 +77,9 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) {
function updateChart(pChart, pExchange, pMarket) {
/*
We need these 3 data structures to be a local objects
accessible while evaluating conditions and formulas.
*/
We need these 3 data structures to be a local objects
accessible while evaluating conditions and formulas.
*/
chart = pChart
exchange = pExchange
market = pMarket
@@ -80,9 +87,9 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) {
function strategyEntry() {
/*
We will record here a snapshot of all values used in conditions and formulas within the Trading System.
If the strategy exists with a trade, this information will be added to the snapshots file.
*/
We will record here a snapshot of all values used in conditions and formulas within the Trading System.
If the strategy exists with a trade, this information will be added to the snapshots file.
*/
strategyKeys = new Map()
strategyValues = []
@@ -147,24 +154,26 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) {
let property = schemaDocument.childrenNodesProperties[i]
switch (property.type) {
case 'node': {
if (property.name !== previousPropertyName) {
case 'node':
{
if (property.name !== previousPropertyName) {
if (node[property.name] !== undefined) {
evalNode(node[property.name])
}
previousPropertyName = property.name
}
break
}
case 'array':
{
if (node[property.name] !== undefined) {
evalNode(node[property.name])
let nodePropertyArray = node[property.name]
for (let m = 0; m < nodePropertyArray.length; m++) {
evalNode(nodePropertyArray[m])
}
}
previousPropertyName = property.name
break
}
break
}
case 'array': {
if (node[property.name] !== undefined) {
let nodePropertyArray = node[property.name]
for (let m = 0; m < nodePropertyArray.length; m++) {
evalNode(nodePropertyArray[m])
}
}
break
}
}
}
}
@@ -173,10 +182,10 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) {
function evalCondition(node) {
/*
The code can be at the condition node if it was done with the Conditions Editor, or it can also be
at a Javascript Code node. If there is a Javascript object we will give it preference and take the code
from there. Otherwise we will take the code from the Condition node.
*/
The code can be at the condition node if it was done with the Conditions Editor, or it can also be
at a Javascript Code node. If there is a Javascript object we will give it preference and take the code
from there. Otherwise we will take the code from the Condition node.
*/
let nodeWithCode = node
if (node.javascriptCode !== undefined) {
@@ -211,14 +220,15 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) {
instruction = instruction.replace(/!==/g, '')
instruction = instruction.replace(/==/g, '')
instruction = instruction.replace(/===/g, '')
instruction = instruction.replace(/{/g, '')
instruction = instruction.replace(/}/g, '')
instruction = instruction.replace(/{/g, '') //this wrecks something in JS markup?
instruction = instruction.replace(/}/g, '') //this wrecks something in JS markup?
if (instruction.indexOf('chart') >= 0) {
let parts = instruction.split('.')
let timeFrame = parts[1]
let product = parts[2]
let property
checkPrevious(3)
function checkPrevious(index) {
property = parts[index]
if (property === 'previous') {
@@ -295,17 +305,17 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) {
let closeHeaders = ['Trade Number', 'Open Datetime', 'Close Datetime', 'Strategy Name', 'Trigger On Situation', 'Take Position Situation', 'Result In Base Asset', 'Result In Quoted Asset', 'ROI in Base Asset', 'ROI in Quoted Asset', 'Exit Type']
closeValues = [
tradingEngine.tradingCurrent.tradingEpisode.tradingEpisodeCounters.positions.value, // Position Number
(new Date(openDatetime)).toISOString(), // Open Datetime
(new Date(closeDatetime)).toISOString(), // Open Datetime
tradingEngine.tradingCurrent.strategy.strategyName.value, // Strategy Name
tradingEngine.tradingCurrent.strategy.situationName.value, // Trigger On Situation
tradingEngine.tradingCurrent.position.situationName.value, // Take Position Situation
tradingEngine.tradingCurrent.position.positionBaseAsset.hitFail.value, // Result in Base Asset
tradingEngine.tradingCurrent.position.positionQuotedAsset.hitFail.value, // Result in Base Asset
tradingEngine.tradingCurrent.position.positionBaseAsset.ROI.value, // ROI in Base Asseet
tradingEngine.tradingCurrent.position.positionQuotedAsset.ROI.value, // ROI in Quoted Asset
tradingEngine.tradingCurrent.position.exitType.value // Exit Type
tradingEngine.tradingCurrent.tradingEpisode.tradingEpisodeCounters.positions.value, // Position Number
(new Date(openDatetime)).toISOString(), // Open Datetime
(new Date(closeDatetime)).toISOString(), // Open Datetime
tradingEngine.tradingCurrent.strategy.strategyName.value, // Strategy Name
tradingEngine.tradingCurrent.strategy.situationName.value, // Trigger On Situation
tradingEngine.tradingCurrent.position.situationName.value, // Take Position Situation
tradingEngine.tradingCurrent.position.positionBaseAsset.hitFail.value, // Result in Base Asset
tradingEngine.tradingCurrent.position.positionQuotedAsset.hitFail.value, // Result in Base Asset
tradingEngine.tradingCurrent.position.positionBaseAsset.ROI.value, // ROI in Base Asseet
tradingEngine.tradingCurrent.position.positionQuotedAsset.ROI.value, // ROI in Quoted Asset
tradingEngine.tradingCurrent.position.exitType.value // Exit Type
]
if (createCloseHeaders === true) {
@@ -333,46 +343,95 @@ exports.newFoundationsBotModulesSnapshots = function (processIndex) {
try {
let fileStorage = TS.projects.foundations.taskModules.fileStorage.newFileStorage(processIndex);
let fileContent = "";
let separator = "\r\n";
let fileName = pFileName + '.csv'
parseRecord(snapshots.headers)
for (let i = 0; i < snapshotArray.length; i++) {
let record = snapshotArray[i];
parseRecord(record)
}
function parseRecord(record) {
for (let j = 0; j < record.length; j++) {
let property = record[j]
fileContent = fileContent + '' + property
if (j !== record.length - 1) {
fileContent = fileContent + ","
}
}
fileContent = fileContent + separator
}
fileContent = "" + fileContent + "";
let fileName = pFileName + '.csv';
let filePath = TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + "/Output/" + TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + "/" + SNAPSHOTS_FOLDER_NAME;
let filePath = TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT + '/Output/' + TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME + '/' + SNAPSHOTS_FOLDER_NAME;
filePath += '/' + fileName
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated);
//Get contents of current file if available, otherwise start empty file
if (snapshotAvailable) {
getSnapShotFile(filePath)
} else {
writeOutput('')
}
function onFileCreated(err) {
if (err.result !== TS.projects.foundations.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] writeSnapshotFile -> onFileCreated -> err = " + err.stack);
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] writeSnapshotFile -> onFileCreated -> filePath = " + filePath);
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] writeSnapshotFile -> onFileCreated -> market = " + TS.projects.foundations.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "_" + TS.projects.foundations.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName);
function writeOutput(fileContent) {
let separator = '\r\n';
parseRecord(snapshots.headers)
for (let i = 0; i < snapshotArray.length; i++) {
let record = snapshotArray[i];
parseRecord(record)
}
function parseRecord(record) {
for (let j = 0; j < record.length; j++) {
let property = record[j]
fileContent = fileContent + '' + property
if (j !== record.length - 1) {
fileContent = fileContent + ","
}
}
fileContent = fileContent + separator
}
fileContent = "" + fileContent + "";
fileStorage.createTextFile(filePath, fileContent + '\n', onFileCreated)
function onFileCreated(err) {
if (err.result !== TS.projects.foundations.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] writeSnapshotFile -> onFileCreated -> err = ' + err.stack);
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] writeSnapshotFile -> onFileCreated -> filePath = ' + filePath);
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] writeSnapshotFile -> onFileCreated -> market = ' + TS.projects.foundations.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + '_' + TS.projects.foundations.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.quotedAsset.referenceParent.config.codeName);
}
}
}
} catch (err) {
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(
processIndex
).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(
MODULE_NAME,
'[ERROR] writeSnapshotFile -> err = ' + err.stack
)
}
catch (err) {
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] writeSnapshotFile -> err = " + err.stack);
//get file contents
function getSnapShotFile(filePath) {
try {
let fileStorage = TS.projects.foundations.taskModules.fileStorage.newFileStorage(processIndex);
fileStorage.getTextFile(filePath, onFileReceived)
function onFileReceived(err, text) {
if (err.result === TS.projects.foundations.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
try {
writeOutput(text)
} catch (err) {
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> fileName = ' + filePath);
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> filePath = ' + filePath);
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> err = ' + err.stack);
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> Asuming this is a temporary situation. Requesting a Retry.');
callBackFunction(TS.projects.foundations.globals.standardResponses.DEFAULT_RETRY_RESPONSE)
}
} else {
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> err = ' + err.stack);
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> filePath = ' + filePath);
TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err;
callBackFunction(err)
}
}
} catch (error) {
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> err = ' + err.stack);
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> filePath = ' + filePath);
TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err;
callBackFunction(err);
}
}
}
}
Projects/Foundations/TS/Task-Modules/FileStorage.js
+20 -12
View File
@@ -8,16 +8,17 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) {
let thisObject = {
asyncGetTextFile: asyncGetTextFile,
asyncCreateTextFile: asyncCreateTextFile,
asyncCreateTextFile: asyncCreateTextFile,
getTextFile: getTextFile,
createTextFile: createTextFile,
deleteTextFile: deleteTextFile
}
let logger
if (TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT === undefined) { // Dummy logger
logger = {}
function write() {
}
@@ -31,16 +32,17 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) {
/* This function allows its caller to work with async / await instead of callbacks */
let promise = new Promise((resolve, reject) => {
getTextFile(filePath, onFileRead, noRetry, canUsePrevious)
getTextFile(filePath, onFileRead, noRetry, canUsePrevious)
function onFileRead(err, text) {
let response = {
err: err,
text: text
}
resolve(response)
}
})
})
return promise
}
@@ -49,15 +51,16 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) {
/* This function allows its caller to work with async / await instead of callbacks */
let promise = new Promise((resolve, reject) => {
createTextFile(filePath, fileContent, onFileWriten, keepPrevious, noTemp)
createTextFile(filePath, fileContent, onFileWriten, keepPrevious, noTemp)
function onFileWriten(err) {
let response = {
err: err
}
resolve(response)
}
})
})
return promise
}
@@ -78,8 +81,13 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) {
fileLocation = global.env.PATH_TO_PROJECTS + '/' + filePath
mustBeJason = false
} else {
if (filePath.indexOf('.csv') > 0) {
mustBeJason = false
} else {
mustBeJason = true
}
fileLocation = global.env.PATH_TO_DATA_STORAGE + '/' + filePath
mustBeJason = true
}
try {
@@ -145,7 +153,7 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) {
}
/*
We are going to check if the file is a valir JSON object
We are going to check if the file is a valid JSON object
*/
try {
let jsonCheck = JSON.parse(text.toString())
@@ -457,7 +465,7 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) {
let request = http.get(url, onResponse);
request.on('error', function (err) {
request.on('error', function(err) {
logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onError -> err = " + err.stack);
logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onError -> Failed to fetch file via HTTP. Will retry later. ");
callBackFunction(TS.projects.foundations.globals.standardResponses.DEFAULT_RETRY_RESPONSE);
@@ -498,4 +506,4 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) {
callBackFunction(TS.projects.foundations.globals.standardResponses.DEFAULT_RETRY_RESPONSE);
}
}
}
}