Merge branch 'develop' of https://github.com/Superalgos/Superalgos into develop

2021-01-16 20:19:35 +01:00
commit baa5b0c0f0
@@ -590,6 +590,7 @@ exports.newHttpInterface = function newHttpInterface(WEB_SERVER, DATA_FILE_SERVE
                switch (requestParameters[2]) { // switch by command
                    case 'Contribute': {
                        try {
+                            console.log(JSON.stringify(requestParameters))
                            const commitMessage = unescape(requestParameters[3])
                            const username = unescape(requestParameters[4])
                            const token = unescape(requestParameters[5])
@@ -2167,6 +2167,7 @@
                "childType": "Github API"
            }
        ],
+        "isPersonalData": true,
        "icon": "bitcoin"
    },
    {
@@ -2202,6 +2203,7 @@
            "compatibleTypes": "->APIs->"
        },
        "propertyNameAtParent": "githubAPI",
+        "isPersonalData": true,
        "icon": "bitcoin"
    },
    {
@@ -505,7 +505,7 @@ exports.newSuperalgosBotModulesHistoricOHLCVs = function (processIndex) {
                                        volume: record[5],
                                        id: record[6]
                                    }
-                                    checkOHLCVMinute()
+                                    setImmediate (checkOHLCVMinute)
                                }
                            }

@@ -87,7 +87,8 @@
                This variable tell us which day we are standing at, specially while working
                with Daily Files. From this Date is that we are going to load the Daily Files.
                */
-                TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.tradingEngine.current.episode.processDate.value = TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeRange.config.initialDatetime).valueOf()
+                TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.tradingEngine.current.episode.processDate.value =
+                    TS.projects.superalgos.utilities.dateTimeFunctions.removeTime(TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeRange.config.initialDatetime).valueOf()
            }

            /* 
@@ -483,35 +484,47 @@
                            continue
                        }

+                        /*
+                        If it is not a dependency then we pass..
+                        */
                        if (dataDependenciesModule.isItADepenency(timeFrameLabel, datasetModule.node.parentNode.config.singularVariableName) !== true) {
                            if (!(TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.tradingParameters.timeFrame.config.label === timeFrameLabel && datasetModule.node.parentNode.config.pluralVariableName === 'candles')) {
                                continue
                            }
                        }

-                        let dateForPath = tradingProcessDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(tradingProcessDate.getUTCDate(), 2);
-                        let filePath
-                        if (dependency.referenceParent.config.codeName === "Multi-Period-Daily") {
-                            filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel + "/" + dateForPath;
-                        } else {
-                            filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + dateForPath;
-                        }
-                        let fileName = "Data.json";
+                        /*
+                        We will need to fetch the data of the current day and the previous day, in order for .previous properties in conditions and formulas to work well.
+                        */
+                        let previousDate = new Date(tradingProcessDate.valueOf() - TS.projects.superalgos.globals.timeConstants.ONE_DAY_IN_MILISECONDS)
+                        let currentDate = new Date(tradingProcessDate.valueOf())

-                        /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */
-                        let response = await asyncGetDatasetFile(datasetModule, filePath, fileName)
+                        let previousFile = await getDataFileFromDate(previousDate)
+                        if (previousFile === false) { return false }
+                        let currentFile = await getDataFileFromDate(currentDate)
+                        if (currentFile === false) { return false }
+                        let bothFiles = previousFile.concat(currentFile)
+                        dataFiles.set(dependency.id, bothFiles)

-                        if (response.err.message === 'File does not exist.') {
-                            return false
-                        }
-                        if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
-                            throw (response.err)
-                        }
+                        async function getDataFileFromDate(processDate) {

-                        let dataFile = JSON.parse(response.text);
-                        dataFiles.set(dependency.id, dataFile);
+                            let dateForPath = processDate.getUTCFullYear() + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processDate.getUTCMonth() + 1, 2) + '/' + TS.projects.superalgos.utilities.miscellaneousFunctions.pad(processDate.getUTCDate(), 2);
+                            let filePath = dependency.referenceParent.parentNode.config.codeName + '/' + dependency.referenceParent.config.codeName + "/" + timeFrameLabel + "/" + dateForPath
+                            let fileName = "Data.json";
+
+                            /* We cut the async calls via callBacks at this point, so as to have a clearer code upstream */
+                            let response = await asyncGetDatasetFile(datasetModule, filePath, fileName)
+
+                            if (response.err.message === 'File does not exist.') {
+                                return false
+                            }
+                            if (response.err.result !== TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
+                                throw (response.err)
+                            }
+
+                            return JSON.parse(response.text)
+                        }
                    }
-
                    let mapKey = TS.projects.superalgos.globals.timeFrames.dailyFilePeriods()[n][1];
                    multiPeriodDataFiles.set(mapKey, dataFiles)
                }
@@ -757,11 +770,6 @@
        }
    }

-    function pad(str, max) {
-        str = str.toString();
-        return str.length < max ? pad(" " + str, max) : str;
-    }
-
    async function asyncGetDatasetFile(datasetModule, filePath, fileName) {

        let promise = new Promise((resolve, reject) => {
@@ -45,6 +45,23 @@ exports.newSuperalgosBotModulesTradingSimulation = function (processIndex) {
            let propertyName = 'at' + sessionParameters.timeFrame.config.label.replace('-', '')
            let candles = chart[propertyName].candles

+            if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES) {
+                /*
+                We need to purge from the candles array all the candles from the previous day 
+                that comes when processing daily files.
+                */
+                let dailyCandles = []
+                for (let i = 0; i < candles.length; i++) {
+                    let candle = candles[i]
+                    if (candle.begin >= TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.tradingEngine.current.episode.processDate.value) {
+                        dailyCandles.push(candle)
+                    }
+                }
+                candles = dailyCandles
+            } else {
+                candles = chart[propertyName].candles
+            }
+
            /* Variables needed for heartbeat functionality */
            let heartBeatDate
            let previousHeartBeatDate
@@ -55,7 +72,8 @@ exports.newSuperalgosBotModulesTradingSimulation = function (processIndex) {
            if (
                TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_FIRST_LOOP === true &&
                TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).IS_SESSION_RESUMING === false) {
-                /* Estimate Initial Candle based on the timeRage configured for the session. */
+
+                /* Estimate Initial Candle based on the timeRange configured for the session. */
                let firstEnd = candles[0].end
                let targetEnd = sessionParameters.timeRange.config.initialDatetime
                let diff = targetEnd - firstEnd
@@ -73,8 +91,8 @@ exports.newSuperalgosBotModulesTradingSimulation = function (processIndex) {
                        '[IMPORTANT] runSimulation -> Data is not up-to-date enough. Stopping the Session now. ')
                    return
                }
-            } else {

+            } else {
                /* 
                In this case we already have at the last candle index the next candle to be
                processed. We will just continue with this candle.
@@ -545,7 +563,7 @@ exports.newSuperalgosBotModulesTradingSimulation = function (processIndex) {
                return true
            }
        } catch (err) {
-            TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, 
+            TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
                '[ERROR] runSimulation -> err = ' + err.stack)
            throw (TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE)
        }
@@ -375,7 +375,7 @@ function newSuperalgosDocsContextMenu() {
            return true
        }
        if (paragraphNode.id.indexOf('-warning') >= 0) {
-            selectedParagraphData = paragraphNode.innerText.trim().substring(10, paragraphNode.innerText.length)
+            selectedParagraphData = paragraphNode.innerText.trim().substring(9, paragraphNode.innerText.length)
            return true
        }
        if (paragraphNode.id.indexOf('-error') >= 0) {
@@ -1186,7 +1186,9 @@ function newSuperalgosTutorialSpace() {
        transformDocsInfoIntoTutorialInfo()
        createTheHTML()

-        UI.projects.superalgos.spaces.docsSpace.navigateTo(nodeConfig.docs.project, 'Tutorial', nodeConfig.docs.type)
+        if (nodeConfig.closeEditor !== true) {
+            UI.projects.superalgos.spaces.docsSpace.navigateTo(nodeConfig.docs.project, 'Tutorial', nodeConfig.docs.type)
+        }

        function syncConfigIconWithDocumentIcon() {
            /*