Merge pull request #547 from Luis-Fernando-Molina/develop

Contribution: Machinge Learning Bot implementation (icons missing)

Projects/ProjectsSchema.json

@@ -845,6 +845,15 @@
                     "fileName": "LearningSystem.js",
                     "folderName": "Learning-Bot/Low-Frequency-Learning"
                 }
+            ],
+            "algorithmModules": [
+                {
+                    "name": "Time Series Forcasting Tensor Flow JS",
+                    "propertyName": "timeSeriesForcastingTensorFlowJS",
+                    "functionName": "newSuperalgosAlgorithmModulesTimeSeriesForcastingTensorFlowJS",
+                    "fileName": "LearningAlgorithm.js",
+                    "folderName": "Learning-Bot/Low-Frequency-Learning/Time-Series-Forcasting/Tensor-Flow-JS"
+                }
             ]
         },
         "CL": {}

Projects/Superalgos/Plugins/Workspaces/Machine-Learning-Demo.json

@@ -0,0 +1 @@
+{"type":"Workspace","name":"Machine-Learning-Demo","rootNodes":[{"type":"Network","name":"My Network","project":"Superalgos","networkNodes":[{"type":"Network Node","name":"My Computer","config":"{ \n\"host\": \"localhost\", \n\"webPort\": \"34248\", \n\"webSocketsPort\": \"18041\"\n}","project":"Superalgos","dataTasks":{"type":"Data Tasks","name":"New Data Tasks","project":"Superalgos","projectDataTasks":[{"type":"Project Data Tasks","name":"Superalgos","project":"Superalgos","exchangeDataTasks":[{"type":"Exchange Data Tasks","name":"Binance","project":"Superalgos","marketDataTasks":[{"type":"Market Data Tasks","name":"Binance BTC/USDT","project":"Superalgos","dataMineTasks":[{"type":"Data Mine Tasks","name":"Masters","project":"Superalgos","taskManagers":[{"type":"Task Manager","name":"Masters","project":"Superalgos","tasks":[{"type":"Task","name":"Exchange Raw Data","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Exchange Raw Data","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Exchange Raw Data Historic-OHLCVs","project":"Superalgos","id":"31e87840-fc8a-4305-a97d-3a78b8e4be61","savedPayload":{"position":{"x":24977.356993272086,"y":30782.11115747996},"targetPosition":{"x":24995.356993272086,"y":30827.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Historic-OHLCVs","id":"5eb11d6a-022e-4f60-97c4-d6cf90b249dc"}}}],"id":"ddab2649-221f-40c4-8b8e-b3cbceca0c4b","savedPayload":{"position":{"x":24995.356993272086,"y":30827.11115747996},"targetPosition":{"x":24900.356993272086,"y":30922.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"1bca451f-f059-4dc2-87cb-ba2bb1c16ef4","savedPayload":{"position":{"x":24900.356993272086,"y":30922.11115747996},"targetPosition":{"x":24938.356993272086,"y":30991.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Webhooks","project":"Superalgos","bot":{"type":"Sensor Bot Instance","name":"Webhooks","config":"{ \n\"startDate\": \"2020-01-01\"\n}","project":"Superalgos","processes":[{"type":"Sensor Process Instance","name":"Webhooks Check-Webhook","project":"Superalgos","id":"4595558a-d0ba-49b1-9c81-67696f23261c","savedPayload":{"position":{"x":24958.356993272086,"y":30985.11115747996},"targetPosition":{"x":24998.356993272086,"y":30997.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Check-Webhook","id":"69f37820-ab16-4e19-bfd8-386dd6e249a3"}}}],"id":"a7978877-bdef-4e69-a97d-af0560ef219e","savedPayload":{"position":{"x":24998.356993272086,"y":30997.11115747996},"targetPosition":{"x":24980.356993272086,"y":31030.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"id":"bf34c6df-d1ac-4f7c-8a6c-7ed9b6de2e00","savedPayload":{"position":{"x":24980.356993272086,"y":31030.11115747996},"targetPosition":{"x":24938.356993272086,"y":30991.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Task","name":"Candles Volumes","project":"Superalgos","bot":{"type":"Indicator Bot Instance","name":"Candles Volumes","project":"Superalgos","processes":[{"type":"Indicator Process Instance","name":"Candles Volumes Multi-Period-Market","project":"Superalgos","id":"7cdc33e0-c56b-43ce-95f5-db03a171f021","savedPayload":{"position":{"x":24917.356993272086,"y":30838.11115747996},"targetPosition":{"x":25001.356993272086,"y":30917.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"Multi-Period-Market","id":"9f683143-a33f-42c1-9fa8-c601f01ee4e1"}}},{"type":"Indicator Process Instance","name":"Candles Volumes Multi-Period-Daily","project":"Superalgos","id":"4da5ecc7-48e6-4283-8a2f-512528a28fe4","savedPayload":{"position":{"x":25093.356993272086,"y":30990.11115747996},"targetPosition":{"x":25001.356993272086,"y":30917.11115747996},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":3,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false},"referenceParent":{"type":"Process Definition","name":"MLine truncated

Projects/Superalgos/Schemas/AppSchema.json

@@ -881,6 +881,40 @@
         ],
         "addLeftIcons": true
     },
+    {
+        "type": "Training Data",
+        "level": 4,
+        "propertyNameAtParent": "trainingData",
+        "editors": {
+            "code": true
+        },
+        "initialValues": {
+            "code": "// Type a list of indicators to be used as training data..."
+        },
+        "attachingRules": {
+            "compatibleTypes": "->Learning System->"
+        },
+        "menuItems": [
+            {
+                "action": "Edit",
+                "actionFunction": "uiObject.codeEditor.activate",
+                "label": "Edit",
+                "iconPathOn": "javascript-code",
+                "iconPathOff": "javascript-code"
+            },
+            {
+                "action": "Delete UI Object",
+                "askConfirmation": true,
+                "confirmationLabel": "Confirm to Delete",
+                "actionFunction": "payload.executeAction",
+                "label": "Delete",
+                "iconPathOn": "delete-entity",
+                "iconPathOff": "delete-entity"
+            }
+        ],
+        "addLeftIcons": true,
+        "icon": "bitcoin"
+    },
     {
         "type": "Javascript Code",
         "level": 4,
@@ -10441,7 +10475,7 @@
             }
         ],
         "initialValues": {
-            "config": "{ \n\"codeName\": \"\"\n}"
+            "config": "{ \n\"codeName\": \"Time Series Forcasting Tensor Flow JS\"\n}"
         },
         "addLeftIcons": true,
         "editors": {
@@ -12640,6 +12674,12 @@
     {
         "type": "Learning System",
         "menuItems": [
+            {
+                "action": "Add UI Object",
+                "label": "Add Training Data",
+                "relatedUiObject": "Training Data",
+                "actionFunction": "payload.executeAction"
+            },
             {
                 "action": "Delete UI Object",
                 "askConfirmation": true,
@@ -12653,7 +12693,13 @@
         "isHierarchyHead": true,
         "addLeftIcons": true,
         "level": 0,
-        "childrenNodesProperties": [],
+        "childrenNodesProperties": [
+            {
+                "name": "trainingData",
+                "type": "array",
+                "childType": "Training Data"
+            }
+        ],
         "icon": "bitcoin"
     },
     {

Projects/Superalgos/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/LearningSystem.js

@@ -26,6 +26,28 @@ exports.newSuperalgosBotModulesLearningSystem = function (processIndex) {
     let sessionParameters
     let dynamicIndicators
 
+    let learningAlgorithmName = TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.learningParameters.learningAlgorithm.config.codeName
+    let learningAlgorithmModuleObject
+    /* 
+    We will scan the project schema until we find the module that will run the algorithm
+    defined by the user at the UI / Session Parameters.
+    */
+    for (let i = 0; i < TS.projects.superalgos.globals.taskConstants.PROJECTS_SCHEMA.length; i++) {
+        let project = TS.projects.superalgos.globals.taskConstants.PROJECTS_SCHEMA[i]
+        if (project.name !== TS.projects.superalgos.globals.taskConstants.PROJECT_DEFINITION_NODE.config.codeName) { continue }
+        for (let j = 0; j < project.TS.algorithmModules.length; j++) {
+            algorithmModuleDefinition = project.TS.algorithmModules[j]
+            if (algorithmModuleDefinition.name === learningAlgorithmName) {
+
+                TS.projects.superalgos.globals.processVariables.TOTAL_PROCESS_INSTANCES_CREATED++
+                let project = TS.projects[TS.projects.superalgos.globals.taskConstants.PROJECT_DEFINITION_NODE.config.codeName.toLowerCase()]
+                let botModule = project.algorithmModules[algorithmModuleDefinition.propertyName]
+                let moduleFunction = botModule[algorithmModuleDefinition.functionName]
+                learningAlgorithmModuleObject = moduleFunction(processIndex)
+            }
+        }
+    }
+
     return thisObject
 
     function initialize() {
@@ -98,6 +120,19 @@ exports.newSuperalgosBotModulesLearningSystem = function (processIndex) {
             learningSystem.infos.push(infoDataArray)
         }
 
+        /*
+        Initialize the Learning Algorithm
+        */
+        learningAlgorithmModuleObject.initialize(onInitializeReady)
+
+        function onInitializeReady(err) {
+            if (err.result === TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
+
+            } else {
+
+            }
+        }
+
         function isItInside(elementWithTimestamp, elementWithBeginEnd) {
             /* 
             The function is to allow in Conditions and Formulas to easily 
@@ -115,6 +150,9 @@ exports.newSuperalgosBotModulesLearningSystem = function (processIndex) {
 
     function finalize() {
 
+        learningAlgorithmModuleObject.finalize()
+        learningAlgorithmModuleObject = undefined
+
         chart = undefined
         exchange = undefined
         market = undefined
@@ -137,10 +175,11 @@ exports.newSuperalgosBotModulesLearningSystem = function (processIndex) {
     }
 
     function mantain() {
-
+        learningAlgorithmModuleObject.mantain()
     }
 
     function reset() {
+        learningAlgorithmModuleObject.reset()
 
         learningSystem.highlights = []
         learningSystem.errors = []
@@ -161,6 +200,8 @@ exports.newSuperalgosBotModulesLearningSystem = function (processIndex) {
         chart = pChart
         exchange = pExchange
         market = pMarket
+
+        learningAlgorithmModuleObject.updateChart(pChart, pExchange, pMarket)
     }
 
     function buildDynamicIndicators() {
@@ -183,6 +224,19 @@ exports.newSuperalgosBotModulesLearningSystem = function (processIndex) {
             /* Dynamic Indicators */
             buildDynamicIndicators()
 
+            /*
+            Initialize the Learning Algorithm
+            */
+            learningAlgorithmModuleObject.run(onRun)
+
+            function onRun(err) {
+                if (err.result === TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE.result) {
+
+                } else {
+
+                }
+            }
+
         } catch (err) {
             /* 
             If an error ocurred during execution, it was alreeady logged and

Projects/Superalgos/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/Time-Series-Forcasting/Tensor-Flow-JS/LearningAlgorithm.js

@@ -0,0 +1,104 @@
+exports.newSuperalgosAlgorithmModulesTimeSeriesForcastingTensorFlowJS = function (processIndex) {
+    /*
+    This module implements a learning algorithm based on Time Series Forcasting with Tensor Flow js,
+    as described at the folliwing web page:
+
+    https://jinglescode.github.io/time-series-forecasting-tensorflowjs/
+    */
+    const MODULE_NAME = 'Time Series Forcasting Tensor Flow JS Learning Algorithm'
+    let thisObject = {
+        updateChart: updateChart,
+        mantain: mantain,
+        reset: reset,
+        run: run,
+        initialize: initialize,
+        finalize: finalize
+    }
+
+    let learningEngine
+    let learningSystem
+    let sessionParameters
+
+    /* 
+    These 3 are the main data structures available to users
+    when writing conditions and formulas.
+    */
+    let chart
+    let exchange
+    let market
+
+    return thisObject
+
+    function initialize(callbackFunction) {
+        learningSystem = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.learningSystem
+        learningEngine = TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.learningEngine
+        sessionParameters = TS.projects.superalgos.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.learningParameters
+
+        callbackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE)
+    }
+
+    function finalize() {
+        learningEngine = undefined
+        learningSystem = undefined
+        sessionParameters = undefined
+
+        chart = undefined
+        exchange = undefined
+        market = undefined
+    }
+
+    function updateChart(pChart, pExchange, pMarket) {
+        /* 
+          We need these 3 data structures  to be a local objects 
+          accessible while evaluating conditions and formulas.
+          */
+        chart = pChart
+        exchange = pExchange
+        market = pMarket
+    }
+
+    function mantain() {
+ 
+    }
+
+    function reset() {
+    }
+
+    function run(callbackFunction) {
+        /*
+        Do your magic here...
+        */
+
+        /*
+        if all is good, return this...
+        */
+        callbackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_OK_RESPONSE)
+
+        /*
+        if something is bad, return this...
+        */
+        //callbackFunction(TS.projects.superalgos.globals.standardResponses.DEFAULT_FAIL_RESPONSE)
+
+    }
+
+    function badDefinitionUnhandledException(err, message, node) {
+
+        let docs = {
+            project: 'Superalgos',
+            category: 'Topic',
+            type: 'TS LF Learning Bot Error - ' + message,
+            placeholder: {}
+        }
+
+        learningSystem.addError([node.id, message, docs])
+
+        TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] -> " + message);
+        TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] -> node.name = " + node.name);
+        TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] -> node.type = " + node.type);
+        TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] -> node.config = " + JSON.stringify(node.config));
+        if (err !== undefined) {
+            TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] -> err.stack = " + err.stack);
+        }
+        throw 'Error Already Recorded'
+    }
+}

Projects/Superalgos/UI/Function-Libraries/DependenciesFilter.js

@@ -43,7 +43,7 @@ function newSuperalgosFunctionLibraryDependenciesFilter() {
         return filters
 
         function recursiveFilter(node) {
-            if (node.type === 'Javascript Code' || node.type === 'Formula') {
+            if (node.type === 'Javascript Code' || node.type === 'Formula' || node.type === 'Training Data') {
                 filter(node.code)
             }
 

TaskServer/MultiProject.js

@@ -30,6 +30,7 @@ exports.newMultiProject = function () {
             projectInstance.taskModules = {}
             projectInstance.processModules = {}
             projectInstance.botModules = {}
+            projectInstance.algorithmModules = {}
 
             /* Set up Utilities of this Project */
             if (projectDefinition.TS.utilities !== undefined) {
@@ -93,12 +94,23 @@ exports.newMultiProject = function () {
             if (projectDefinition.TS.botModules !== undefined) {
                 for (let j = 0; j < projectDefinition.TS.botModules.length; j++) {
                     let botModuleDefinition = projectDefinition.TS.botModules[j]
-                    let path = global.env.PATH_TO_PROJECTS_REQUIRED + '/' + projectDefinition.name + '/' + 'TS' + '/' + 'Bot-Modules' + '/' +  botModuleDefinition.folderName + '/' + botModuleDefinition.fileName
+                    let path = global.env.PATH_TO_PROJECTS_REQUIRED + '/' + projectDefinition.name + '/' + 'TS' + '/' + 'Bot-Modules' + '/' + botModuleDefinition.folderName + '/' + botModuleDefinition.fileName
 
                     let requiredObject = require(path)
                     projectInstance.botModules[botModuleDefinition.propertyName] = requiredObject
                 }
             }
+
+            /* Set up Algorithm Modules for this Project */
+            if (projectDefinition.TS.algorithmModules !== undefined) {
+                for (let j = 0; j < projectDefinition.TS.algorithmModules.length; j++) {
+                    let algorithmModuleDefinition = projectDefinition.TS.algorithmModules[j]
+                    let path = global.env.PATH_TO_PROJECTS_REQUIRED + '/' + projectDefinition.name + '/' + 'TS' + '/' + 'Bot-Modules' + '/' + algorithmModuleDefinition.folderName + '/' + algorithmModuleDefinition.fileName
+
+                    let requiredObject = require(path)
+                    projectInstance.algorithmModules[algorithmModuleDefinition.propertyName] = requiredObject
+                }
+            }
         }
 
         TS.projects.superalgos.globals.taskConstants.PROJECTS_SCHEMA = PROJECTS_SCHEMA