Bug fixes that improve resilience when running live

AppSchema.json

@@ -805,12 +805,19 @@
         "type": "Condition",
         "docURL": "https://docs.superalgos.org/suite-hierarchy-trading-system.html#condition",
         "menuItems": [
+            {
+                "action": "Edit Condition",
+                "actionFunction": "uiObject.conditionEditor.activate",
+                "label": "Edit Condition",
+                "iconPathOn": "html",
+                "iconPathOff": "html"
+            },
             {
                 "action": "Add UI Object",
                 "disableIfPropertyIsDefined": true,
                 "propertyToCheckFor": "javascriptCode",
                 "label": "Add Code",
-                "relatedUiObject": "Javascript Code",
+                "relatedUiObject": "Condition",
                 "actionFunction": "payload.onMenuItemClick"
             },
             {
@@ -4020,12 +4027,14 @@
             {
                 "name": "positionSize",
                 "type": "node",
-                "childType": "Position Size"
+                "childType": "Position Size",
+                "autoAdd": true
             },
             {
                 "name": "positionRate",
                 "type": "node",
-                "childType": "Position Rate"
+                "childType": "Position Rate",
+                "autoAdd": true
             }
         ],
         "propertyNameAtParent": "initialDefinition"
@@ -4379,7 +4388,8 @@
             {
                 "name": "formula",
                 "type": "node",
-                "childType": "Formula"
+                "childType": "Formula",
+                "autoAdd": true
             }
         ],
         "propertyNameAtParent": "positionSize"
@@ -4416,7 +4426,8 @@
             {
                 "name": "formula",
                 "type": "node",
-                "childType": "Formula"
+                "childType": "Formula",
+                "autoAdd": true
             }
         ],
         "propertyNameAtParent": "positionRate"

BackendServers/WebServer/AppLoader.js

@@ -94,6 +94,7 @@ function newAppLoader() {
                 'FloatingSpace/ConfigEditor.js',
                 'FloatingSpace/ConditionEditor.js',
                 'FloatingSpace/FormulaEditor.js',
+                'FloatingSpace/Picker.js',
 
                 'Scales/RateScale.js',
                 'Scales/TimeScale.js',

CanvasApp/DesignSpace/Workspace/Workspace.js

@@ -152,6 +152,17 @@ function newWorkspace () {
   }
 
   function saveWorkspace () {
+    /*  When there is an exception while loading the app, the rootNodes of the workspace get into null value. To avoid saving a corrupt staate we are going to verufy we are not in that situation before saving. */
+    let workspace = canvas.designSpace.workspace.workspaceNode
+
+    for (let i = 0; i < workspace.rootNodes.length; i++) {
+      let rootNode = workspace.rootNodes[i]
+      if (rootNode === null) {
+        canvas.cockpitSpace.setStatus('Could not save the Workspace. The state of the workspace in memory is corrupt, please reload the App.', 150, canvas.cockpitSpace.statusTypes.WARNING)
+        return
+      }
+    }
+
     let savedSessionTimestamp = window.localStorage.getItem('Session Timestamp')
     if (Number(savedSessionTimestamp) !== sessionTimestamp) {
       canvas.cockpitSpace.setStatus('Could not save the Workspace. You have more that one instance of the Superlagos User Interface open at the same time. Plese close this instance as it is older than the others.', 150, canvas.cockpitSpace.statusTypes.WARNING)

CanvasApp/FloatingSpace/CodeEditor.js

@@ -4,8 +4,6 @@ function newCodeEditor () {
 
   let thisObject = {
     isVisibleFunction: undefined,
-    iconOK: undefined,
-    iconNOT_OK: undefined,
     visible: false,
     imagePathOK: undefined,
     imagePathNOT_OK: undefined,
@@ -41,29 +39,12 @@ function newCodeEditor () {
   function finalize () {
     thisObject.container.finalize()
     thisObject.container = undefined
-    thisObject.iconOK = undefined
-    thisObject.iconNOT_OK = undefined
     thisObject.payload = undefined
     thisObject.isVisibleFunction = undefined
   }
 
   function initialize () {
-    /* Load Images */
 
-    thisObject.iconOK = new Image()
-    thisObject.iconOK.onload = onImageLoad
-
-    function onImageLoad () {
-      thisObject.iconNOT_OK = new Image()
-      thisObject.iconNOT_OK.onload = onImageLoad
-
-      function onImageLoad () {
-        thisObject.canDrawIcon = true
-      }
-      thisObject.iconNOT_OK.src = window.canvasApp.urlPrefix + thisObject.imagePathNOT_OK
-    }
-    // thisObject.iconOK.src = window.canvasApp.urlPrefix + thisObject.imagePathOK
-    // thisObject.icon = thisObject.iconOK // The default value is ON.
   }
 
   function deactivate () {

CanvasApp/FloatingSpace/ConditionEditor.js

@@ -4,8 +4,6 @@ function newConditionEditor () {
 
   let thisObject = {
     isVisibleFunction: undefined,
-    iconOK: undefined,
-    iconNOT_OK: undefined,
     visible: false,
     imagePathOK: undefined,
     imagePathNOT_OK: undefined,
@@ -34,43 +32,25 @@ function newConditionEditor () {
   thisObject.container.frame.position.y = 0
 
   let isMouseOver = false
+  let operatorA = {}
+  let operatorB = {}
 
   return thisObject
 
   function finalize () {
     thisObject.container.finalize()
     thisObject.container = undefined
-    thisObject.iconOK = undefined
-    thisObject.iconNOT_OK = undefined
     thisObject.payload = undefined
     thisObject.isVisibleFunction = undefined
   }
 
   function initialize () {
-    /* Load Images */
 
-    thisObject.iconOK = new Image()
-    thisObject.iconOK.onload = onImageLoad
-
-    function onImageLoad () {
-      thisObject.iconNOT_OK = new Image()
-      thisObject.iconNOT_OK.onload = onImageLoad
-
-      function onImageLoad () {
-        thisObject.canDrawIcon = true
-      }
-      thisObject.iconNOT_OK.src = window.canvasApp.urlPrefix + thisObject.imagePathNOT_OK
-    }
-    // thisObject.iconOK.src = window.canvasApp.urlPrefix + thisObject.imagePathOK
-    // thisObject.icon = thisObject.iconOK // The default value is ON.
   }
 
   function deactivate () {
     if (thisObject.visible === true) {
       thisObject.visible = false
-      let textArea = document.getElementById('textArea')
-      textArea.style.display = 'none'
-      thisObject.payload.node.code = textArea.value
     }
     EDITOR_ON_FOCUS = false
   }
@@ -83,24 +63,72 @@ function newConditionEditor () {
     thisObject.currentRadius = 0
     thisObject.payload.uiObject.setErrorMessage('', 0)
 
-    let textArea = document.getElementById('textArea')
-    textArea.value = payload.node.code
-    textArea.style = 'resize: none;' +
-                     ' border: none;' +
-                     ' outline: none;' +
-                     'box-shadow: none;' +
-                     'overflow:hidden;' +
-                     'font-family: ' + UI_FONT.PRIMARY + ';' +
-                     'font-size: 14px;' +
-                     'background-color: rgb(' + UI_COLOR.GREEN + ');' +
-                     'color:rgb(255, 255, 255);' +
-                    'width: ' + thisObject.container.frame.width + 'px;' +
-                     'height: ' + thisObject.container.frame.height + 'px'
-    textArea.style.display = 'block'
-    textArea.focus()
+    scanDataMines()
+    setUpPickers()
     EDITOR_ON_FOCUS = true
   }
 
+  function scanDataMines () {
+    let selector = {}
+
+    let workspace = canvas.designSpace.workspace.workspaceNode
+
+    for (let i = 0; i < workspace.rootNodes.length; i++) {
+      let rootNode = workspace.rootNodes[i]
+      if (rootNode.type !== 'Data Mine') { continue }
+      let dataMine = rootNode
+      let dataMineName = loadPropertyFromNodeConfig(dataMine.payload, 'codeName')
+      selector[dataMineName] = {}
+      let bots = dataMine.sensorBots.concat(dataMine.indicatorBots)
+      for (let j = 0; j < bots.length; j++) {
+        let bot = bots[j]
+        let botName = loadPropertyFromNodeConfig(bot.payload, 'codeName')
+        let selectorDataMine = selector[dataMineName]
+        selectorDataMine[botName] = {}
+        for (let k = 0; k < bot.products.length; k++) {
+          let product = bot.products[k]
+          let productName = loadPropertyFromNodeConfig(product.payload, 'singularVariableName')
+          if (productName === undefined) { continue }
+          let selectorProduct = selectorDataMine[botName]
+          selectorProduct[productName] = {}
+          if (product.record === undefined) { continue }
+          for (let m = 0; m < product.record.properties.length; m++) {
+            let property = product.record.properties[m]
+            let propertyName = loadPropertyFromNodeConfig(property.payload, 'codeName')
+            let selectorProperty = selectorProduct[productName]
+            selectorProperty[propertyName] = {}
+            let possibleValues = loadPropertyFromNodeConfig(property.payload, 'possibleValues')
+            if (possibleValues === undefined) { possibleValues = [] }
+            let selectorPossibleValue = selectorProperty[propertyName]
+            selectorPossibleValue.possibleValues = possibleValues
+          }
+          let productKeys = Object.keys(selectorProduct[productName])
+          if (productKeys.length === 0) {
+            selectorProduct[productName] = undefined
+          }
+        }
+        let botKeys = Object.keys(selectorDataMine[botName])
+        if (botKeys.length === 0) {
+          selectorDataMine[botName] = undefined
+        }
+      }
+      let dataMineKeys = Object.keys(selector[dataMineName])
+      if (dataMineKeys.length === 0) {
+        selector[dataMineName] = undefined
+      }
+    }
+    operatorA.selector = JSON.parse(JSON.stringify(selector))
+    operatorB.selector = JSON.parse(JSON.stringify(selector))
+  }
+
+  function setUpPickers () {
+    operatorA.dataMinePicker = newPicker()
+    operatorA.dataMinePicker.container.connectToParent(thisObject.container)
+    operatorA.dataMinePicker.container.frame.position.x = 0 - operatorA.dataMinePicker.container.frame.width / 2
+    operatorA.dataMinePicker.container.frame.position.y = 0 - operatorA.dataMinePicker.container.frame.height / 2
+    operatorA.dataMinePicker.initialize(Object.keys(operatorA.selector))
+  }
+
   function getContainer (point) {
     let container
     if (thisObject.visible === true) {
@@ -120,6 +148,17 @@ function newConditionEditor () {
   }
 
   function physics () {
+    thisObjectphysics()
+    operatorsPhysics()
+  }
+
+  function operatorsPhysics () {
+    if (operatorA.dataMinePicker !== undefined) {
+      operatorA.dataMinePicker.physics()
+    }
+  }
+
+  function thisObjectphysics () {
     if (Math.abs(thisObject.currentRadius - thisObject.targetRadius) >= 0.5) {
       if (thisObject.currentRadius < thisObject.targetRadius) {
         thisObject.currentRadius = thisObject.currentRadius + 0.5
@@ -131,31 +170,22 @@ function newConditionEditor () {
     thisObject.container.frame.position.x = 0
     thisObject.container.frame.position.y = 0
 
-    thisObject.container.frame.width = thisObject.container.frame.radius * 1.8 * 2
+    thisObject.container.frame.width = thisObject.container.frame.radius * 1 * 2
     thisObject.container.frame.height = thisObject.container.frame.radius * 1 * 2
-
-    let textAreaPosition = {
-      x: 0 - thisObject.container.frame.width / 2,
-      y: 0 - thisObject.container.frame.height * 3 / 7 + CURRENT_TOP_MARGIN
-    }
-
-    textAreaPosition = thisObject.container.frame.frameThisPoint(textAreaPosition)
-    if (thisObject.visible === true) {
-      let checkPosition = {
-        x: textAreaPosition.x,
-        y: textAreaPosition.y - CURRENT_TOP_MARGIN
-      }
-      if (thisObject.isVisibleFunction(checkPosition) === false) {
-        deactivate()
-      }
-    }
-    if (thisObject.visible === true) {
-      let textAreaDiv = document.getElementById('textAreaDiv')
-      textAreaDiv.style = 'position:fixed; top:' + textAreaPosition.y + 'px; left:' + textAreaPosition.x + 'px; z-index:1; '
-    }
   }
 
   function drawBackground () {
+    thisObjectDrawBackground()
+    childrenDrawBackground()
+  }
+
+  function childrenDrawBackground () {
+    if (operatorA.dataMinePicker !== undefined) {
+      operatorA.dataMinePicker.drawBackground()
+    }
+  }
+
+  function thisObjectDrawBackground () {
     if (thisObject.visible === true) {
       let position = {
         x: 0,
@@ -164,7 +194,7 @@ function newConditionEditor () {
 
       position = thisObject.container.frame.frameThisPoint(position)
 
-      let radius = thisObject.container.frame.radius * 2
+      let radius = thisObject.container.frame.radius * 1
 
       if (radius > 0.5) {
         browserCanvasContext.beginPath()
@@ -176,15 +206,27 @@ function newConditionEditor () {
         browserCanvasContext.beginPath()
         browserCanvasContext.arc(position.x, position.y, radius * 1.3, 0, Math.PI * 2, true)
         browserCanvasContext.closePath()
-        browserCanvasContext.fillStyle = 'rgba(' + UI_COLOR.GREEN + ', ' + 1 + ')'
+        browserCanvasContext.fillStyle = 'rgba(' + UI_COLOR.GREY + ', ' + 1 + ')'
         browserCanvasContext.fill()
       }
     }
   }
+
   function drawForeground () {
+    thisObjectDrawForeground()
+    childrenDrawForeground()
+  }
+
+  function childrenDrawForeground () {
+    if (operatorA.dataMinePicker !== undefined) {
+      operatorA.dataMinePicker.drawForeground()
+    }
+  }
+
+  function thisObjectDrawForeground () {
     let iconPosition = {
       x: 0,
-      y: thisObject.currentRadius * 1.5
+      y: thisObject.currentRadius * 1
     }
 
     iconPosition = thisObject.container.frame.frameThisPoint(iconPosition)

CanvasApp/FloatingSpace/ConfigEditor.js

@@ -4,8 +4,6 @@ function newConfigEditor () {
 
   let thisObject = {
     isVisibleFunction: undefined,
-    iconOK: undefined,
-    iconNOT_OK: undefined,
     visible: false,
     imagePathOK: undefined,
     imagePathNOT_OK: undefined,
@@ -41,29 +39,12 @@ function newConfigEditor () {
   function finalize () {
     thisObject.container.finalize()
     thisObject.container = undefined
-    thisObject.iconOK = undefined
-    thisObject.iconNOT_OK = undefined
     thisObject.payload = undefined
     thisObject.isVisibleFunction = undefined
   }
 
   function initialize () {
-    /* Load Images */
 
-    thisObject.iconOK = new Image()
-    thisObject.iconOK.onload = onImageLoad
-
-    function onImageLoad () {
-      thisObject.iconNOT_OK = new Image()
-      thisObject.iconNOT_OK.onload = onImageLoad
-
-      function onImageLoad () {
-        thisObject.canDrawIcon = true
-      }
-      thisObject.iconNOT_OK.src = window.canvasApp.urlPrefix + thisObject.imagePathNOT_OK
-    }
-    // thisObject.iconOK.src = window.canvasApp.urlPrefix + thisObject.imagePathOK
-    // thisObject.icon = thisObject.iconOK // The default value is ON.
   }
 
   function deactivate () {

CanvasApp/FloatingSpace/FormulaEditor.js

@@ -4,8 +4,6 @@ function newFormulaEditor () {
 
   let thisObject = {
     isVisibleFunction: undefined,
-    iconOK: undefined,
-    iconNOT_OK: undefined,
     visible: false,
     imagePathOK: undefined,
     imagePathNOT_OK: undefined,
@@ -41,29 +39,12 @@ function newFormulaEditor () {
   function finalize () {
     thisObject.container.finalize()
     thisObject.container = undefined
-    thisObject.iconOK = undefined
-    thisObject.iconNOT_OK = undefined
     thisObject.payload = undefined
     thisObject.isVisibleFunction = undefined
   }
 
   function initialize () {
-    /* Load Images */
 
-    thisObject.iconOK = new Image()
-    thisObject.iconOK.onload = onImageLoad
-
-    function onImageLoad () {
-      thisObject.iconNOT_OK = new Image()
-      thisObject.iconNOT_OK.onload = onImageLoad
-
-      function onImageLoad () {
-        thisObject.canDrawIcon = true
-      }
-      thisObject.iconNOT_OK.src = window.canvasApp.urlPrefix + thisObject.imagePathNOT_OK
-    }
-    // thisObject.iconOK.src = window.canvasApp.urlPrefix + thisObject.imagePathOK
-    // thisObject.icon = thisObject.iconOK // The default value is ON.
   }
 
   function deactivate () {

CanvasApp/FloatingSpace/Picker.js

@@ -0,0 +1,62 @@
+
+function newPicker () {
+  const MODULE_NAME = 'Picker'
+
+  let thisObject = {
+    container: undefined,
+    selectedItem: 0,
+    physics: physics,
+    drawBackground: drawBackground,
+    drawForeground: drawForeground,
+    getContainer: getContainer,
+    finalize: finalize,
+    initialize: initialize
+  }
+
+  thisObject.container = newContainer()
+  thisObject.container.initialize(MODULE_NAME)
+  thisObject.container.isClickeable = false
+  thisObject.container.isDraggeable = false
+  thisObject.container.frame.radius = 0
+  thisObject.container.frame.position.x = 0
+  thisObject.container.frame.position.y = 0
+  thisObject.container.frame.width = 50
+  thisObject.container.frame.height = 100
+
+  let optionsList
+  return thisObject
+
+  function finalize () {
+    thisObject.container.finalize()
+    thisObject.container = undefined
+    optionsList = undefined
+  }
+
+  function initialize (pOptionsList) {
+    optionsList = pOptionsList
+  }
+
+  function getContainer (point) {
+    let container
+
+    if (thisObject.container.frame.isThisPointHere(point, true, false) === true) {
+      return thisObject.container
+    }
+  }
+
+  function physics () {
+
+  }
+
+  function drawBackground () {
+
+  }
+
+  function drawForeground () {
+    const FONT_SIZE = 15
+
+    for (let i = 0; i < optionsList.length; i++) {
+      drawLabel(optionsList[i], 1 / 2, i / optionsList.length, 0, 0, FONT_SIZE, thisObject.container, UI_COLOR.WHITE, undefined, undefined)
+    }
+  }
+}

Data-Mines/Masters.json

@@ -1 +1 @@
-{"type":"Data Mine","name":"Masters","code":"{\n\"codeName\": \"Masters\"\n}","sensorBots":[{"type":"Sensor Bot","name":"Exchange Raw Data","code":"{\n\"codeName\": \"Exchange-Raw-Data\",\n\"repo\": \"Exchange-Raw-Data-Sensor-Bot\"\n}","processes":[{"type":"Process Definition","name":"Historic-Trades","code":"{\n    \"codeName\": \"Historic-Trades\",\n    \"description\": \"Retrieves and saves the historical trades in batches going backwards from the current time until reaching the begining of the market.\",\n    \"startMode\": {\n        \"noTime\": {\n            \"run\": \"true\"\n        },\n        \"fixedInterval\": {\n            \"run\": \"false\",\n            \"interval\": 0\n        }\n    },\n    \"deadWaitTime\": 0,\n    \"normalWaitTime\": 60000,\n    \"retryWaitTime\": 10000\n}","processOutput":{"type":"Process Output","name":"New Process Output","outputDatasets":[],"id":"e3008f15-43ae-4452-9c28-bfbb6c1a5e84","savedPayload":{"position":{"x":13889.892304785908,"y":17976.7326423862},"targetPosition":{"x":13788.99537092664,"y":18353.285125876908},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":4,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","statusDependencies":[{"type":"Status Dependency","name":"Self Reference","code":"{\n    \"mainUtility\": \"Self Reference\"\n}","id":"f796022a-7871-4bde-bcc7-bcf2818f1eaf","savedPayload":{"position":{"x":14299.325525762739,"y":17993.12379366583},"targetPosition":{"x":13938.17907937083,"y":17993.12379366583},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":1,"distanceToParent":3,"arrangementStyle":0},"uiObject":{},"referenceParent":{"type":"Status Report","name":"New Status Report","id":"116834de-6646-4387-9f4b-4f711cea9235"}}}],"dataDependencies":[],"id":"1332b41e-7cd2-4e69-a40a-39ea47df67b5","savedPayload":{"position":{"x":13938.17907937083,"y":17993.12379366583},"targetPosition":{"x":13788.99537092664,"y":18353.285125876908},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":4,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"statusReport":{"type":"Status Report","name":"New Status Report","id":"116834de-6646-4387-9f4b-4f711cea9235","savedPayload":{"position":{"x":13983.913279342734,"y":18015.677405195172},"targetPosition":{"x":13788.99537092664,"y":18353.285125876908},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":4,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionStartedEvent":{"type":"Execution Started Event","name":"New Execution Started Event","id":"a9ea7169-fcd8-48c3-b0a9-b4e8941aa170","savedPayload":{"position":{"x":14026.312379860185,"y":18044.00757842793},"targetPosition":{"x":13788.99537092664,"y":18353.285125876908},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":4,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionFinishedEvent":{"type":"Execution Finished Event","name":"New Execution Finished Event","id":"51908884-c7a2-46ca-8a55-209070ae5fa6","savedPayload":{"position":{"x":14064.650920558077,"y":18077.62957624547},"targetPosition":{"x":13788.99537092664,"y":18353.285125876908},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":4,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"id":"7a6a9399-474f-4a93-bcbd-28a594ef1943","savedPayload":{"position":{"x":14351.93363503598,"y":18425.400341491753},"targetPosition":{"x":13888.04337325749,"y":18696.024787447222},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":3,"distanceToParent":2,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Process Definition","name":"Historic-OHLCVs","code":"{\n    \"codeName\": \"Historic-OHLCVs\",\n    \"description\": \"Retrieves and saves historical maket data\",\n    \"startMode\": {\n        \"noTime\": {\n            \"run\": \"true\"\n        },\n        \"fixedInterval\": {\n            \"run\": \"false\",\n            \"interval\": 0\n        }\n    },\n    \"deadWaitTime\": 0,\n    \"normalWaitTime\": 60000,\n    \"retryWaitTime\": 10000\n}","processOutput":{"type":"Process Output","name":"New Process Output","outputDatasets":[],"id":"dfe2556e-a765-4e38-9d30-7f91090c0069","savedPayload":{"position":{"x":14546.018334597831,"y":18039.39680205973},"targetPosition":{"x":14407.127481538195,"y":18558.43550715096},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","statusDependencies":[{"type":"Status Dependency","name":"Exchange Raw Data Historic-OHLCVs","code":"{\n    \"mainUtility\": \"Self ReferencLine truncated
+{"type":"Data Mine","name":"Masters","code":"{\n\"codeName\": \"Masters\"\n}","sensorBots":[{"type":"Sensor Bot","name":"Exchange Raw Data","code":"{\n\"codeName\": \"Exchange-Raw-Data\",\n\"repo\": \"Exchange-Raw-Data-Sensor-Bot\"\n}","processes":[{"type":"Process Definition","name":"Historic-Trades","code":"{\n    \"codeName\": \"Historic-Trades\",\n    \"description\": \"Retrieves and saves the historical trades in batches going backwards from the current time until reaching the begining of the market.\",\n    \"startMode\": {\n        \"noTime\": {\n            \"run\": \"true\"\n        },\n        \"fixedInterval\": {\n            \"run\": \"false\",\n            \"interval\": 0\n        }\n    },\n    \"deadWaitTime\": 0,\n    \"normalWaitTime\": 60000,\n    \"retryWaitTime\": 10000\n}","processOutput":{"type":"Process Output","name":"New Process Output","outputDatasets":[],"id":"e3008f15-43ae-4452-9c28-bfbb6c1a5e84","savedPayload":{"position":{"x":13889.892304785908,"y":17976.7326423862},"targetPosition":{"x":13788.99537092664,"y":18353.285125876908},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":4,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","statusDependencies":[{"type":"Status Dependency","name":"Self Reference","code":"{\n    \"mainUtility\": \"Self Reference\"\n}","id":"f796022a-7871-4bde-bcc7-bcf2818f1eaf","savedPayload":{"position":{"x":14299.325525762739,"y":17993.12379366583},"targetPosition":{"x":13938.17907937083,"y":17993.12379366583},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":1,"distanceToParent":3,"arrangementStyle":0},"uiObject":{},"referenceParent":{"type":"Status Report","name":"New Status Report","id":"116834de-6646-4387-9f4b-4f711cea9235"}}}],"dataDependencies":[],"id":"1332b41e-7cd2-4e69-a40a-39ea47df67b5","savedPayload":{"position":{"x":13938.17907937083,"y":17993.12379366583},"targetPosition":{"x":13788.99537092664,"y":18353.285125876908},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":4,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"statusReport":{"type":"Status Report","name":"New Status Report","id":"116834de-6646-4387-9f4b-4f711cea9235","savedPayload":{"position":{"x":13983.913279342734,"y":18015.677405195172},"targetPosition":{"x":13788.99537092664,"y":18353.285125876908},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":4,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionStartedEvent":{"type":"Execution Started Event","name":"New Execution Started Event","id":"a9ea7169-fcd8-48c3-b0a9-b4e8941aa170","savedPayload":{"position":{"x":14026.312379860185,"y":18044.00757842793},"targetPosition":{"x":13788.99537092664,"y":18353.285125876908},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":4,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"executionFinishedEvent":{"type":"Execution Finished Event","name":"New Execution Finished Event","id":"51908884-c7a2-46ca-8a55-209070ae5fa6","savedPayload":{"position":{"x":14064.650920558077,"y":18077.62957624547},"targetPosition":{"x":13788.99537092664,"y":18353.285125876908},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":4,"distanceToParent":3,"arrangementStyle":0},"uiObject":{}}},"id":"7a6a9399-474f-4a93-bcbd-28a594ef1943","savedPayload":{"position":{"x":14351.93363503598,"y":18425.400341491753},"targetPosition":{"x":13888.04337325749,"y":18696.024787447222},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":true,"angleToParent":3,"distanceToParent":2,"arrangementStyle":0},"uiObject":{"isRunning":false}}},{"type":"Process Definition","name":"Historic-OHLCVs","code":"{\n    \"codeName\": \"Historic-OHLCVs\",\n    \"description\": \"Retrieves and saves historical maket data\",\n    \"startMode\": {\n        \"noTime\": {\n            \"run\": \"true\"\n        },\n        \"fixedInterval\": {\n            \"run\": \"false\",\n            \"interval\": 0\n        }\n    },\n    \"deadWaitTime\": 0,\n    \"normalWaitTime\": 60000,\n    \"retryWaitTime\": 10000\n}","processOutput":{"type":"Process Output","name":"New Process Output","outputDatasets":[],"id":"dfe2556e-a765-4e38-9d30-7f91090c0069","savedPayload":{"position":{"x":14546.018334597831,"y":18039.39680205973},"targetPosition":{"x":14407.127481538195,"y":18558.43550715096},"floatingObject":{"isPinned":false,"isFrozen":false,"isCollapsed":false,"angleToParent":2,"distanceToParent":3,"arrangementStyle":0},"uiObject":{"isRunning":false}}},"processDependencies":{"type":"Process Dependencies","name":"New Process Dependencies","statusDependencies":[{"type":"Status Dependency","name":"Exchange Raw Data Historic-OHLCVs","code":"{\n    \"mainUtility\": \"Self ReferencLine truncated

README.md

@@ -1,3 +1,7 @@
+# Contact Us
+
+Please join our Telegram Community to contact us there: https://t.me/superalgoscommunity
+
 # Superalgos
 
 Superalgos provides a toolbox with many integrated tools that could help you research crypto markets and design / automate trading strategies. You can use Superalgos to download market data from crypto exchanges, make arbitrary calculations with it (including indicators), plot the results of your calculations, design, test and automate trading strategies, and more. 

TaskServer/DebugLog.js

@@ -17,7 +17,8 @@ exports.newDebugLog = function newDebugLog() {
         write: write,
         newInternalLoop: newInternalLoop,
         persist: persist,           // This method is executed at the end of each Main Loop.
-        initialize: initialize
+        initialize: initialize,
+        finalize: finalize
     };
 
     let accumulatedLog = "[";
@@ -37,6 +38,10 @@ exports.newDebugLog = function newDebugLog() {
         }
     }
 
+    function finalize() {
+        persist()
+    }
+
     function newInternalLoop(pBot, pProcess, date, percentage) {
         if (percentage === undefined) {
             percentage = ""
@@ -64,6 +69,7 @@ exports.newDebugLog = function newDebugLog() {
         /* Here we actually write the content of the in-memory log to a file */
 
         try {
+            if (accumulatedLog === "[") {return} // nothing to persist at the moment.
 
             internalLoopCounter++;
 

TaskServer/ExchangeAPI.js

@@ -38,7 +38,7 @@
             'secret': secret,
             'timeout': 30000,
             'enableRateLimit': true,
-            verbose: true,
+            verbose: false,
             options: options
         }
 

TaskServer/FileStorage.js

@@ -58,7 +58,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) {
                     fs.readFile(fileLocation, onFileRead)
                 } else {
                     /* We read the file via a web server over http */
-                    getFileViaHTTP(fileLocation, onFileRead)
+                    getFileViaHTTP(fileLocation, onFileRead, callBackFunction)
                 }
 
                 function onFileRead(err, text) {
@@ -130,7 +130,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) {
                                 fs.readFile(fileLocation + '.Previous.json', onPreviousFileRead)
                             } else {
                                 /* We read the file via a web server over http */
-                                getFileViaHTTP(fileLocation + '.Previous.json', onPreviousFileRead)
+                                getFileViaHTTP(fileLocation + '.Previous.json', onPreviousFileRead, callBackFunction)
                             }
 
                             function onPreviousFileRead(err, text) {
@@ -435,7 +435,7 @@ exports.newFileStorage = function newFileStorage(logger, host, port) {
     }, initDir);
     }
 
-    function getFileViaHTTP(filePath, callback) {
+    function getFileViaHTTP(filePath, callback, callBackFunction) {
         try {
 
             /* The filePath received is the one that is needed to fetch data from with fs. To do it via http we need to remove the prefix that includes this: ./Data-Storage/  */
@@ -447,36 +447,49 @@ exports.newFileStorage = function newFileStorage(logger, host, port) {
                 '/Storage/' +
                 filePath
 
-            http.get(url, onResponse);
+            logger.write(MODULE_NAME, '[INFO] FileStorage -> getFileViaHTTP -> url = ' + url)
+
+            let request = http.get(url, onResponse);
+
+            request.on('error', function (err) {
+                logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onError -> err = " + err.stack);
+                logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onError -> Failed to fetch file via HTTP. Will retry later. ");
+                callBackFunction(global.DEFAULT_RETRY_RESPONSE);
+            }) 
 
             function onResponse(response) {
 
-                const chunks = []
+                try {
+                    const chunks = []
 
-                response.on('data', onMessegesArrived)
-                response.on('end', onEnd)
+                    response.on('data', onMessegesArrived)
+                    response.on('end', onEnd)
 
-                function onMessegesArrived(chunk) {
-                    chunks.push(chunk)
-                }
-
-                function onEnd() {
-                    let fileContent = Buffer.concat(chunks).toString('utf8')
-                    let err = null
-                    if (fileContent.indexOf('does not exist') >= 0) {
-                        err = {
-                            code: "ENOENT" // This is how fs would have returned upon this situation.
-                        }
-                        fileContent = undefined
+                    function onMessegesArrived(chunk) {
+                        chunks.push(chunk)
                     }
-                    callback(err, fileContent)
+
+                    function onEnd() {
+                        let fileContent = Buffer.concat(chunks).toString('utf8')
+                        let err = null
+                        if (fileContent.indexOf('does not exist') >= 0) {
+                            err = {
+                                code: "ENOENT" // This is how fs would have returned upon this situation.
+                            }
+                            fileContent = undefined
+                        }
+                        callback(err, fileContent)
+                    }
+                } catch (err) {
+                    logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onResponse -> err = " + err.stack);
+                    logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> onResponse -> Failed to fetch file via HTTP. Will retry later. ");
+                    callBackFunction(global.DEFAULT_RETRY_RESPONSE);
                 }
             }
-
         } catch (err) {
             logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> err = " + err.stack);
-            callBackFunction(global.DEFAULT_FAIL_RESPONSE);
-            abort = true
+            logger.write(MODULE_NAME, "[ERROR] getFileViaHTTP -> Failed to fetch file via HTTP. Will retry later. ");
+            callBackFunction(global.DEFAULT_RETRY_RESPONSE);
         }
     }
 }

TaskServer/IndicatorBotProcessMainLoop.js

@@ -116,7 +116,11 @@
 
                     /* For each loop we want to create a new log file. */
 
+                    if (logger !== undefined) {
+                        logger.finalize()
+                    }
                     logger = DEBUG_MODULE.newDebugLog();
+                    global.LOGGER_MAP.set(MODULE_NAME, logger)
                     logger.bot = bot;
                     logger.initialize();
 

TaskServer/MultiPeriod.js

@@ -124,7 +124,7 @@
                     thisReport = statusReport.file;
 
                     if (thisReport.lastFile === undefined) {
-                        logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> reportKey = " + reportKey);
+                        logger.write(MODULE_NAME, "[WARN] start -> getContextVariables -> Undefined Last File. -> thisReport = " + JSON.stringify(thisReport));
 
                         let customOK = {
                             result: global.CUSTOM_OK_RESPONSE.result,

TaskServer/Root.js

@@ -391,6 +391,7 @@
                             let logger;
 
                             logger = DEBUG_MODULE.newDebugLog();
+                            global.LOGGER_MAP.set('runSensorBot', logger)
                             logger.bot = pBotConfig;
 
                             if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runSensorBot -> Entering function."); }
@@ -455,6 +456,7 @@
                             let logger;
 
                             logger = DEBUG_MODULE.newDebugLog();
+                            global.LOGGER_MAP.set('runIndicatorBot', logger)
                             logger.bot = pBotConfig;
 
                             if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runIndicatorBot -> Entering function."); }
@@ -517,6 +519,7 @@
                             let logger;
 
                             logger = DEBUG_MODULE.newDebugLog();
+                            global.LOGGER_MAP.set('runTradingBot', logger)
                             logger.bot = pBotConfig;
 
                             if (FULL_LOG === true) { logger.write(MODULE_NAME, "[INFO] start -> bootingBot -> runTradingBot -> Entering function."); }

TaskServer/SensorBotProcessMainLoop.js

@@ -121,7 +121,11 @@
 
                     /* For each loop we want to create a new log file. */
 
+                    if (logger !== undefined) {
+                        logger.finalize()
+                    }
                     logger = DEBUG_MODULE.newDebugLog();
+                    global.LOGGER_MAP.set(MODULE_NAME, logger)
                     logger.bot = bot;
                     logger.initialize();
 

TaskServer/Session.js

@@ -39,6 +39,7 @@
 
             /* Listen to event to start or stop the session. */
             bot.sessionKey = bot.processNode.session.name + '-' + bot.processNode.session.type + '-' + bot.processNode.session.id
+            global.SESSION_MAP.set(bot.sessionKey, bot.sessionKey)
  
             global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Run Session', undefined, bot.sessionKey, undefined, runSession)
             global.EVENT_SERVER_CLIENT.listenToEvent(bot.sessionKey, 'Stop Session', undefined, bot.sessionKey, undefined, stopSession)

TaskServer/TradingBotProcessMainLoop.js

@@ -124,8 +124,11 @@
                     }
 
                     /* For each loop we want to create a new log file. */
-
+                    if (logger !== undefined) {
+                        logger.finalize()
+                    }
                     logger = DEBUG_MODULE.newDebugLog();
+                    global.LOGGER_MAP.set(MODULE_NAME, logger)
                     logger.bot = bot;
                     logger.initialize();
 

TaskServer/server.js

@@ -26,19 +26,37 @@ global.CUSTOM_FAIL_RESPONSE = {
 };
 
 const ONE_DAY_IN_MILISECONDS = 24 * 60 * 60 * 1000;
+global.LOGGER_MAP = new Map()
+global.SESSION_MAP = new Map()
 
 process.on('uncaughtException', function (err) {
     console.log('[ERROR] Task Server -> server -> uncaughtException -> err.message = ' + err.message)
     console.log('[ERROR] Task Server -> server -> uncaughtException -> err.stack = ' + err.stack)
-    process.exit(1)
+    global.EXIT_NODE_PROCESS()
 })
 
 process.on('unhandledRejection', (reason, p) => {
     console.log('[ERROR] Task Server -> server -> unhandledRejection -> reason = ' + JSON.stringify(reason))
     console.log('[ERROR] Task Server -> server -> unhandledRejection -> p = ' + JSON.stringify(p))
-    process.exit(1)
+    global.EXIT_NODE_PROCESS()
 })
 
+function finalizeLoggers() {
+    global.LOGGER_MAP.forEach(forEachLogger)
+
+    function forEachLogger(logger) {
+        logger.finalize()
+    }
+}
+
+function finalizeSessions() {
+    global.SESSION_MAP.forEach(forEachSession)
+
+    function forEachSession(session) {
+        global.EVENT_SERVER_CLIENT.raiseEvent(session, 'Stopped')
+    }
+}
+
 process.on('exit', function (code) {
 
     if (global.TASK_NODE !== undefined) {
@@ -69,8 +87,16 @@ process.on('message', message => {
     } 
 });
 
+
+let shuttingDownProcess = false
 global.EXIT_NODE_PROCESS = function exitProcess() {
 
+    if (shuttingDownProcess === true) { return }
+    shuttingDownProcess = true
+
+    /* Signal that all sessions are stopping. */
+    finalizeSessions()
+
     /* Cleaning Before Exiting. */
     clearInterval(global.HEARTBEAT_INTERVAL_HANDLER)
 
@@ -82,6 +108,7 @@ global.EXIT_NODE_PROCESS = function exitProcess() {
         global.EVENT_SERVER_CLIENT.raiseEvent(key, 'Stopped') // Meaning Process Stopped
     }
 
+    finalizeLoggers()
     //console.log("[INFO] Task Server -> " + global.TASK_NODE.name + " -> EXIT_NODE_PROCESS -> Task Server will stop in 10 seconds.");
 
     setTimeout(process.exit, 10000) // We will give 10 seconds to logs be written on file