Merge pull request #1655 from Luis-Fernando-Molina/develop

Contribution: Foundations Project: Client Code Mantainance @ HTTP Interface

.vscode/Superalgos.code-workspace

@@ -0,0 +1,10 @@
+{
+	"folders": [
+		{
+			"path": "C:\\Superalgos"
+		}
+	],
+	"settings": {
+		"search.maintainFileSearchCache": true
+	}
+}

Client/WebServer/suite-dummy.html

@@ -1,54 +0,0 @@
-<h2 id="trading-engine">Trading Engine</h2>
-
-<table class="definitionTable"><tr><td>
-
-<img src="images/icons/nodes/png150/trading-engine.png" />
-
-</td><td>
-
-<strong>The trading engine hierarchy is the data structure used by the trading bot to keep runtime information highly accessible and exposed to others.</strong>
-
-</td></tr></table>
-
-<p>You will use this hierarchy for two main purposes:</p>
-
-<p>The hierarchy exposes—literally—all the information processed by the trading bot, providing a comprehensive feedback loop to trading systems and analysis capabilities to users. The hierarchy exposes—literally—all the information processed by the trading bot, providing a comprehensive feedback loop to trading systems and analysis capabilities to users.</p>
-
-<ul>
-  <li>
-    <p><strong>To let your trading systems:</strong> This allows strategies to keep track of and react to current and past events—including those involving the exchange, such as orders placed or filled—as the bot is running.</p>
-  </li>
-  <li>
-    <p><strong>To keep track of the actions:</strong> By analyzing runtime information, you may gain a detailed understanding of what happens, when, and why, throughout a trading session.</p>
-  </li>
-</ul>
-
-<p>The hierarchy exposes—literally—all the information processed by the trading bot, providing a comprehensive feedback loop to trading systems and analysis capabilities to users.</p>
-
-<div class="bs-callout bs-callout-success">Este callout se usa para resaltar una idea particularmente importante. Algo se sobresale, como una conclusion, o algo por el estilo.</div>
-
-<p>The hierarchy exposes all the information processed by the trading bot, providing a comprehensive feedback loop to trading systems and analysis capabilities to users. The hierarchy exposes all the information processed by the trading bot, providing a comprehensive feedback loop to trading systems and analysis capabilities to users.</p>
-
-<div class="alert alert-info" role="alert"><i class="fa fa-info-circle"></i> <b>Note:</b> Esto es una nota, algo noteworthy.</div>
-
-<div class="alert alert-success" role="alert"><i class="fa fa-check-square-o"></i> <b>Tip:</b> Esto es un tip.</div>
-
-<div class="alert alert-warning" role="alert"><i class="fa fa-warning"></i> <b>Important:</b> Esto es algo importante.</div>
-
-<div class="alert alert-danger" role="alert"><i class="fa fa-exclamation-circle"></i> <b>Warning:</b> Esto es una aviso de alerta.</div>
-
-<h4 id="adding-a-trading-engine-node">Adding a Trading Engine Node</h4>
-
-<p>Calculo que con este segundo nivel de titulo alcanza.</p>
-
-<h4 id="este-es-otro-formato-tipico-de-la-doc--trading-engine">Este es otro formato tipico de la doc:  Trading Engine</h4>
-
-<table class="definitionTable"><tr><td>
-
-<img src="images/icons/nodes/png50/trading-engine.png" />
-
-</td><td>
-
-The trading engine hierarchy is the data structure used by the trading bot to keep runtime information highly accessible and exposed to others.
-
-</td></tr></table>

Client/client.js

@@ -6,7 +6,7 @@ global.env = ENVIRONMENT_MODULE
 
 process.on('uncaughtException', function (err) {
     if (err.message && err.message.indexOf("EADDRINUSE") > 0) {
-        console.log("A Superalgos Client cannot be started. Reason: the port configured migth be being used by another application.")
+        console.log("A Superalgos Client cannot be started. Reason: the port configured migth be being used by another application, or Superalgos might be already running.")
         return
     }
     console.log('[ERROR] Client -> client-> uncaughtException -> err.message = ' + err.message)
@@ -97,7 +97,10 @@ try {
     */
     SA.nodeModules = {
         fs: require('fs'),
-        nodeFetch: require('node-fetch')
+        nodeFetch: require('node-fetch'),
+        open: require('open'),
+        http: require('http'),
+        ccxt: require('ccxt')
     }
     /*
     Setting up servers running inside this Client.
@@ -162,13 +165,11 @@ try {
 
     WEB_SOCKETS_INTERFACE = WEB_SOCKETS_INTERFACE.newWebSocketsInterface()
     WEB_SOCKETS_INTERFACE.initialize()
-    WEB_SOCKETS_INTERFACE.run()
     console.log('Web Sockets Interface ....................................... Listening at port ' + global.env.CLIENT_WEB_SOCKETS_INTERFACE_PORT)
 
     HTTP_INTERFACE = HTTP_INTERFACE.newHttpInterface()
     HTTP_INTERFACE.initialize()
-    HTTP_INTERFACE.run()
-    console.log('Http Interface .............................................. Listening at port ' + global.env.HTTP_INTERFACE_PORT)
+    console.log('Http Interface .............................................. Listening at port ' + global.env.CLIENT_HTTP_INTERFACE_PORT)
 
     console.log('')
     console.log("You are running Superalgos Beta 11")

Client/webSocketsInterface.js

@@ -4,8 +4,7 @@
 
     let thisObject = {
         initialize: initialize,
-        finalize: finalize,
-        run: run
+        finalize: finalize
     }
 
     const WEB_SOCKET = require('ws')
@@ -15,15 +14,11 @@
     return thisObject
 
     function finalize() {
-
+        socketServer = undefined
     }
 
     function initialize() {
-
-    }
-
-    function run() {
-       setUpWebSocketServer()
+        setUpWebSocketServer()
     }
 
     function setUpWebSocketServer() {

Desktop/httpInterface.js

@@ -0,0 +1,52 @@
+exports.newHttpInterface = function newHttpInterface() {
+
+    /*
+    This module represent the HTTP API of the 
+    Desktop App. All HTTP request are processed
+    by this module.
+    */
+    let thisObject = {
+        initialize: initialize,
+        finalize: finalize
+    }
+
+    return thisObject
+
+    function finalize() {
+    }
+
+    function initialize() {
+        /*
+        We will create an HTTP Server and leave it running forever.
+        */
+       SA.nodeModules.http.createServer(onHttpRequest).listen(global.env.CLIENT_HTTP_INTERFACE_PORT)
+       /* Starting the browser now is optional */
+       if (process.argv.includes("noBrowser")) {
+           //Running Client only with no UI.
+       } else {
+           SA.nodeModules.open('http://localhost:' + global.env.CLIENT_HTTP_INTERFACE_PORT)
+       }
+    }
+
+    function onHttpRequest(httpRequest, httpResponse) {
+        try {
+            let requestPathAndParameters = httpRequest.url.split('?') // Remove version information
+            let requestPath = requestPathAndParameters[0].split('/')
+            let endpointOrFile = requestPath[1]
+
+            switch (endpointOrFile) {
+                case 'XXX':
+                    {
+
+                    }
+                    break
+                default:
+                    {
+                        SA.projects.foundations.utilities.httpResponses.respondWithWebFile(httpResponse, endpointOrFile,  global.env.PATH_TO_CLIENT)
+                    }
+            }
+        } catch (err) {
+            console.log(err.stack)
+        }
+    }
+}

Environment.js

@@ -3,7 +3,7 @@ exports.newEnvironment = function () {
     let thisObject = {
         WEB_SERVER_URL: 'localhost',
         CLIENT_WEB_SOCKETS_INTERFACE_PORT: 18041,
-        HTTP_INTERFACE_PORT: 34248,
+        CLIENT_HTTP_INTERFACE_PORT: 34248,
         PATH_TO_TASK_SERVER: './TaskServer',
         PATH_TO_DATA_STORAGE: './Data-Storage',
         PATH_TO_PROJECTS: './Projects',

EnvironmentForDebug.js

@@ -4,7 +4,8 @@ exports.newEnvironment = function () {
         WEB_SERVER_URL: 'localhost',
         CLIENT_WEB_SOCKETS_INTERFACE_PORT: 18041,
         NETWORK_WEB_SOCKETS_INTERFACE_PORT: 18042,
-        HTTP_INTERFACE_PORT: 34248,
+        CLIENT_HTTP_INTERFACE_PORT: 34248,
+        DESKTOP_HTTP_INTERFACE_PORT: 34249,
         PATH_TO_TASK_SERVER: './TaskServer',
         PATH_TO_DATA_STORAGE: './Data-Storage',
         PATH_TO_PROJECTS: './Projects',

Projects/Foundations/SA/Utilities/FilesAndDirectories.js

@@ -0,0 +1,54 @@
+exports.newFoundationsUtilitiesFilesAndDirectories = function () {
+
+    let thisObject = {
+        getDirectories: getDirectories,
+        getAllFilesInDirectoryAndSubdirectories: getAllFilesInDirectoryAndSubdirectories
+    }
+
+    return thisObject
+
+    function getDirectories(path) {
+        try {
+            const fs = SA.nodeModules.fs
+            return fs.readdirSync(path).filter(function (file) {
+                return fs.statSync(path + '/' + file).isDirectory();
+            });
+        } catch (err) {
+            return []
+        }
+    }
+
+    function getAllFilesInDirectoryAndSubdirectories(dir, callback) {
+        const { promisify } = require('util');
+        const { resolve } = require('path');
+        const fs = SA.nodeModules.fs;
+        const readdir = promisify(fs.readdir);
+        const stat = promisify(fs.stat);
+
+        getFiles(dir)
+            .then(files => {
+                let splittedDir = dir.split('/')
+                let lastFolder = splittedDir[splittedDir.length - 2]
+                let pathAndNames = []
+                for (let i = 0; i < files.length; i++) {
+                    let file = files[i]
+                    let pathName = file.substring(file.indexOf(lastFolder) + lastFolder.length, file.length)
+                    pathName = pathName.substring(1, pathName.length)
+                    pathAndNames.push(pathName)
+                }
+                callback(pathAndNames)
+            })
+            .catch(e => {
+                callback([])
+            });
+
+        async function getFiles(dir) {
+            const subdirs = await readdir(dir);
+            const files = await Promise.all(subdirs.map(async (subdir) => {
+                const res = resolve(dir, subdir);
+                return (await stat(res)).isDirectory() ? getFiles(res) : res;
+            }));
+            return files.reduce((a, f) => a.concat(f), []);
+        }
+    }
+}

Projects/Foundations/SA/Utilities/HttpRequests.js

@@ -0,0 +1,35 @@
+exports.newFoundationsUtilitiesHttpRequests = function () {
+
+    let thisObject = {
+        getRequestBody: getRequestBody
+    }
+
+    return thisObject
+
+    function getRequestBody(httpRequest, callback) { // Gets the de body from a POST httpRequest to the web server
+        try {
+
+            let body = ''
+
+            httpRequest.on('data', function (data) {
+                body += data
+                // Too much POST data
+                //if (body.length > 1e6) {
+                //    httpRequest.connection.destroy()
+                //}
+            })
+
+            httpRequest.on('end', function () {
+                callback(body)
+            })
+
+            httpRequest.on('error', function (err) {
+                console.log('[ERROR] getBody -> err.stack = ' + err.stack)
+                SA.projects.foundations.utilities.httpResponses.respondWithContent(JSON.stringify(global.DEFAULT_FAIL_RESPONSE), httpResponse)
+            })
+        } catch (err) {
+            console.log('[ERROR] getBody -> err.stack = ' + err.stack)
+            SA.projects.foundations.utilities.httpResponses.respondWithContent(JSON.stringify(global.DEFAULT_FAIL_RESPONSE), httpResponse)
+        }
+    }
+}

Projects/Foundations/SA/Utilities/HttpResponses.js

@@ -0,0 +1,214 @@
+exports.newFoundationsUtilitiesHttpResponses = function () {
+
+    let thisObject = {
+        respondWithFile: respondWithFile,
+        respondWithContent: respondWithContent,
+        respondWithImage: respondWithImage,
+        respondWithFont: respondWithFont,
+        respondWithEmptyArray: respondWithEmptyArray,
+        respondWithStyleSheet: respondWithStyleSheet,
+        respondWithProjectFolderFileList: respondWithProjectFolderFileList,
+        respondWithWebFile: respondWithWebFile
+    }
+
+    return thisObject
+
+    function respondWithFile(fileName, httpResponse) {
+        let fs = SA.nodeModules.fs
+        if (fileName.indexOf('undefined') > 0) {
+            console.log('[WRN] respondWithFile -> Received httpRequest for undefined file. ')
+            respondWithContent(undefined, httpResponse)
+        } else {
+            try {
+                fs.readFile(fileName, onFileRead)
+
+                function onFileRead(err, file) {
+                    if (!err) {
+                        respondWithContent(file.toString(), httpResponse)
+                    } else {
+                        //console.log('File requested not found: ' + fileName)
+                        respondWithContent(undefined, httpResponse)
+                    }
+                }
+            } catch (err) {
+                respondWithEmptyArray(httpResponse)
+            }
+        }
+    }
+
+    function respondWithContent(content, httpResponse, contentType) {
+        try {
+            httpResponse.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate') // HTTP 1.1.
+            httpResponse.setHeader('Pragma', 'no-cache') // HTTP 1.0.
+            httpResponse.setHeader('Expires', '0') // Proxies.
+            httpResponse.setHeader('Access-Control-Allow-Origin', '*') // Allows to access data from other domains.
+
+            if (content !== undefined) {
+                if (contentType !== undefined) {
+                    httpResponse.writeHead(200, { 'Content-Type': contentType })
+                } else {
+                    httpResponse.writeHead(200, { 'Content-Type': 'text/html' })
+                }
+                httpResponse.write(content)
+            } else {
+                httpResponse.writeHead(404, { 'Content-Type': 'text/html' })
+                httpResponse.write('The specified key does not exist.')
+            }
+            httpResponse.end('\n')
+        } catch (err) {
+            respondWithEmptyArray(httpResponse)
+        }
+    }
+
+    function respondWithImage(fileName, httpResponse) {
+        let fs = SA.nodeModules.fs
+        fs.readFile(fileName, onFileRead)
+
+        function onFileRead(err, file) {
+            if (err) {
+                console.log('[ERROR] respondWithImage -> Image Not Found: ' + fileName + ' or Error = ' + err.stack)
+                return
+            }
+
+            httpResponse.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate') // HTTP 1.1.
+            httpResponse.setHeader('Pragma', 'no-cache') // HTTP 1.0.
+            httpResponse.setHeader('Expires', '0') // Proxies.
+            httpResponse.setHeader('Access-Control-Allow-Origin', '*') // Allows to access data from other domains.
+
+            httpResponse.writeHead(200, { 'Content-Type': 'image/png' })
+            httpResponse.end(file, 'binary')
+        }
+    }
+
+    function respondWithFont(fileName, httpResponse) {
+        let fs = SA.nodeModules.fs
+        fs.readFile(fileName, onFileRead)
+
+        function onFileRead(err, file) {
+            if (err) {
+                console.log('[ERROR] respondWithFont -> File Not Found: ' + fileName + ' or Error = ' + err.stack)
+                return
+            }
+            httpResponse.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate') // HTTP 1.1.
+            httpResponse.setHeader('Pragma', 'no-cache') // HTTP 1.0.
+            httpResponse.setHeader('Expires', '0') // Proxies.
+            httpResponse.setHeader('Access-Control-Allow-Origin', '*') // Allows to access data from other domains.
+
+            if (fileName.indexOf('2') < 0) {
+                httpResponse.writeHead(200, { 'Content-Type': 'font/woff' })
+            } else {
+                httpResponse.writeHead(200, { 'Content-Type': 'font/woff2' })
+            }
+            httpResponse.end(file, 'binary')
+        }
+    }
+
+    function respondWithEmptyArray(httpResponse) {
+        httpResponse.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate') // HTTP 1.1.
+        httpResponse.setHeader('Pragma', 'no-cache') // HTTP 1.0.
+        httpResponse.setHeader('Expires', '0') // Proxies.
+
+        httpResponse.writeHead(200, { 'Content-Type': 'text/html' })
+        httpResponse.write('[]')
+        httpResponse.end('\n')
+    }
+
+    function respondWithStyleSheet(httpResponse, path, fileName) {
+        let fs = SA.nodeModules.fs
+        let filePath = path + 'WebServer/css/' + fileName
+        fs.readFile(filePath, onFileRead)
+
+        function onFileRead(err, file) {
+            try {
+                let fileContent = file.toString()
+
+                SA.projects.foundations.utilities.httpResponses.respondWithContent(fileContent, httpResponse, 'text/css')
+            } catch (err) {
+                console.log('[ERROR] respondWithStyleSheet -> File Not Found: ' + fileName + ' or Error = ' + err.stack)
+            }
+        }
+    }
+
+    function respondWithProjectFolderFileList(httpResponse, projectFolderName, rootObjectName) {
+        {
+            let allLibraries = []
+            let projects = SA.projects.foundations.utilities.filesAndDirectories.getDirectories(global.env.PATH_TO_PROJECTS)
+            let projectsCount = 0
+
+            for (let i = 0; i < projects.length; i++) {
+                let project = projects[i]
+
+                let dirPath = global.env.PATH_TO_PROJECTS + '/' + project + '/' + rootObjectName + '/' + projectFolderName
+                try {
+                    let fs = SA.nodeModules.fs
+                    fs.readdir(dirPath, onDirRead)
+
+                    function onDirRead(err, fileList) {
+                        let updatedFileList = []
+                        if (err) {
+                            /*
+                            If we have a problem reading this folder we will assume that it is
+                            because this project does not need this folder and that's it.
+                            */
+                        } else {
+                            for (let i = 0; i < fileList.length; i++) {
+                                let name = fileList[i]
+                                updatedFileList.push([project, name])
+                            }
+                        }
+                        allLibraries = allLibraries.concat(updatedFileList)
+                        projectsCount++
+                        if (projectsCount === projects.length) {
+                            SA.projects.foundations.utilities.httpResponses.respondWithContent(JSON.stringify(allLibraries), httpResponse)
+                        }
+                    }
+                } catch (err) {
+                    console.log('[ERROR] returnProjectFolderFileList -> Error reading a directory content. filePath = ' + dirPath)
+                    console.log('[ERROR] returnProjectFolderFileList -> err.stack = ' + err.stack)
+
+                    SA.projects.foundations.utilities.httpResponses.respondWithContent(JSON.stringify(global.DEFAULT_FAIL_RESPONSE), httpResponse)
+                    return
+                }
+            }
+        }
+    }
+
+    function respondWithWebFile(httpResponse, fileNameReceived, path) {
+
+        if (fileNameReceived === '') {
+            /*
+            When there is no endpoint specidied we will respond with this app's Home Page.
+            */
+            let fs = SA.nodeModules.fs
+
+            let fileName = path + 'WebServer/index.html'
+            fs.readFile(fileName, onFileRead)
+
+            function onFileRead(err, file) {
+
+                let fileContent = file.toString()
+
+                SA.projects.foundations.utilities.httpResponses.respondWithContent(fileContent, httpResponse)
+            }
+        } else {
+            /*
+            When there is a parameter but it does not match any of the available endpoints, we 
+            will serve the file with the same name at at a location that depends on it's file extention. 
+            */
+            let completeFileName = fileNameReceived
+            let fileExtension = completeFileName.split('.')[1]
+
+            switch (fileExtension) {
+                case 'js': {
+                    let path = global.env.PATH_TO_UI + '/' + completeFileName
+                    SA.projects.foundations.utilities.httpResponses.respondWithFile(path, httpResponse)
+                    break
+                }
+                case 'css': {
+                    SA.projects.foundations.utilities.httpResponses.respondWithStyleSheet(httpResponse, path, completeFileName)
+                    break
+                }
+            }
+        }
+    }
+}

Projects/Foundations/TS/Bot-Modules/Sensor-Bot/Webhooks/CheckWebhook.js

@@ -90,7 +90,7 @@ exports.newFoundationsBotModulesCheckWebhook = function (processIndex) {
                 try {
                     let http = require('http');
                     let url = 'http://' + global.env.WEB_SERVER_URL +
-                        ':' + global.env.HTTP_INTERFACE_PORT +
+                        ':' + global.env.CLIENT_HTTP_INTERFACE_PORT +
                         '/Webhook/Fetch-Messages/' +
                         TS.projects.foundations.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.parentNode.parentNode.config.codeName + "/" +
                         TS.projects.foundations.globals.taskConstants.TASK_NODE.parentNode.parentNode.parentNode.referenceParent.baseAsset.referenceParent.config.codeName + "-" + 

Projects/ProjectsSchema.json

@@ -900,6 +900,24 @@
                     "propertyName": "miscellaneousFunctions",
                     "functionName": "newFoundationsUtilitiesMiscellaneousFunctions",
                     "fileName": "MiscellaneousFunctions.js"
+                },
+                {
+                    "name": "Files And Directories",
+                    "propertyName": "filesAndDirectories",
+                    "functionName": "newFoundationsUtilitiesFilesAndDirectories",
+                    "fileName": "FilesAndDirectories.js"
+                },
+                {
+                    "name": "Http Responses",
+                    "propertyName": "httpResponses",
+                    "functionName": "newFoundationsUtilitiesHttpResponses",
+                    "fileName": "HttpResponses.js"
+                },
+                {
+                    "name": "Http Requests",
+                    "propertyName": "httpRequests",
+                    "functionName": "newFoundationsUtilitiesHttpRequests",
+                    "fileName": "HttpRequests.js"
                 }
             ]
         },