From 5bf5aa5600f0e91d41903ffc44fc6bf69412331e Mon Sep 17 00:00:00 2001 From: ssplatt Date: Wed, 22 Sep 2021 21:09:36 -0400 Subject: [PATCH] getting started --- .dockerignore | 9 ++-- Docker/Dockerfile | 54 +++++++++++++++++------- Docker/install.js | 105 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 41 ++++++++++++++---- 4 files changed, 181 insertions(+), 28 deletions(-) create mode 100644 Docker/install.js diff --git a/.dockerignore b/.dockerignore index b15d17eb1..0fa7f056d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,7 +9,10 @@ debug.log .vscode README.md LICENSE -CONTRIBUITING.md +CONTRIBUTING.md CODE_OF_CONDUCT.md -./**/.vscode -Dockerfile +*/**/.vscode +*/**/Dockerfile +*/**/docker-compose.yml +*/**/node_modules +.env diff --git a/Docker/Dockerfile b/Docker/Dockerfile index ee38a11a6..236292da5 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -1,25 +1,47 @@ -FROM python:3-slim -ENV NODE_VERSION=node_16.x -ENV PYTHONUNBUFFERED 1 -RUN apt-get update \ - && apt-get install -y curl gnupg lsb-release build-essential \ - && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \ - && DISTRO="$(lsb_release -s -c)" \ - && echo "deb https://deb.nodesource.com/$NODE_VERSION $DISTRO main" | tee /etc/apt/sources.list.d/nodesource.list \ - && echo "deb-src https://deb.nodesource.com/$NODE_VERSION $DISTRO main" | tee -a /etc/apt/sources.list.d/nodesource.list \ - && apt-get update \ - && apt-get install -y nodejs \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* +FROM node:16-alpine as BASE + +FROM BASE as BUILDER +RUN apk add --no-cache python3 make g++ curl \ + && curl -sf https://gobinaries.com/tj/node-prune | sh + WORKDIR /app COPY . . +WORKDIR /app/Platform +RUN npm ci --only=production && node-prune +WORKDIR /app/Projects/Foundations/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data +RUN npm ci --only=production && node-prune +WORKDIR /app/Projects/Foundations/TS/Bot-Modules/API-Data-Fetcher-Bot +RUN npm ci --only=production && node-prune +WORKDIR /app/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Announcements +RUN npm ci --only=production && node-prune +WORKDIR /app/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/APIs +RUN npm ci --only=production && node-prune +WORKDIR /app/Projects/Foundations/TS/Task-Modules +RUN npm ci --only=production && node-prune +WORKDIR /app/Projects/TensorFlow/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning +RUN npm ci --only=production && node-prune + +FROM BASE +ENV PYTHONUNBUFFERED 1 +RUN apk add --no-cache python3 +WORKDIR /app +COPY . . +COPY --from=BUILDER /app/Platform/node_modules /app/Platform/node_modules +COPY --from=BUILDER /app/Projects/Foundations/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/node_modules /app/Projects/Foundations/TS/Bot-Modules/Sensor-Bot/Exchange-Raw-Data/node_modules +COPY --from=BUILDER /app/Projects/Foundations/TS/Bot-Modules/API-Data-Fetcher-Bot/node_modules /app/Projects/Foundations/TS/Bot-Modules/API-Data-Fetcher-Bot/node_modules +COPY --from=BUILDER /app/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Announcements/node_modules /app/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Announcements/node_modules +COPY --from=BUILDER /app/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/APIs/node_modules /app/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/APIs/node_modules +COPY --from=BUILDER /app/Projects/Foundations/TS/Task-Modules/node_modules /app/Projects/Foundations/TS/Task-Modules/node_modules +COPY --from=BUILDER /app/Projects/TensorFlow/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/node_modules /app/Projects/TensorFlow/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/node_modules + RUN mkdir -p Data-Storage Log-Files My-Workspaces \ - && groupadd superalgos \ - && useradd --no-log-init -m -g superalgos superalgos \ + && addgroup superalgos \ + && adduser --disabled-password --no-create-home --ingroup superalgos superalgos \ && chown -R superalgos:superalgos /app + USER superalgos -RUN node setup noShortcuts EXPOSE 34248 EXPOSE 18041 VOLUME ["/app/Data-Storage", "/app/Log-Files", "/app/My-Workspaces"] ENTRYPOINT ["node", "platform", "noBrowser"] + diff --git a/Docker/install.js b/Docker/install.js new file mode 100644 index 000000000..628d55230 --- /dev/null +++ b/Docker/install.js @@ -0,0 +1,105 @@ +const path = require("path"); +const process = require("process"); +const { exec } = require("child_process"); +const systemCheck = require('./Launch-Scripts/system-check') +const platform = require('os').platform(); // Possible values are: 'aix', 'darwin', 'freebsd', 'linux', 'openbsd', 'sunos', and 'win32'. + +// Check system is set up correctly +systemCheck(); + +// Handle adding shortcuts +if (process.argv.includes("noShortcuts")) { + // Cancel running script if flag provided + console.log(''); + console.log('noShortcuts ................................................... Setting up without shortcuts.') + +} else { + // Run create-shortcuts script + try { + const { fork } = require('child_process') + fork('./Launch-Scripts/create-shortcut.js') + } catch (err) { + console.log('') + console.log(err) + console.log('') + + } +} + +// Create Operating System compatable paths to each node_modules directory. +let nodeModulesDirs = [ + path.join( process.cwd(), "Platform"), + path.join( process.cwd(), "Projects", "Foundations", "TS", "Bot-Modules", "Sensor-Bot", "Exchange-Raw-Data"), + path.join( process.cwd(), "Projects", "Foundations", "TS", "Bot-Modules", "API-Data-Fetcher-Bot"), + path.join( process.cwd(), "Projects", "Foundations", "TS", "Bot-Modules", "Trading-Bot", "Announcements"), + path.join( process.cwd(), "Projects", "Foundations", "TS", "Bot-Modules", "Trading-Bot", "Low-Frequency-Trading", "APIs"), + path.join( process.cwd(), "Projects", "Foundations", "TS", "Task-Modules") +]; + +if ( + process.argv.includes("TensorFlow") || + process.argv.includes("tensorflow") || + process.argv.includes("--TensorFlow") || + process.argv.includes("-TensorFlow") || + process.argv.includes("--tensorflow") || + process.argv.includes("--tensorflow") + ) { + + nodeModulesDirs = [ + path.join( process.cwd(), "Projects", "TensorFlow", "TS", "Bot-Modules", "Learning-Bot", "Low-Frequency-Learning") + ] +} + +// Check if WinOS because extra dependencies may be required for tfjs: +var tfjsWinInstallFlag = false; +if (platform === 'win32') { // *Note: win32 == win32 || win64 + tfjsWinInstallFlag = true; +} + +console.log(''); +console.log("Installing node dependencies at the following directories:"); +console.log(''); +console.log(nodeModulesDirs); +console.log(''); + +for (let dir of nodeModulesDirs) { + // Loop through directories and run npm ci in a child process shell. + let command = "echo Results of install at "+ dir + " & npm ci"; + exec( command, + { + cwd: dir + }, + function ( error, stdout ){ + if (error) { + console.log(''); + console.log("There was an error installing some dependencies error: "); + console.log(''); + console.log( error ); + if (tfjsWinInstallFlag == true && dir == path.join(process.cwd(), "Projects", "TensorFlow", "TS", "Bot-Modules", "Learning-Bot", "Low-Frequency-Learning")) { + tfjsWinInstall(); + } + return; + } + console.log(''); + console.log( stdout ); + }); +}; + + +/* If WinOS && error with tfjs installation: + * This is a well known and documented issue with @Tensorflow/tfjs-node installation on WinOS. + * npm's windows-build-tools via: `npm install --global --production windows-build-tools` && + * `npm install -g node-gyp` can sometimes correct the issue but not always depending on the + * version and options the user defined when installing npm/nodejs. + * The best reliable way to correct that I (Pluvtech) have found to work on many versions of WinOS + * is to reinstall nodejs and select to install chocolatey during the installation configuration. + */ +function tfjsWinInstall() { + console.log("\n*** Note to Windows Users Regarding the Superalgos TensorFlow Project ***"); + console.log("If you have no other errors with this setup script and do not wish to use the TensorFlow Project:\nThen all is successful with your Superalgos setup and you may proceed to use the software."); + console.log("\nIf you do wish to install the @Tensorflow/tfjs-node modules required for use with the Superalgos TensorFlow Project:"); + console.log("In this case it is likely your system is missing some dependencies required to build the TensorFlow (tfjs-node) bindings and you will need to follow the below steps:"); + console.log("\n1.) Reinstall node.js from https://nodejs.org/en/download/ @Version 14.17.5 or higher.\nDuring installation, when you arrive at the 'Tools for Native Modules' screen you need to select the box to:\n\t'Automatically install the necessary tools...Chocolatey...'"); + console.log("Doing this will cause a pop-up script in a new window which will install many npm/node tools and dependencies for use within the WinOS ecosystem."); + console.log("2.) When the installations finally complete you need to navigate back to your Superalgos directory and run `node setup` again.\n"); +} \ No newline at end of file diff --git a/package.json b/package.json index e59e25f2d..da01a05a9 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,34 @@ { - "dependencies": { - "@octokit/rest": "^18.10.0", - "http": "0.0.1-security", - "node-fetch": "^2.6.1", - "open": "^8.2.1", - "simple-git": "^2.45.1", - "web3": "^1.5.2", - "ws": "^8.2.1" - } + "name": "Superalgos-Plarform", + "version": "0.0.11", + "description": "Superalgos Plarform", + "main": "PlatformApp.js", + "author": { + "name": "Luis-Fernando-Molina" + }, + "dependencies": { + "ip": "^1.1.5", + "debug": "^4.3.1", + "minimist": "^1.2.5", + "module-alias": "^2.2.2", + "ms": "^2.1.2", + "sandwich-stream": "^2.0.2", + "telegraf": "^3.38.0", + "telegram-typings": "^3.6.1", + "http": "0.0.1-security", + "@octokit/rest": "^18.10.0", + "ccxt": "^1.51.98", + "child_process": "^1.0.2", + "ethereumjs-common": "^1.5.2", + "ethereumjs-tx": "^2.1.2", + "ethers": "^5.4.4", + "lookpath": "^1.2.2", + "node-fetch": "^2.6.1", + "open": "^8.2.1", + "rimraf": "^3.0.2", + "simple-git": "^2.42.0", + "web3": "^1.5.2", + "ws": "^8.2.1", + "@tensorflow/tfjs-node": "^3.8.0" + } }