Adding base version for Operations Module

This commit is contained in:
mati108@gmail.com committed 2018-12-26 19:41:13 -03:00
1 parent ae08794750
commit d397c87d68
64 files changed
+3524 -1

No files matched your search

+31
View File
@@ -0,0 +1,31 @@
{
"apiVersion": "batch/v1",
"kind": "Job",
"metadata": {
"labels": {
"app": "aadev"
},
"name": "aadev-deployment"
},
"spec": {
"template": {
"spec": {
"restartPolicy": "Never",
"containers": [
{
"name": "aadev",
"image": "benme/aadev:v3.2",
"command": ["node", "Run.js"],
"env": [
]
}
]
},
"metadata": {
"labels": {
"app": "aadev"
}
}
}
}
}
+47
View File
@@ -0,0 +1,47 @@
import 'dotenv/config'
const env = []
env.push({
"name": "PLATFORM_ENVIRONMENT",
"value": process.env.PLATFORM_ENVIRONMENT
})
// FIXME
env.push({
"name": "GATEWAY_ENDPOINT",
"value": "https://platform-api-dev.advancedalgos.net/graphql"
//"value": process.env.GATEWAY_ENDPOINT
})
env.push({
"name": "STORAGE_BASE_URL",
"value": process.env.STORAGE_BASE_URL
})
env.push({
"name": "STORAGE_CONNECTION_STRING",
"value": process.env.STORAGE_CONNECTION_STRING
})
env.push({
"name": "AUTH_URL",
"value": process.env.AUTH_URL
})
env.push({
"name": "AUTH_CLIENT_ID",
"value": process.env.AUTH_CLIENT_ID
})
env.push({
"name": "AUTH_CLIENT_SECRET",
"value": process.env.AUTH_CLIENT_SECRET
})
env.push({
"name": "AUTH_AUDIENCE",
"value": process.env.AUTH_AUDIENCE
})
export default env
+42
View File
@@ -0,0 +1,42 @@
import { createLogger, format, transports } from 'winston'
const { combine, timestamp, simple } = format
import { inspect } from 'util'
const options = {
file: {
level: 'info',
filename: '../logs/app.log',
handleExceptions: true,
json: true,
maxsize: 5242880, // 5MB
maxFiles: 5,
colorize: false,
timestamp: true
},
console: {
level: 'debug',
handleExceptions: true,
json: false,
colorize: true,
timestamp: true
}
}
const logger = createLogger({
level: 'info',
format: format.combine(
format.colorize(),
format.splat(),
format.timestamp({ format: 'YY/MM/DD HH:mm' }),
format.printf(
info => `${info.timestamp} - ${info.level}: ${JSON.parse(JSON.stringify(inspect(info.message)))}`
)
),
transports: [
new transports.File(options.file),
new transports.Console(options.console)
],
exitOnError: false // do not exit on handled exceptions
})
export default logger
+9
View File
@@ -0,0 +1,9 @@
export function toPlatformDatetime (epoch) {
const platformDate = new Date(0)
platformDate.setUTCSeconds(epoch)
return platformDate.toISOString()
}
export function isDefined (d) {
return d !== null && typeof d !== 'undefined'
}
+11
View File
@@ -0,0 +1,11 @@
export const LIVE = 'Live';
export const BACKTEST = 'Backtest';
export const COMPETITION = 'Competition';
export const CloneModeEnum = [
LIVE,
BACKTEST,
COMPETITION
];
export default CloneModeEnum;
+19
View File
@@ -0,0 +1,19 @@
export const UNPUBLISHED = 'UNPUBLISHED';
export const PUBLISHED = 'PUBLISHED';
export const CANCELLED = 'CANCELLED';
export const RUNNING = 'RUNNING';
export const INTERRUPTED = 'INTERRUPTED';
export const FINISHED = 'FINISHED';
export const ARCHIVED = 'ARCHIVED';
export const CloneStateEnum = [
UNPUBLISHED,
PUBLISHED,
CANCELLED,
RUNNING,
INTERRUPTED,
FINISHED,
ARCHIVED,
];
export default CloneStateEnum;
+66
View File
@@ -0,0 +1,66 @@
import logger from '../config/logger'
export class AuthentificationError extends Error {
code = 401
message = 'Autentification not found, you have to be authentificated to perform this action'
constructor(){
super()
logger.error(this.stack)
}
}
export class DatabaseError extends Error {
code = 404
message = 'Ressource not found : ' + this.message
constructor(message){
super(message)
logger.error(this.stack)
}
}
export class WrongArgumentsError extends Error {
code = 400
message = 'Wrong arguments : ' + this.message
constructor(message){
super(message)
logger.error(this.stack)
}
}
export class ServiceUnavailableError extends Error {
code = 503
message = 'At least one service is unresponding ' + this.message
constructor(message){
super(message)
logger.error(this.stack)
}
}
export class OperationsError extends Error {
code = 500
message = 'OperationsModule Internal Error. ' + this.message
constructor(message){
super(message)
logger.error(this.stack)
this.stack = ''
}
}
export class KubernateError extends Error {
code = 500
message = 'OperationsModule Kubernate Error. ' + this.message
constructor(message){
super(message)
logger.error(this.stack)
}
}
export class CustomError extends Error {
code = 500
message = this.message
constructor(message){
super(message)
logger.error(this.stack)
this.stack = ''
}
}
+8
View File
@@ -0,0 +1,8 @@
{
"errors": [
{
"code": 401,
"message": "Wait, you're not supposed to be here, go through the gate as everyone else"
}
]
}
+43
View File
@@ -0,0 +1,43 @@
import express from 'express'
import 'dotenv/config'
import graphqlHTTP from 'express-graphql'
import schema from './schema'
import mongoose from 'mongoose'
import cors from 'cors'
import logger from './config/logger'
import wrongPreshared from './errors/notAllowed.json';
const app = express()
app.post('/graphql', (req, res, next) => {
if (req.headers.preshared === process.env.OPERATIONS_API_PRESHARED) {
next();
} else {
res.send(wrongPreshared);
}
});
// Allow cross origing request
app.use(cors())
// Database Connection
mongoose.connect(`mongodb://${process.env.DB_USERNAME}:${process.env.DB_PASSWORD}@${process.env.DB_DOMAIN}`, { useNewUrlParser: true })
mongoose.connection.once('open', () => {
logger.info('Connected to the DB.')
})
app.use('/graphql', graphqlHTTP(req => {
return {
schema: schema,
context: {
userId: req.headers.userid,
authorization: req.headers.authorization
},
graphiql: true
}
}))
app.listen(process.env.PORT, () => {
logger.info(`Listening on port ${process.env.PORT}.`)
})
+99
View File
@@ -0,0 +1,99 @@
import logger from '../config/logger'
import { toPlatformDatetime } from '../config/utils'
import envConfig from '../config/envConfig'
import { KubernateError } from '../errors'
import { Client, config } from 'kubernetes-client'
// Get base Deployment config
import deploymentManifest from '../config/clone-deployment.json'
import { BACKTEST} from '../enums/CloneMode';
const createClone = async (clone, cloneName) => {
try {
logger.info("createClone %j", cloneName)
const client = new Client({config: config.fromKubeconfig(), version: '1.9'})
// Make changes to base deployment config
deploymentManifest.metadata.name = cloneName
logger.debug("createClone Set basic environment configuration.")
let env = envConfig
logger.debug("createClone Configuration for team, bot and process.")
env.push({
"name": "DEV_TEAM",
"value": clone.teamId
})
env.push({
"name": "BOT",
"value": clone.botId
})
// TODO allow other types
env.push({
"name": "PROCESS",
"value": "Trading-Process"
})
// TODO allow other types
env.push({
"name": "TYPE",
"value": "Trading"
})
env.push({
"name": "RUN_AS_TEAM",
"value": clone.runAsTeam.toString()
})
env.push({
"name": "USER_LOGGED_IN",
"value": clone.userLoggedIn
})
logger.debug("createClone Configuration for the execution.")
env.push({
"name": "START_MODE",
"value": clone.mode.toLowerCase()
})
env.push({
"name": "RESUME_EXECUTION",
"value": clone.resumeExecution.toString()
})
if(clone.mode === BACKTEST){
env.push({
"name": "BEGIN_DATE_TIME",
"value": toPlatformDatetime(clone.beginDatetime)
})
env.push({
"name": "END_DATE_TIME",
"value": toPlatformDatetime(clone.endDatetime)
})
env.push({
"name": "WAIT_TIME",
"value": clone.waitTime.toString()
})
}
deploymentManifest.spec.template.spec.containers[0].env = env
// Create a new Deployment
const create = await client.apis.batch.v1.namespaces('default').jobs.post(
{ body: deploymentManifest })
logger.debug("Create: %j ", create)
// Fetch the Deployment we just created
const deployment = await client.apis.batch.v1.namespaces('default').jobs(
deploymentManifest.metadata.name).get()
logger.debug("Job: %j ", deployment)
logger.info("createClone %j succesfull", cloneName)
} catch (err) {
throw new KubernateError(err)
}
}
export default createClone
@@ -0,0 +1,32 @@
import logger from '../config/logger'
import { KubernateError } from '../errors'
import { Client, config } from 'kubernetes-client'
const getClonePodStatus = async (cloneName) => {
try {
logger.info("getClonePodStatus %j", cloneName)
const client = new Client({config: config.fromKubeconfig(), version: '1.9'})
let query = {
"qs":{
"labelSelector": "job-name="+cloneName
}
}
// Get The Pod
const pod = await client.api.v1.namespaces('default').pods.get(query)
logger.debug("pod: %j ", pod)
let clonePodStatus = pod.body.items[0].status.containerStatuses[0].state
logger.debug("clonePodStatus: %j ", clonePodStatus)
if (clonePodStatus.hasOwnProperty('terminated')
&& clonePodStatus.terminated.hasOwnProperty('containerID')) {
delete clonePodStatus.terminated.containerID
}
logger.info("getClonePodStatus %j succesfull", cloneName)
return JSON.stringify(clonePodStatus, null, 2)
} catch (err) {
throw new KubernateError(err)
}
}
export default getClonePodStatus
+34
View File
@@ -0,0 +1,34 @@
import logger from '../config/logger'
import { KubernateError } from '../errors'
import { Client, config } from 'kubernetes-client'
const getClonePodStatus = async (cloneName) => {
try {
logger.info("getClonePodStatus %j", cloneName)
const client = new Client({config: config.fromKubeconfig(), version: '1.9'})
let query = {
"qs":{
"labelSelector": "job-name="+cloneName
}
}
// Get the pod
const pod = await client.api.v1.namespaces('default').pods.get(query)
// logger.debug("pod: %j ", pod)
let queryLogs = {
"qs":{
"tailLines": "15"
}
}
const logs = await client.api.v1.namespaces('default')
.pods(pod.body.items[0].metadata.name).log.get(queryLogs)
let logsArray = logs.body.split('\n')
// logger.debug("logs: %j ", logsArray)
return JSON.stringify(logsArray, null, 2)
} catch (err) {
throw new KubernateError(err)
}
}
export default getClonePodStatus
+32
View File
@@ -0,0 +1,32 @@
import logger from '../config/logger'
import { KubernateError } from '../errors'
import { Client, config } from 'kubernetes-client'
const removeClone = async (cloneName) => {
try {
logger.info("removeClone %j", cloneName)
const client = new Client({config: config.fromKubeconfig(), version: '1.9'})
let query = {
"qs":{
"labelSelector": "job-name="+cloneName
}
}
// Remove the Job
const removed = await client.apis.batch.v1.namespaces('default').jobs(
cloneName).delete()
logger.debug("Job Removed: %j ", removed)
// Get the pod
const pod = await client.api.v1.namespaces('default').pods.get(query)
const podRemoved = await client.api.v1.namespaces('default')
.pods(pod.body.items[0].metadata.name).delete()
logger.debug("Pod Removed: %j ", podRemoved)
logger.info("removeClone %j succesfull", cloneName)
} catch (err) {
throw new KubernateError(err)
}
}
export default removeClone
+47
View File
@@ -0,0 +1,47 @@
import mongoose, { Schema } from 'mongoose'
import {
UNPUBLISHED,
CloneStateEnum,
} from '../enums/CloneState';
import {
BACKTEST,
CloneModeEnum,
} from '../enums/CloneMode';
const cloneSchema = new Schema({
authId: {
type: String,
required: true,
},
teamId: {
type: String,
required: true,
},
botId: {
type: String,
required: true,
},
mode: {
type: String,
enum: CloneModeEnum,
default: BACKTEST,
},
resumeExecution: Boolean,
beginDatetime: Number,
endDatetime: Number,
waitTime: Number,
state: {
type: String,
enum: CloneStateEnum,
default: UNPUBLISHED,
},
stateDatetime: Number,
createDatetime: Number,
runAsTeam: Boolean
})
const Clone = mongoose.model('Clone', cloneSchema)
export default Clone
+5
View File
@@ -0,0 +1,5 @@
import Clone from './clone'
export {
Clone
}
+13
View File
@@ -0,0 +1,13 @@
import {
GraphQLSchema
} from 'graphql'
import Mutation from './mutations'
import RootQuery from './queries'
const Schema = new GraphQLSchema({
query: RootQuery,
mutation: Mutation
})
export default Schema
@@ -0,0 +1,125 @@
import { CloneInputType } from '../types/input';
import { CloneType } from '../types';
import { Clone } from '../../models';
import { CloneModeEnum , BACKTEST} from '../../enums/CloneMode';
import {
AuthentificationError,
WrongArgumentsError,
OperationsError,
CustomError
} from '../../errors'
import axios from 'axios';
import logger from '../../config/logger'
import createKubernetesClone from '../../kubernetes/createClone'
import { isDefined } from '../../config/utils'
const args = {
clone: {type: CloneInputType }
}
const resolve = async(parent, { clone }, context) => {
logger.debug('addClone -> Entering Fuction.')
if (!context.userId) {
throw new AuthentificationError()
}
if(!Object.values(CloneModeEnum).includes(clone.mode)){
throw new WrongArgumentsError('The mode selected is not valid.')
}
// Disable execution dates if mode is not backtest
if((clone.hasOwnProperty('beginDatetime')
|| clone.hasOwnProperty('endDatetime'))
&& clone.mode !== BACKTEST){
throw new WrongArgumentsError('Ony Backtest mode allows begin and end date selection.')
}
let existingClone = await Clone.find(
{
authId: context.userId,
mode: clone.mode
})
logger.debug('addClone -> Checking existing clone %j', existingClone)
if(isDefined(existingClone) && existingClone.length > 0){
throw new CustomError('You can only create one clone per mode. Remove the'
+ ' existing clone of type: ' + clone.mode + ' and try again.')
}
logger.debug('addClone -> Creating a new clone. %j', clone)
try{
const userTeam = await axios({
url: process.env.GATEWAY_ENDPOINT,
method: 'post',
data: {
query: `
query Teams_FbByTeamMember {
teams_FbByTeamMember {
id
name
slug
fb {
id
name
slug
}
members {
member {
alias
}
}
}
}
`,
},
headers: {
authorization: context.authorization,
preshared: process.env.OPERATIONS_API_PRESHARED
}
})
clone.teamId = userTeam.data.data.teams_FbByTeamMember.slug
clone.botId = userTeam.data.data.teams_FbByTeamMember.fb[0].slug
clone.userLoggedIn = userTeam.data.data.teams_FbByTeamMember.members[0].member.alias
clone.createDatetime = new Date().valueOf() / 1000|0
let newClone = new Clone(clone)
await createKubernetesClone(clone, newClone._id)
newClone.id = newClone._id
newClone.authId = context.userId
return new Promise((resolve, reject) => {
newClone.save((err) => {
if (err){
logger.debug('addClone -> Error: %j', err)
reject('There has been an error storing the clone.')
}
else {
//TODO transaction on database
// saveAuditLog(newClone.id, 'addKey', context)
logger.debug('addClone -> Save clone sucessful.')
resolve(newClone)
}
})
})
} catch(error) {
logger.debug('addClone -> Error: %j', error)
throw new OperationsError('There has been an error creating the clone.')
}
}
const mutation = {
addClone: {
type: CloneType,
args,
resolve
}
}
export default mutation
@@ -0,0 +1,54 @@
import {
GraphQLNonNull,
GraphQLID,
GraphQLString
} from 'graphql'
import {
AuthentificationError,
OperationsError
} from '../../errors'
import { Clone } from '../../models'
import logger from '../../config/logger'
import removeKuberneteClone from '../../kubernetes/removeClone'
//import saveAuditLog from './AddAuditLog'
const args = {
id: {type: new GraphQLNonNull(GraphQLID)}
}
const resolve = async (parent, { id }, context) => {
logger.debug('removeClone -> Entering Fuction.')
if (!context.userId) {
throw new AuthentificationError()
}
await removeKuberneteClone(id)
var query = {
_id: id,
authId: context.userId
}
//saveAuditLog(id, 'removeClone', context)
logger.debug('removeClone -> Removing Clone from DB.')
Clone.deleteOne(query, function (err) {
if (err){
throw new OperationsError(err)
}else{
return 'Clone Removed'
}
})
}
const mutation = {
removeClone: {
type: GraphQLString,
args,
resolve
}
}
export default mutation
+16
View File
@@ -0,0 +1,16 @@
import {
GraphQLObjectType
} from 'graphql'
import AddCloneMutation from './AddCloneMutation'
import RemoveCloneMutation from './RemoveCloneMutation'
const Mutation = new GraphQLObjectType({
name: 'Mutation',
fields: Object.assign(
AddCloneMutation,
RemoveCloneMutation
)
})
export default Mutation
@@ -0,0 +1,40 @@
import {
GraphQLList,
GraphQLString,
GraphQLNonNull
} from 'graphql'
import {
AuthentificationError
} from '../../errors'
import { AuditLogType } from '../types'
import { AuditLog } from '../../models'
import logger from '../../config/logger'
const args = {
key: { type: new GraphQLNonNull(GraphQLString) }
}
const resolve = (parent, { key }, context) => {
logger.debug('auditLogs -> Entering Fuction.')
if (!context.userId) {
throw new AuthentificationError()
}
return AuditLog.find({
authId: context.userId,
keyId: key
}).sort({date: -1}).limit(20)
}
const query = {
auditLogs: {
type: new GraphQLList(AuditLogType),
args,
resolve
}
}
export default query
@@ -0,0 +1,23 @@
import {
GraphQLList
} from 'graphql'
import { ExchangeType } from '../types'
import { Exchange } from '../../models'
import logger from '../../config/logger'
const args = {}
const resolve = (parent, args, context) => {
logger.debug('exchanges -> Entering Fuction.')
return Exchange
}
const query = {
exchanges: {
type: new GraphQLList(ExchangeType),
args,
resolve
}
}
export default query
+41
View File
@@ -0,0 +1,41 @@
import {
GraphQLList
} from 'graphql'
import {
AuthentificationError
} from '../../errors'
import { CloneType } from '../types'
import { Clone } from '../../models'
import logger from '../../config/logger'
import getKuberneteClonePodStatus from '../../kubernetes/getClonePodStatus'
import getKuberneteClonePodLogs from '../../kubernetes/getClonePodLogs'
const args = {}
const resolve = async(parent, args, context) => {
logger.debug('List Clones -> Entering Fuction.')
if (!context.userId) {
throw new AuthentificationError()
}
let clones = await Clone.find({authId: context.userId})
for (var i = 0; i < clones.length; i++) {
clones[i].state = await getKuberneteClonePodStatus(clones[i]._id)
clones[i].lastLogs = await getKuberneteClonePodLogs(clones[i]._id)
}
return clones
}
const query = {
clones: {
type: new GraphQLList(CloneType),
args,
resolve
}
}
export default query
+14
View File
@@ -0,0 +1,14 @@
import {
GraphQLObjectType
} from 'graphql'
import ListClones from './ListClones'
const RootQuery = new GraphQLObjectType({
name: 'RootQueryType',
fields: Object.assign(
ListClones
)
})
export default RootQuery
+29
View File
@@ -0,0 +1,29 @@
import {
GraphQLObjectType,
GraphQLID,
GraphQLString,
GraphQLInt
} from 'graphql'
import {
KeyType
} from './index'
const AuditLogType = new GraphQLObjectType({
name: 'AuditLog',
fields: () => ({
id: {type: GraphQLID},
authId: {type: GraphQLID},
action: {type: GraphQLString},
details: {type: GraphQLString},
date: {type: GraphQLInt},
key: {
type: KeyType,
resolve (parent, args) {
return parent.key
}
}
})
})
export default AuditLogType
+28
View File
@@ -0,0 +1,28 @@
import {
GraphQLObjectType,
GraphQLString,
GraphQLInt,
GraphQLBoolean,
GraphQLID
} from 'graphql'
const CloneType = new GraphQLObjectType({
name: 'Clone',
fields: () => ({
id: {type: GraphQLID},
authId: {type: GraphQLID},
teamId: {type: GraphQLString},
botId: {type: GraphQLString},
mode: {type: GraphQLString},
resumeExecution: {type: GraphQLBoolean},
beginDatetime: {type: GraphQLInt},
endDatetime: {type: GraphQLInt},
waitTime: {type: GraphQLInt},
state: {type: GraphQLString},
stateDatetime: {type: GraphQLInt},
createDatetime: {type: GraphQLInt},
lastLogs: {type: GraphQLString},
runAsTeam: {type: GraphQLBoolean}
})
})
export default CloneType
+16
View File
@@ -0,0 +1,16 @@
import {
GraphQLObjectType,
GraphQLID,
GraphQLString
} from 'graphql'
const ExchangeType = new GraphQLObjectType({
name: 'Exchange',
fields: () => ({
id: {type: GraphQLID},
name: {type: GraphQLString},
description: {type: GraphQLString}
})
})
export default ExchangeType
@@ -0,0 +1,15 @@
import {
GraphQLObjectType,
GraphQLString
} from 'graphql'
const SignedTransactionType = new GraphQLObjectType({
name: 'SignedTransaction',
fields: () => ({
key: {type: GraphQLString},
signature: {type: GraphQLString},
date: {type: GraphQLString}
})
})
export default SignedTransactionType
+5
View File
@@ -0,0 +1,5 @@
import CloneType from './CloneType'
export {
CloneType
}
+24
View File
@@ -0,0 +1,24 @@
import {
GraphQLString,
GraphQLInt,
GraphQLInputObjectType,
GraphQLBoolean,
GraphQLNonNull
} from 'graphql';
const Type = new GraphQLInputObjectType({
name: 'CloneInput',
description: 'Payload for clone input',
fields: () => ({
teamId: { type: new GraphQLNonNull(GraphQLString) },
botId: { type: new GraphQLNonNull(GraphQLString) },
mode: { type: GraphQLString },
resumeExecution: { type: GraphQLBoolean },
beginDatetime: { type: GraphQLInt },
endDatetime: { type: GraphQLInt },
waitTime: { type: GraphQLInt },
runAsTeam: { type: GraphQLBoolean }
})
});
export default Type;
+5
View File
@@ -0,0 +1,5 @@
import CloneInputType from './Clone';
export {
CloneInputType
};