Partial commit for saweb-api migration

This commit is contained in:
Matias authored and Matias committed 2019-05-24 20:09:11 -03:00
1 parent 19044bda86
commit e4fc210f11
12 files changed
+193 -771

No files matched your search

+67 -172
View File
@@ -10,26 +10,14 @@
}
let storageData;
let serverConfig;
let Minio = require('minio');
let minioClient;
let ecosystem;
return thisObject;
function initialize(pStorageData, pServerConfig) {
function initialize(pStorageData, pEcosystem) {
storageData = pStorageData;
serverConfig = pServerConfig;
minioClient = new Minio.Client({
endPoint: process.env.MINIO_END_POINT,
port: JSON.parse(process.env.MINIO_PORT),
useSSL: JSON.parse(process.env.MINIO_USE_SSL),
accessKey: process.env.MINIO_ACCESS_KEY,
secretKey: process.env.MINIO_SECRET_KEY
})
ecosystem = pEcosystem;
}
function readData(pOrg, pPath, pFile, saveAtCache, callBackFunction) {
@@ -58,56 +46,8 @@
if (CONSOLE_LOG === true) { console.log("[INFO] Azure Storage -> readData -> " + pOrg + '.' + pPath + '.' + pFile + " NOT found at cache."); }
let storage = require('azure-storage');
let connectionString;
switch (serverConfig.environment) {
case "Develop": {
connectionString = serverConfig.configAndPlugins.Develop.connectionString;
break;
}
case "Production": {
connectionString = serverConfig.configAndPlugins.Production.connectionString;
break;
}
}
let blobService = storage.createBlobService(connectionString);
let pParam1 = pOrg;
let pParam2 = pPath;
let pParam3 = pFile;
let container = 'aaplatform';
let azurePath = pOrg + "/" + pPath + "/" + pFile;
let bucketName = 'aaplatform';
let textFilename;
if (pParam1 === undefined) {
/* This is used when the requests come from the browser, for background compatibility. */
container = pParam3;
azurePath = pParam2;
bucketName = pParam3;
textFilename = pParam2.substring(pParam2.indexOf("/") + 1, pParam2.length);
} else {
/* This is used for internal requests within AAWeb, for background compatibility. */
bucketName = 'aaplatform';
textFilename = pParam1 + "/" + pParam2 + "/" + pParam3;
}
blobService.getBlobToText(container, azurePath, onFileReceived);
let connectionDetails = getConnectionDetails(pOrg, pPath, pFile);
connectionDetails.blobService.getBlobToText(connectionDetails.container, connectionDetails.filePath, onFileReceived);
function onFileReceived(err, text, response) {
@@ -143,23 +83,6 @@
}
/* TEMPORARY CODE TO UPDATE THE MINIO SERVER DURING TRANSITION. */
// try {
// minioClient.putObject(bucketName, textFilename, text, 'text/plain', function (err) {
// if (err) {
// console.log("ERROR AT MINIO SERVER PUTTING OBJECT : " + err);
// return;
// }
// console.log(textFilename + ' Successfully uploaded ' + textFilename + ' the MINIO SERVER');
// })
// } catch (err) {
// console.log("ERROR UPDATING MINIO SERVER : " + err);
// }
callBackFunction(global.DEFAULT_OK_RESPONSE, text);
} catch (err) {
@@ -175,6 +98,7 @@
}
}
// TODO remove this functions and resolve them inside the Teams module.
function writeData(pOrg, pPath, pFile, pFileContent, callBackFunction) {
try {
@@ -184,29 +108,9 @@
if (CONSOLE_LOG === true) { console.log("[INFO] Azure Storage -> writeData -> pPath = " + pPath); }
if (CONSOLE_LOG === true) { console.log("[INFO] Azure Storage -> writeData -> pFile = " + pFile); }
let storage = require('azure-storage');
let connectionString;
switch (serverConfig.environment) {
case "Develop": {
connectionString = serverConfig.configAndPlugins.Develop.connectionString;
break;
}
case "Production": {
connectionString = serverConfig.configAndPlugins.Production.connectionString;
break;
}
}
let blobService = storage.createBlobService(connectionString);
let blobPath = pOrg + "/" + pPath + "/" + pFile;
let blobText = pFileContent.toString();
blobService.createBlockBlobFromText('aaplatform', blobPath, blobText, onFileCreated);
let connectionDetails = getConnectionDetails(pOrg, pPath, pFile);
connectionDetails.blobService.createBlockBlobFromText(connectionDetails.container, connectionDetails.filePath, blobText, onFileCreated);
function onFileCreated(err, text, response) {
@@ -252,28 +156,8 @@
if (CONSOLE_LOG === true) { console.log("[INFO] Azure Storage -> deleteBlob -> pPath = " + pPath); }
if (CONSOLE_LOG === true) { console.log("[INFO] Azure Storage -> deleteBlob -> pFile = " + pFile); }
let storage = require('azure-storage');
let connectionString;
switch (serverConfig.environment) {
case "Develop": {
connectionString = serverConfig.configAndPlugins.Develop.connectionString;
break;
}
case "Production": {
connectionString = serverConfig.configAndPlugins.Production.connectionString;
break;
}
}
let blobService = storage.createBlobService(connectionString);
let blobPath = pOrg + "/" + pPath + "/" + pFile;
blobService.deleteBlob('aaplatform', blobPath, onBlobDeleted);
let connectionDetails = getConnectionDetails(pOrg, pPath, pFile);
connectionDetails.blobService.deleteBlob(connectionDetails.container, connectionDetails.filePath, onBlobDeleted);
function onBlobDeleted(err, text, response) {
@@ -317,28 +201,11 @@
if (CONSOLE_LOG === true) { console.log("[INFO] Azure Storage -> createContainer -> Entering function."); }
if (CONSOLE_LOG === true) { console.log("[INFO] Azure Storage -> createContainer -> pContainerName = " + pContainerName); }
let storage = require('azure-storage');
let connectionString;
switch (serverConfig.environment) {
case "Develop": {
connectionString = serverConfig.configAndPlugins.Develop.connectionString;
break;
}
case "Production": {
connectionString = serverConfig.configAndPlugins.Production.connectionString;
break;
}
}
let blobService = storage.createBlobService(connectionString);
let containerName = pContainerName.toLowerCase();
let storage = require('azure-storage');
let connectionString = process.env.STORAGE_CONNECTION_STRING;
let blobService = storage.createBlobService(connectionString);
blobService.createContainer(containerName, onContainerCreated);
function onContainerCreated(err) {
@@ -366,9 +233,6 @@
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
callBackFunction(global.DEFAULT_OK_RESPONSE);
@@ -388,28 +252,11 @@
if (CONSOLE_LOG === true) { console.log("[INFO] Azure Storage -> deleteContainer -> Entering function."); }
if (CONSOLE_LOG === true) { console.log("[INFO] Azure Storage -> deleteContainer -> pContainerName = " + pContainerName); }
let storage = require('azure-storage');
let connectionString;
switch (serverConfig.environment) {
case "Develop": {
connectionString = serverConfig.configAndPlugins.Develop.connectionString;
break;
}
case "Production": {
connectionString = serverConfig.configAndPlugins.Production.connectionString;
break;
}
}
let blobService = storage.createBlobService(connectionString);
let containerName = pContainerName.toLowerCase();
let storage = require('azure-storage');
let connectionString = process.env.STORAGE_CONNECTION_STRING;
let blobService = storage.createBlobService(connectionString);
blobService.deleteContainer(containerName, onContainerDeleted);
function onContainerDeleted(err) {
@@ -437,9 +284,6 @@
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
callBackFunction(global.DEFAULT_OK_RESPONSE);
@@ -451,4 +295,55 @@
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function getConnectionDetails(pOrg, pPath, pFile){
let filePath, container, host, accessKey, teamCodeName;
if (pOrg === undefined) {
/* This is used when the requests come from the browser, for background compatibility. */
filePath = pPath;
teamCodeName = pFile;
} else {
/* This is used for internal requests within AAWeb, for background compatibility. */
filePath = pOrg + "/" + pPath + "/" + pFile;
teamCodeName = pOrg.split("/")[0];
}
// TODO Temporary looking on on the server side for the team connection, so the client calls are not modified.
for (let i = 0; i < ecosystem.devTeams.length; i++) {
const devTeam = ecosystem.devTeams[i];
if(devTeam.codeName.toLowerCase() === teamCodeName.toLowerCase()){
host = devTeam.host.storage;
accessKey = devTeam.host.accessKey;
container = devTeam.host.container;
break;
}
}
for (let i = 0; i < ecosystem.hosts.length; i++) {
const devTeam = ecosystem.hosts[i];
if(devTeam.codeName.toLowerCase() === teamCodeName.toLowerCase()){
host = devTeam.host.storage;
accessKey = devTeam.host.accessKey;
container = devTeam.host.container;
break;
}
}
if(host === undefined){
console.log("[ERROR] Azure Storage -> readData -> Host has not been found for team: " + teamCodeName );
}
if(accessKey === undefined){
console.log("[ERROR] Azure Storage -> readData -> Access key has not been found for team: " + teamCodeName);
}
let storage = require('azure-storage');
let blobService = storage.createBlobServiceWithSas(host, accessKey);
return {
blobService,
container,
filePath
}
}
}
-79
View File
@@ -1,79 +0,0 @@
exports.newAzureStorageAccessManager = function newAzureStorageAccessManager() {
let thisObject = {
getPermission: getPermission,
initialize: initialize
}
let connectionString;
let blobEndpoint;
return thisObject;
function initialize(serverConfig, callBackFunction) {
switch (serverConfig.environment) {
case "Develop": {
blobEndpoint = serverConfig.productsStorage.Develop.fileUri;
connectionString = serverConfig.configAndPlugins.Develop.connectionString;
break;
}
case "Production": {
blobEndpoint = serverConfig.productsStorage.Production.fileUri;
connectionString = serverConfig.configAndPlugins.Production.connectionString;
break;
}
}
callBackFunction();
}
function getPermission(pContainer, pType, pDays) {
let azure = require('azure-storage');
let blobService = azure.createBlobService(connectionString);
let startDate = new Date();
startDate.setDate(startDate.getDate() - 1);
let expiryDate = new Date(startDate);
expiryDate.setDate(startDate.getDate() + pDays);
let azurePermissions;
switch (pType) {
case "READ": {
azurePermissions = azure.BlobUtilities.SharedAccessPermissions.READ;
break;
}
case "WRITE": {
azurePermissions = azure.BlobUtilities.SharedAccessPermissions.WRITE;
break;
}
case "DELETE": {
azurePermissions = azure.BlobUtilities.SharedAccessPermissions.DELETE;
break;
}
}
let blobName = undefined;
let sharedAccessPolicy = {
AccessPolicy: {
Permissions: azurePermissions,
Start: startDate,
Expiry: expiryDate
}
};
let sasToken = blobService.generateSharedAccessSignature(pContainer, blobName, sharedAccessPolicy);
return "BlobEndpoint=" + blobEndpoint + ";SharedAccessSignature=" + sasToken + "";
}
}
+3 -6
View File
@@ -7,18 +7,15 @@
let storage;
let serverConfig;
let storageData;
return thisObject;
function initialize(pServerConfig, pStorage, callBackFunction) {
function initialize(ecosystem, pStorage, callBackFunction) {
storage = pStorage;
let storageData = new Map(); // For backwards compatibility only. Not used.
storage.initialize(storageData, pServerConfig);
storage.initialize(storageData, ecosystem);
callBackFunction();
}
@@ -76,4 +73,4 @@
return text.replace(new RegExp(search, 'g'), replacement);
}
}
}
}
+17 -159
View File
@@ -5,128 +5,30 @@
initialize: initialize
}
const GITHUB = require('../Server/Github');
let github = GITHUB.newGithub();
let storage;
let serverConfig;
let storageData;
let ecosystem;
let ecosystemObject;
return thisObject;
function initialize(pEcosystem, pEcosystemObject, pStorageData, pStorage, callBackFunction) {
function initialize(pEcosystemObject, pStorageData, pStorage, callBackFunction) {
storage = pStorage;
storageData = pStorageData;
ecosystem = pEcosystem;
ecosystemObject = pEcosystemObject;
readAAWebConfig();
function readAAWebConfig() {
try {
if (CONSOLE_LOG === true) { console.log("[INFO] ConfigReader -> readAAWebConfig -> Entering function."); }
let fs = require('fs');
let fileName = './this.server.config.json';
fs.readFile(fileName, onFileRead);
function onFileRead(err, file) {
try {
if (CONSOLE_LOG === true) { console.log("[INFO] ConfigReader -> readAAWebConfig -> onFileRead -> Entering function."); }
let fileText;
fileText = file.toString();
fileText = fileText.trim(); // remove first byte with some encoding.
serverConfig = JSON.parse(fileText);
if (LOG_FILE_CONTENT === true) { console.log("[INFO] ConfigReader -> readAAWebConfig -> onFileRead -> fileText = " + fileText); }
CONSOLE_LOG = serverConfig.webServerLog.console;
LOG_FILE_CONTENT = serverConfig.webServerLog.fileContent;
/* Finalize initializations. */
storage.initialize(storageData, serverConfig);
callBackFunction(global.DEFAULT_OK_RESPONSE, serverConfig);
}
catch (err) {
console.log("[ERROR] ConfigReader -> readAAWebConfig -> onFileRead -> File = " + fileName + " Error = " + err);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
console.log("[ERROR] ConfigReader -> readAAWebConfig -> Error = " + err);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
try {
if (CONSOLE_LOG === true) { console.log("[INFO] ConfigReader -> initialize -> Initializing storage."); }
storage.initialize(storageData, ecosystemObject);
callBackFunction(global.DEFAULT_OK_RESPONSE);
} catch (err) {
console.log("[ERROR] ConfigReader -> initialize -> Error initilizing the storage. Stack:", err.stack);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function loadConfigs(callBackFunction) {
readEcosystemConfig();
function readEcosystemConfig() {
try {
if (CONSOLE_LOG === true) { console.log("[INFO] ConfigReader -> readEcosystemConfig -> Entering function."); }
/*
This configuration file is the backbone of the system. The first file we are going to get is a template where other configurations are
injected and the files ends up inflated with all these configs in one single JSON object that in turn is later injected into a
javascript module with an object that is going to instantiate it at run-time.
*/
storage.readData('AdvancedAlgos', 'AAPlatform', 'ecosystem.json', true, onDataArrived);
function onDataArrived(err, pData) {
try {
if (CONSOLE_LOG === true) { console.log("[INFO] ConfigReader -> readEcosystemConfig -> Cloud -> onDataArrived -> Entering function."); }
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
console.log("[ERROR] ConfigReader -> readEcosystemConfig -> Cloud -> onDataArrived -> Could not read a file. ");
console.log("[ERROR] ConfigReader -> readEcosystemConfig -> Cloud -> onDataArrived -> err.message = " + err.message);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
ecosystem = pData.toString();
ecosystem = ecosystem.trim(); // remove first byte with some encoding.
ecosystemObject = JSON.parse(ecosystem);
readHostsConfigs();
}
catch (err) {
console.log("[ERROR] ConfigReader -> loadConfigs -> readEcosystemConfig -> onDataArrived -> Error = " + err);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
console.log("[ERROR] ConfigReader -> loadConfigs -> readEcosystemConfig -> Error = " + err);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
readHostsConfigs();
function readHostsConfigs() {
@@ -294,13 +196,13 @@
if (CONSOLE_LOG === true) { console.log("[INFO] ConfigReader -> readDevTeamsConfigs -> Entering function."); }
/*
Each bot has its configuration at its own repo since each team must be able to change it at will.
So what we do here is to use the master config at the AAPlatform repo that we already have on
memory and inject into it the config of each bot.
Inmediatelly after that, we also load the Plotters configs using the same technique.
*/
let requestsSent = 0;
@@ -361,8 +263,6 @@
configObj.repo = bot.repo;
configObj.configFile = bot.configFile;
addStoragePermissions(configObj);
/* Here we will add all the possible Clones of this bot */
if (configObj.genes !== undefined) {
@@ -387,9 +287,9 @@
}
/*
/*
Now we have all the possible genes values in a multi-dimensional matrix.
We will go thorugh each of the elements of each dimension and get each combination possible with the other dimmensions.
We will go thorugh each of the elements of each dimension and get each combination possible with the other dimmensions.
*/
let combinations = [];
@@ -478,7 +378,7 @@
if (requestsSent === responsesReceived) {
callBackFunction(global.DEFAULT_OK_RESPONSE, ecosystem, ecosystemObject);
callBackFunction(global.DEFAULT_OK_RESPONSE, ecosystemObject);
}
@@ -550,7 +450,7 @@
if (requestsSent === responsesReceived) {
callBackFunction(global.DEFAULT_OK_RESPONSE, ecosystem, ecosystemObject);
callBackFunction(global.DEFAULT_OK_RESPONSE, ecosystemObject);
}
}
@@ -569,48 +469,6 @@
}
}
function addStoragePermissions(pConfigObj) {
try {
if (CONSOLE_LOG === true) { console.log("[INFO] ConfigReader -> readDevTeamsConfigs -> addStoragePermissions -> Entering function."); }
if (pConfigObj.storage !== undefined) { return; } // If this information is at the config file, then we take it, otherwise, we define it here.
let fileUri;
let sas;
switch (serverConfig.environment) {
case "Develop": {
fileUri = serverConfig.productsStorage.Develop.fileUri;
sas = serverConfig.productsStorage.Develop.sas;
break;
}
case "Production": {
fileUri = serverConfig.productsStorage.Production.fileUri;
sas = serverConfig.productsStorage.Production.sas;
break;
}
}
pConfigObj.storage = {
sas: sas,
fileUri: fileUri
};
}
catch (err) {
console.log("[ERROR] ConfigReader -> readDevTeamsConfigs -> addStoragePermissions -> Error = " + err);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
console.log("[ERROR] ConfigReader -> readDevTeamsConfigs -> Error = " + err);
@@ -618,4 +476,4 @@
}
}
}
}
}
+1 -3
View File
@@ -10,17 +10,15 @@
}
let storageData;
let serverConfig;
let Minio = require('minio');
let minioClient;
return thisObject;
function initialize(pStorageData, pServerConfig) {
function initialize(pStorageData) {
storageData = pStorageData;
serverConfig = pServerConfig;
minioClient = new Minio.Client({
endPoint: process.env.MINIO_END_POINT,
+44 -38
View File
@@ -9,49 +9,55 @@
return thisObject;
function initialize(pServerConfig, storage, callBackFunction) {
function initialize(ecosystem, storage, callBackFunction) {
readSessions();
try {
// TODO Sessions need to be handled by the master app
function readSessions() {
const sessions = require('./config/sessions.json');
try {
storage.initialize(undefined, pServerConfig);
storage.readData("AdvancedAlgos", "AAPlatform", "open.sessions.json", false, onData);
function onData(err, pText) {
if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
console.log("[ERROR] SessionManager -> initialize -> readSessions -> onData -> Could not read a file. ");
console.log("[ERROR] SessionManager -> initialize -> readSessions -> onData -> err.message = " + err.message);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
let sessions = JSON.parse(pText);
for (let i = 0; i < sessions.length; i++) {
let session = sessions[i];
activeSessions.set(session.sessionToken, session);
}
callBackFunction(global.DEFAULT_OK_RESPONSE);
}
for (let i = 0; i < sessions.length; i++) {
let session = sessions[i];
activeSessions.set(session.sessionToken, session);
}
catch (err) {
console.log("[ERROR] readSessions -> err = " + err.message);
console.log("[HINT] You need to have a file at this path -> " + filePath);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
callBackFunction(global.DEFAULT_OK_RESPONSE);
// storage.initialize(undefined, ecosystem);
// storage.readData("AdvancedAlgos", "AAPlatform", "open.sessions.json", false, onData);
// function onData(err, pText) {
// if (err.result !== global.DEFAULT_OK_RESPONSE.result) {
// console.log("[ERROR] SessionManager -> initialize -> readSessions -> onData -> Could not read a file. ");
// console.log("[ERROR] SessionManager -> initialize -> readSessions -> onData -> err.message = " + err.message);
// callBackFunction(global.DEFAULT_FAIL_RESPONSE);
// return;
// }
// let sessions = JSON.parse(pText);
// for (let i = 0; i < sessions.length; i++) {
// let session = sessions[i];
// activeSessions.set(session.sessionToken, session);
// }
// callBackFunction(global.DEFAULT_OK_RESPONSE);
// }
}
catch (err) {
console.log("[ERROR] readSessions -> err = " + err.message);
console.log("[HINT] You need to have a file at this path -> " + filePath);
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function getSession(pSessionToken) {
@@ -59,4 +65,4 @@
return activeSessions.get(pSessionToken);
}
}
}