From a92ccff26558c746ec3d801c291f7bbcf4d5cd1a Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 6 Apr 2019 21:24:06 +0200 Subject: [PATCH] Working with Minio & Azure (missing Team management) --- Scripts/AppPreLoader.js | 51 ++++++++++------- Server/AzureStorage.js | 34 +++++++++-- Server/MinioStorage.js | 122 ++++++++++++++++++++++++---------------- server.js | 7 +-- 4 files changed, 135 insertions(+), 79 deletions(-) diff --git a/Scripts/AppPreLoader.js b/Scripts/AppPreLoader.js index e35291b0e..c8a3bdf8a 100644 --- a/Scripts/AppPreLoader.js +++ b/Scripts/AppPreLoader.js @@ -138,37 +138,48 @@ function callServer(pContentToSend, pPath, callBackFunction) { } -function getBlobToText(pContainerName, pPath, callBackFunction) { +function newFileStorage() { - let xhttp = new XMLHttpRequest(); - xhttp.onreadystatechange = function () { - if (this.readyState === 4 && this.status === 200) { - - let response = JSON.parse(xhttp.responseText); - callBackFunction(response.err, response.text, ""); - - } + thisObject = { + getBlobToText: getBlobToText }; - try { + return thisObject; - let request = { - conatinerName: pContainerName, - path: pPath + function getBlobToText(pContainerName, pPath, callBackFunction) { + + let xhttp = new XMLHttpRequest(); + xhttp.onreadystatechange = function () { + if (this.readyState === 4 && this.status === 200) { + + let response = JSON.parse(xhttp.responseText); + callBackFunction(response.err, response.text, ""); + + } }; - request = JSON.stringify(request); + try { - let path = "FileService"; + let request = { + conatinerName: pContainerName, + path: pPath + }; - let blob = new Blob([request], { type: 'text/plain' }); + request = JSON.stringify(request); - xhttp.open("POST", path, true); - xhttp.send(blob); + let path = "FileService"; - } catch (err) { + let blob = new Blob([request], { type: 'text/plain' }); - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] getBlobToText -> err.message = " & err.message); } + xhttp.open("POST", path, true); + xhttp.send(blob); + } catch (err) { + + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] getBlobToText -> err.message = " & err.message); } + + } } } + + diff --git a/Server/AzureStorage.js b/Server/AzureStorage.js index 707187d56..8a8a29b24 100644 --- a/Server/AzureStorage.js +++ b/Server/AzureStorage.js @@ -78,7 +78,35 @@ let blobService = storage.createBlobService(connectionString); - blobService.getBlobToText('aaplatform', pOrg + "/" + pPath + "/" + pFile, onFileReceived); + let pParam1 = pOrg; + let pParam2 = pPath; + let pParam3 = pFile; + let container = 'aaplatform'; + let azurePath = pOrg + "/" + pPath + "/" + pFile; + + let bucketName = 'aaplatform'; + let textFilename; + + if (pParam1 === undefined) { + + /* This is used when the requests come from the browser, for background compatibility. */ + + container = pParam3; + azurePath = pParam2; + + bucketName = pParam3; + textFilename = pParam2.substring(pParam2.indexOf("/") + 1, pParam2.length); + + } else { + + /* This is used for internal requests within AAWeb, for background compatibility. */ + + bucketName = 'aaplatform'; + textFilename = pParam1 + "/" + pParam2 + "/" + pParam3; + } + + + blobService.getBlobToText(container, azurePath, onFileReceived); function onFileReceived(err, text, response) { @@ -113,10 +141,6 @@ try { - let bucketName = 'aaplatform'; - - let textFilename = pOrg + "/" + pPath + "/" + pFile; - minioClient.putObject(bucketName, textFilename, text, 'text/plain', function (err) { if (err) { console.log("ERROR AT MINIO SERVER PUTTING OBJECT : " + err); diff --git a/Server/MinioStorage.js b/Server/MinioStorage.js index 5a129d343..050647ba7 100644 --- a/Server/MinioStorage.js +++ b/Server/MinioStorage.js @@ -32,31 +32,31 @@ } - function readData(pOrg, pPath, pFile, saveAtCache, callBackFunction) { + function readData(pParam1, pParam2, pParam3, saveAtCache, callBackFunction) { try { if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> Entering function."); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> pOrg = " + pOrg); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> pPath = " + pPath); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> pFile = " + pFile); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> pParam1 = " + pParam1); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> pParam2 = " + pParam2); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> pParam3 = " + pParam3); } let cacheVersion; if (storageData !== undefined) { - cacheVersion = storageData.get(pOrg + '.' + pPath + '.' + pFile); + cacheVersion = storageData.get(pParam1 + '.' + pParam2 + '.' + pParam3); } if (cacheVersion !== undefined) { - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> " + pOrg + '.' + pPath + '.' + pFile + " found at cache."); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> " + pParam1 + '.' + pParam2 + '.' + pParam3 + " found at cache."); } callBackFunction(global.DEFAULT_OK_RESPONSE, cacheVersion); } else { - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> " + pOrg + '.' + pPath + '.' + pFile + " NOT found at cache."); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> " + pParam1 + '.' + pParam2 + '.' + pParam3 + " NOT found at cache."); } let storage = require('azure-storage'); let connectionString; @@ -76,10 +76,41 @@ } } - let blobService = storage.createBlobService(connectionString); + let bucketName = 'aaplatform'; + let textFilename; - blobService.getBlobToText('aaplatform', pOrg + "/" + pPath + "/" + pFile, onFileReceived); + if (pParam1 === undefined) { + /* This is used when the requests come from the browser, for background compatibility. */ + + bucketName = pParam3; + textFilename = pParam2.substring(pParam2.indexOf("/") + 1, pParam2.length); + + } else { + + /* This is used for internal requests within AAWeb, for background compatibility. */ + + bucketName = 'aaplatform'; + textFilename = pParam1 + "/" + pParam2 + "/" + pParam3; + } + + minioClient.getObject(bucketName, textFilename, function (err, dataStream) { + let data = ''; + + if (err) { + onFileReceived(err, null, "Error retrieving file " + textFilename + " from bucket " + bucketName); + return + } + dataStream.on('data', function (chunk) { + data += chunk + }) + dataStream.on('end', function () { + onFileReceived(null, data, "Data retrieved."); + }) + dataStream.on('error', function (err) { + onFileReceived(err, null, "Error on data stream while retrieving file " + textFilename + " from bucket " + bucketName); + }) + }) function onFileReceived(err, text, response) { @@ -88,13 +119,13 @@ if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> onFileReceived -> Entering function."); } if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> onFileReceived -> err = " + JSON.stringify(err)); } if (LOG_FILE_CONTENT === true) { console.log("[INFO] Storage -> readData -> onFileReceived -> response = " + JSON.stringify(response)); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> onFileReceived -> pOrg = " + pOrg); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> onFileReceived -> pPath = " + pPath); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> onFileReceived -> pFile = " + pFile); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> onFileReceived -> pParam1 = " + pParam1); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> onFileReceived -> pParam2 = " + pParam2); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> readData -> onFileReceived -> pParam3 = " + pParam3); } if (saveAtCache === true) { - storageData.set(pOrg + '.' + pPath + '.' + pFile, text); + storageData.set(pParam1 + '.' + pParam2 + '.' + pParam3, text); } @@ -104,32 +135,23 @@ if (CONSOLE_ERROR_LOG === true) { console.log("[ERROR] Storage -> readData -> onFileReceived -> err = " + JSON.stringify(err)); } if (CONSOLE_ERROR_LOG === true) { console.log("[ERROR] Storage -> readData -> onFileReceived -> Returning an empty JSON object string. "); } + if (err.code === 'NoSuchKey') { + + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'FileNotFound', + code: 'FileNotFound' + } + + callBackFunction(customErr); + return; + } + callBackFunction(global.DEFAULT_FAIL_RESPONSE); return; } - /* TEMPORARY CODE TO UPDATE THE MINIO SERVER DURING TRANSITION. */ - - try { - - let bucketName = 'aaplatform'; - - let textFilename = pOrg + "/" + pPath + "/" + pFile; - - minioClient.putObject(bucketName, textFilename, text, 'text/plain', function (err) { - if (err) { - console.log("ERROR AT MINIO SERVER PUTTING OBJECT : " + err); - return; - } - console.log(textFilename + ' Successfully uploaded ' + textFilename + ' the MINIO SERVER'); - }) - - } catch (err) { - console.log("ERROR UPDATING MINIO SERVER : " + err); - } - - callBackFunction(global.DEFAULT_OK_RESPONSE, text); } catch (err) { @@ -145,14 +167,14 @@ } } - function writeData(pOrg, pPath, pFile, pFileContent, callBackFunction) { + function writeData(pParam1, pParam2, pParam3, pFileContent, callBackFunction) { try { if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> Entering function."); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> pOrg = " + pOrg); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> pPath = " + pPath); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> pFile = " + pFile); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> pParam1 = " + pParam1); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> pParam2 = " + pParam2); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> pParam3 = " + pParam3); } let storage = require('azure-storage'); let connectionString; @@ -173,7 +195,7 @@ } let blobService = storage.createBlobService(connectionString); - let blobPath = pOrg + "/" + pPath + "/" + pFile; + let blobPath = pParam1 + "/" + pParam2 + "/" + pParam3; let blobText = pFileContent.toString(); blobService.createBlockBlobFromText('aaplatform', blobPath, blobText, onFileCreated); @@ -205,9 +227,9 @@ if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> onFileCreated -> Entering function."); } if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> onFileCreated -> err = " + JSON.stringify(err)); } if (LOG_FILE_CONTENT === true) { console.log("[INFO] Storage -> writeData -> onFileCreated -> response = " + JSON.stringify(response)); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> onFileCreated -> pOrg = " + pOrg); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> onFileCreated -> pPath = " + pPath); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> onFileCreated -> pFile = " + pFile); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> onFileCreated -> pParam1 = " + pParam1); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> onFileCreated -> pParam2 = " + pParam2); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> writeData -> onFileCreated -> pParam3 = " + pParam3); } if (err !== null || text === null) { @@ -233,14 +255,14 @@ } } - function deleteBlob(pOrg, pPath, pFile, callBackFunction) { + function deleteBlob(pParam1, pParam2, pParam3, callBackFunction) { try { if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> Entering function."); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> pOrg = " + pOrg); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> pPath = " + pPath); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> pFile = " + pFile); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> pParam1 = " + pParam1); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> pParam2 = " + pParam2); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> pParam3 = " + pParam3); } let storage = require('azure-storage'); let connectionString; @@ -261,7 +283,7 @@ } let blobService = storage.createBlobService(connectionString); - let blobPath = pOrg + "/" + pPath + "/" + pFile; + let blobPath = pParam1 + "/" + pParam2 + "/" + pParam3; blobService.deleteBlob('aaplatform', blobPath, onBlobDeleted); @@ -291,9 +313,9 @@ if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> onBlobDeleted -> Entering function."); } if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> onBlobDeleted -> err = " + JSON.stringify(err)); } if (LOG_FILE_CONTENT === true) { console.log("[INFO] Storage -> deleteBlob -> onBlobDeleted -> response = " + JSON.stringify(response)); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> onBlobDeleted -> pOrg = " + pOrg); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> onBlobDeleted -> pPath = " + pPath); } - if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> onBlobDeleted -> pFile = " + pFile); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> onBlobDeleted -> pParam1 = " + pParam1); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> onBlobDeleted -> pParam2 = " + pParam2); } + if (CONSOLE_LOG === true) { console.log("[INFO] Storage -> deleteBlob -> onBlobDeleted -> pParam3 = " + pParam3); } if (err !== null || text === null) { diff --git a/server.js b/server.js index 581a5ca6a..c17c6097c 100644 --- a/server.js +++ b/server.js @@ -221,20 +221,19 @@ function onBrowserRequest (request, response) { let request = JSON.parse(pData); - fileStorage.getFile(container, path, onReady); + storage.readData(undefined, request.path, request.conatinerName, false, onReady) function onReady(err, pFileContent) { - let response = { + let responseToClient = { err: err, text: pFileContent }; - respondWithContent(JSON.stringify(response)); + respondWithContent(JSON.stringify(responseToClient), response); } } - respondWithFile(filePath, response) break; } case 'Plotter.js':