diff --git a/AzureStorage.js b/AzureStorage.js new file mode 100644 index 000000000..a69617887 --- /dev/null +++ b/AzureStorage.js @@ -0,0 +1,263 @@ +exports.newAzureStorage = function newAzureStorage(BOT) { + + let FULL_LOG = true; + let LOG_FILE_CONTENT = false; + + let bot = BOT; + const ROOT_DIR = './'; + + const MODULE_NAME = "AzureStorage"; + + const DEBUG_MODULE = require('./DebugLog'); + const logger = DEBUG_MODULE.newDebugLog(); + logger.fileName = MODULE_NAME; + logger.bot = bot; + logger.initialize(true); + + let thisObject = { + createBlobService: createBlobService + }; + + return thisObject; + + function createBlobService(pConnectionString) { + + return newBlobService(pConnectionString); + + } + + function newBlobService(pConnectionString) { + + let endPoint; + let SAS; + let https = require('https'); + + let thisObject = { + getBlobToText: getBlobToText, + createBlockBlobFromText: createBlockBlobFromText + }; + + let properties = pConnectionString.split(";"); + endPoint = properties[0].substring("BlobEndpoint=https://".length); + SAS = properties[1].substring("SharedAccessSignature=".length); + + return thisObject; + + function getBlobToText(pContainerName, pBlobPath, callBackFunction) { + + getBlob(endPoint, pContainerName, pBlobPath, SAS, onContentArrived); + + function onContentArrived(err, pContent) { + + let response = null; + + if (err === null) { + err = checkForStorageErrors(pContent) + } + + callBackFunction(err, pContent, response); + + } + } + + function createBlockBlobFromText(pContainerName, pBlobPath, pBlobContent, callBackFunction) { + + putBlob(endPoint, pContainerName, pBlobPath, pBlobContent, SAS, onFinished); + + function onFinished(err, pContent) { + + let result = null; + let response = null; + + if (err === null) { + err = checkForStorageErrors(pContent) + } + + callBackFunction(err, result, response); + + } + } + + function checkForStorageErrors(pData) { + + let err = null; + + if (pData !== undefined) { + + if (pData.indexOf("BlobNotFound") > 0) { + + err = { + code: 'BlobNotFound' + }; + } + + if (pData.indexOf("ParentNotFound") > 0) { + + err = { + code: 'ParentNotFound' + }; + } + + if (pData.indexOf("AuthenticationFailed") > 0) { + + err = { + code: 'AuthenticationFailed' + }; + } + + if (pData.indexOf("ServerBusy") > 0) { + + err = { + code: 'ServerBusy' + }; + } + + if (pData.indexOf("AuthorizationPermissionMismatch") > 0) { + + err = { + code: 'AuthorizationPermissionMismatch' + }; + } + } + + return err; + } + + function getBlob(pEndPoint, pContainer, pBlobPath, pSAS, callBack) { + + let options = { + host: pEndPoint, + port: 443, + path: '/' + pContainer + '/' + pBlobPath + '?' + pSAS, + method: 'GET', + headers: {} + } + + let responseData = ''; + let err = null; + let alreadyCalledBack = false; + + /* We add the headers. */ + + options.headers['x-ms-version'] = '2017-07-29'; + options.headers['Connection'] = 'Keep-Alive'; + + /* Making the https get call. */ + + let request = https.request(options, onResponse); + request.end(); //This actually sends the request. + request.on('error', onError); + + function onError(err) { + console.log("Error: ", err); + + if (alreadyCalledBack === false) { + alreadyCalledBack = true; + callBack(err); + } else { + console.log("Not calling back for the same problem."); + } + } + + function onResponse(response) { + console.log("\nstatus code: ", response.statusCode); + response.on('data', onPieceOfDataArrived); + response.on('end', onAllDataArrived); + } + + function onPieceOfDataArrived(pData) { + console.log('New Data Arrived!'); + console.log(pData.toString()); + + responseData = responseData + pData.toString(); + } + + function onAllDataArrived() { + + try { + let obj = JSON.parse(responseData); + } + catch (err) { + + console.log("Cannot parse this."); + } + + callBack(err, responseData) + + } + } + + function putBlob(pEndPoint, pContainer, pBlobPath, pBlobContent, pSAS, callBack) { + + let options = { + host: pEndPoint, + port: 443, + path: '/' + pContainer + '/' + pBlobPath + '?' + pSAS, + method: 'PUT', + headers: {} + } + + let responseData = ''; + let err = null; + let responseCode; + let alreadyCalledBack = false; + + /* We add the headers. */ + + options.headers['x-ms-version'] = '2017-07-29'; + options.headers['Content-Length'] = pBlobContent.length; + options.headers['x-ms-blob-type'] = 'BlockBlob'; + options.headers['Connection'] = 'Keep-Alive'; + + /* Making the https get call. */ + + let request = https.request(options, onResponse); + request.body = pBlobContent; + request.write(pBlobContent); + request.end(); //This actually sends the request. + request.on('error', onError); + + function onError(err) { + console.log("Error: ", err); + + if (alreadyCalledBack === false) { + alreadyCalledBack = true; + callBack(err); + } else { + console.log("Not calling back for the same problem."); + } + } + + function onResponse(response) { + + responseCode = response.statusCode; + + if (responseCode === 201) { + + callBack(err); // No problems. + + } + + response.on('data', onPieceOfDataArrived); + response.on('end', onAllDataArrived); + } + + function onPieceOfDataArrived(pData) { + console.log('New Data Arrived!'); + console.log(pData.toString()); + + responseData = responseData + pData.toString(); + } + + function onAllDataArrived() { + + if (responseCode !== 201) { + + callBack(err, responseData);; // Some problem found. + + } + } + } + } +} + diff --git a/BlobStorage.js b/BlobStorage.js index c1ae7c2a4..a87052dac 100644 --- a/BlobStorage.js +++ b/BlobStorage.js @@ -9,7 +9,8 @@ exports.newBlobStorage = function newBlobStorage(BOT) { const MODULE_NAME = "BlobStorage"; - let storage = require('azure-storage'); + let AZURE_STORAGE = require('./AzureStorage'); + let storage = AZURE_STORAGE.newAzureStorage(bot); const DEBUG_MODULE = require('./DebugLog'); let logger; @@ -130,15 +131,18 @@ exports.newBlobStorage = function newBlobStorage(BOT) { logger.write("[ERROR] createTextFile -> onFileCreated -> result = " + JSON.stringify(result)); logger.write("[ERROR] createTextFile -> onFileCreated -> response = " + JSON.stringify(response)); - if (err.code === 'ServerBusy' - || err.code === 'ECONNRESET' - || err.code === 'ENOTFOUND' - || err.code === 'ESOCKETTIMEDOUT' - || err.code === 'ETIMEDOUT' - || err.code === 'ECONNREFUSED' - || (err.code === 'AuthenticationFailed' && err.authenticationerrordetail.indexOf('Request date header too old') === 0) - || err.code === 'OperationTimedOut' - || err.code === 'EADDRINUSE') { + if ( + err.code === 'ECONNRESET' || + err.code === 'ENOTFOUND' || + err.code === 'ESOCKETTIMEDOUT' || + err.code === 'ETIMEDOUT' || + err.code === 'ECONNREFUSED' || + err.code === 'EADDRINUSE' || + err.code === 'AuthenticationFailed' || + err.code === 'OperationTimedOut' || + err.code === 'ServerBusy' + ) + { setTimeout(secondTry, 1000); return; @@ -232,15 +236,19 @@ exports.newBlobStorage = function newBlobStorage(BOT) { if (err) { - if (err.code === 'ServerBusy' - || err.code === 'ECONNRESET' - || err.code === 'ENOTFOUND' - || err.code === 'ESOCKETTIMEDOUT' - || err.code === 'ETIMEDOUT' - || err.code === 'ECONNREFUSED' - || (err.code === 'AuthenticationFailed' && err.authenticationerrordetail.indexOf('Request date header too old') === 0) - || err.code === 'OperationTimedOut' - || err.code === 'EADDRINUSE') { + if ( + err.code === 'ECONNRESET' || + err.code === 'ENOTFOUND' || + err.code === 'ESOCKETTIMEDOUT' || + err.code === 'ETIMEDOUT' || + err.code === 'ECONNREFUSED' || + err.code === 'EADDRINUSE' || + err.code === 'AuthenticationFailed' || + err.code === 'OperationTimedOut' || + err.code === 'ServerBusy' + ) + + { setTimeout(secondTry, 1000); return; diff --git a/DebugLog.js b/DebugLog.js index c0c9033ed..9a5c725fd 100644 --- a/DebugLog.js +++ b/DebugLog.js @@ -30,10 +30,16 @@ function initialize(pdisableLogging) { - disableLogging = pdisableLogging + try { - if (disableLogging !== true) { - thisObject.bot.eventHandler.listenToEvent("Close Log File", onLoopFinished); + disableLogging = pdisableLogging + + if (disableLogging !== true) { + thisObject.bot.eventHandler.listenToEvent("Close Log File", onLoopFinished); + } + + } catch (err) { + console.log("[ERROR] DebugLog -> initialize -> err = " + err.message); } } diff --git a/ExtractionBotProcessMainLoop.js b/ExtractionBotProcessMainLoop.js index 9cb3f7b3c..f8309b91b 100644 --- a/ExtractionBotProcessMainLoop.js +++ b/ExtractionBotProcessMainLoop.js @@ -147,7 +147,8 @@ if (bot.runAtFixedInterval === true) { intervalHandle = setInterval(loop, bot.fixedInterval); - + loop(); // First run. + } else { loop(); diff --git a/Project.njsproj b/Project.njsproj index ea7ab1eb3..96c0cc835 100644 --- a/Project.njsproj +++ b/Project.njsproj @@ -23,6 +23,9 @@ Code + + Code + Code