fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/develop'

Browse Source
This commit is contained in:
Matias committed 2019-04-09 21:11:08 +02:00
2 parents eea12dbfa0 3fea9e84ee
commit a92acf5d84
10 files changed
+1649 -1023
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Assistant.js
+1 -1
View File
@@ -1366,7 +1366,7 @@
function addExtraData(pExtraDataArray) {
context.extraData = pExtraDataArray;
context.extraData.push(pExtraDataArray);
}
function truncDecimals(pFloatValue, pDecimals) {
AzureBlobStorage.js
+420
View File
@@ -0,0 +1,420 @@
exports.newAzureBlobBlobStorage = function newAzureBlobBlobStorage(BOT, logger) {
let FULL_LOG = false;
let ERROR_LOG = true;
let LOG_FILE_CONTENT = false;
let bot = BOT;
const ROOT_DIR = './';
const MODULE_NAME = "Azure Blob Storage";
const SECOND_TRY_WAIT_TIME = 10000;
let AZURE_STORAGE = require('./AzureStorage')
let storage = AZURE_STORAGE.newAzureStorage(bot, logger);
let thisObject = {
initialize: initialize,
createFolder: createFolder,
createTextFile: createTextFile,
getTextFile: getTextFile
};
let readOnlyBlobService;
let writeOnlyBlobService;
let containerName;
let environment = global.CURRENT_ENVIRONMENT;
let Minio = require('minio'); // Temporary during migration
let minioClient; // Temporary during migration
return thisObject;
function initialize(pDataOwner, callBackFunction, disableLogging) {
try {
minioClient = new Minio.Client({
endPoint: process.env.MINIO_END_POINT,
port: JSON.parse(process.env.MINIO_PORT),
useSSL: JSON.parse(process.env.MINIO_USE_SSL),
accessKey: process.env.MINIO_ACCESS_KEY,
secretKey: process.env.MINIO_SECRET_KEY
})
containerName = pDataOwner.toLowerCase();
if (pDataOwner.environment !== undefined) { // This is use for data migration from one environment to the other.
environment = pDataOwner.environment;
}
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> environment = " + environment); }
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> containerName = " + containerName); }
let readConnectionString = global.USER_PROFILE.storagePermissions.get(pDataOwner + "." + "READ");
if (readConnectionString !== undefined) {
readOnlyBlobService = storage.createBlobService(readConnectionString);
}
let writeConnectionString = global.USER_PROFILE.storagePermissions.get(pDataOwner + "." + "WRITE");
if (writeConnectionString !== undefined) {
writeOnlyBlobService = storage.createBlobService(writeConnectionString);
}
if (readOnlyBlobService !== undefined || writeOnlyBlobService !== undefined) {
callBackFunction(global.DEFAULT_OK_RESPONSE);
return;
} else {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> No READ or WRITE permissions found."); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> pDataOwner = " + pDataOwner); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function createFolder(pFolderPath, callBackFunction) {
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] createFolder -> Entering function."); }
try {
/* Folders do not need to be created when using blobs. */
callBackFunction(global.DEFAULT_OK_RESPONSE);
return;
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createFolder -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function createTextFile(pFolderPath, pFileName, pFileContent, callBackFunction) {
if (writeOnlyBlobService === undefined) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> initialize function not executed or failed. Can not process this request. Sorry."); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] createTextFile -> About to create a text file.");
logger.write(MODULE_NAME, "[INFO] createTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[INFO] createTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[INFO] createTextFile -> pFileName = " + pFileName);
}
writeOnlyBlobService.createBlockBlobFromText(containerName, pFolderPath + "/" + pFileName, pFileContent, onFileCreated);
/* REPLICATION TO MINIO */
minioClient.putObject(containerName, pFolderPath + "/" + pFileName, pFileContent, 'text/plain', function (err) {
console.log(MODULE_NAME, "[INFO] 'createTextFile' -> Replicating to MINIO -> File = " + pFolderPath + "/" + pFileName);
if (err) {
console.log(MODULE_NAME, "[WARN] 'createTextFile' -> Replicating to MINIO -> err = " + err.message);
}
}); // Temporary during migration
function onFileCreated(err, result, response) {
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> Response from Azure received.");
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> err = " + JSON.stringify(err));
}
if (LOG_FILE_CONTENT === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> result = " + JSON.stringify(result));
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileReceived -> response = " + JSON.stringify(response));
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileReceived -> pFileContent = " + pFileContent);
}
if (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> err = " + JSON.stringify(err)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> result = " + JSON.stringify(result)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> response = " + JSON.stringify(response)); }
if (ERROR_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> Error trying to create this file.");
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> err.code = " + err.code);
}
if (
err.code === 'ECONNRESET' ||
err.code === 'ENOTFOUND' ||
err.code === 'ESOCKETTIMEDOUT' ||
err.code === 'ETIMEDOUT' ||
err.code === 'ECONNREFUSED' ||
err.code === 'EADDRINUSE' ||
err.code === 'EAI_AGAIN' ||
err.code === 'AuthenticationFailed' ||
err.code === 'OperationTimedOut' ||
err.code === 'ServerBusy'
) {
setTimeout(secondTry, SECOND_TRY_WAIT_TIME);
return;
function secondTry() {
try {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> secondTry -> Retrying to create the file.");
writeOnlyBlobService.createBlockBlobFromText(containerName, pFolderPath + "/" + pFileName, pFileContent, onSecondTry);
function onSecondTry(err, result, response) {
try {
if (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> File not created. Giving Up."); }
if (ERROR_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> err.code = " + err.code);
}
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
} else {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> secondTry -> File succesfully created on second try.");
callBackFunction(global.DEFAULT_OK_RESPONSE);
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> secondTry -> onSecondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> secondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> Dont know what to do here. Cancelling operation. "); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
} else {
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> File Created."); }
callBackFunction(global.DEFAULT_OK_RESPONSE);
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function getTextFile(pFolderPath, pFileName, callBackFunction) {
if (readOnlyBlobService === undefined) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> initialize function not executed or failed. Can not process this request. Sorry."); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] getTextFile -> About to get a text file.");
logger.write(MODULE_NAME, "[INFO] getTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[INFO] getTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[INFO] getTextFile -> pFileName = " + pFileName);
}
readOnlyBlobService.getBlobToText(containerName, pFolderPath + "/" + pFileName, onFileReceived);
function onFileReceived(err, text, response) {
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> Response from Azure received.");
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> err = " + JSON.stringify(err));
}
if (LOG_FILE_CONTENT === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> response = " + JSON.stringify(response));
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> text = " + text);
}
if (err) {
if (ERROR_LOG === true && logger !== undefined && err.code !== 'BlobNotFound') {
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> Error trying to get this file.");
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> err.code = " + err.code);
}
if (
err.code === 'ECONNRESET' ||
err.code === 'ENOTFOUND' ||
err.code === 'ESOCKETTIMEDOUT' ||
err.code === 'ETIMEDOUT' ||
err.code === 'ECONNREFUSED' ||
err.code === 'EADDRINUSE' ||
err.code === 'EAI_AGAIN' ||
err.code === 'AuthenticationFailed' ||
err.code === 'OperationTimedOut' ||
err.code === 'ServerBusy'
) {
setTimeout(secondTry, SECOND_TRY_WAIT_TIME);
return;
function secondTry() {
try {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> secondTry -> Retrying to get the file.");
readOnlyBlobService.getBlobToText(containerName, pFolderPath + "/" + pFileName, onSecondTry);
function onSecondTry(err, text, response) {
try {
if (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> File not retrieved. Giving Up."); }
if (ERROR_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> err.code = " + err.code);
}
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
} else {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> secondTry -> File succesfully retrieved on second try.");
callBackFunction(global.DEFAULT_OK_RESPONSE, text);
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> secondTry -> onSecondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> secondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
if (err.code === "BlobNotFound") {
/* This is how Azure tell us the file does not exist. */
let customErr = {
result: global.CUSTOM_FAIL_RESPONSE.result,
message: "File does not exist."
};
logger.write(MODULE_NAME, "[WARN] getTextFile -> onFileReceived -> Custom Response -> message = " + customErr.message);
logger.write(MODULE_NAME, "[WARN] getTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFileName = " + pFileName);
callBackFunction(customErr);
return;
}
if (err.code === "ParentNotFound") {
/* This is how Azure tell us the folder where the file was supposed to be does not exist. We map this to our own response. */
let customErr = {
result: global.CUSTOM_FAIL_RESPONSE.result,
message: "Folder does not exist."
};
logger.write(MODULE_NAME, "[WARN] getTextFile -> onFileReceived -> Custom Response -> message = " + customErr.message);
logger.write(MODULE_NAME, "[WARN] getTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFileName = " + pFileName);
callBackFunction(customErr);
return;
}
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileCreated -> err = " + JSON.stringify(err)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileCreated -> response = " + JSON.stringify(response)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> Dont know what to do here. Cancelling operation. "); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
} else {
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> File retrieved."); }
callBackFunction(global.DEFAULT_OK_RESPONSE, text);
/* REPLICATION TO MINIO */
minioClient.putObject(containerName, pFolderPath + "/" + pFileName, text, 'text/plain', function (err, dataStream) {
console.log(MODULE_NAME, "[INFO] 'getTextFile' -> onFileReceived -> Replicating to MINIO -> File = " + pFolderPath + "/" + pFileName);
if (err) {
console.log(MODULE_NAME, "[WARN] 'getTextFile' -> onFileReceived -> Replicating to MINIO -> err = " + err.message);
}
}); // Temporary during migration
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
};
AzureStorage.js
+1 -1
View File
@@ -6,7 +6,7 @@
let bot = BOT;
const ROOT_DIR = './';
const MODULE_NAME = "Azure Storage";
const MODULE_NAME = "Azure Blob Storage";
const REQUEST_TIMEOUT = 180;
let thisObject = {
BlobStorage.js
+16 -371
View File
@@ -9,384 +9,29 @@ exports.newBlobStorage = function newBlobStorage(BOT, logger) {
const ROOT_DIR = './';
const MODULE_NAME = "Blob Storage";
const SECOND_TRY_WAIT_TIME = 10000;
let AZURE_STORAGE = require('./AzureStorage');
let storage = AZURE_STORAGE.newAzureStorage(bot, logger);
let thisObject = {
initialize: initialize,
createFolder: createFolder,
createTextFile: createTextFile,
getTextFile: getTextFile
};
let storage;
let readOnlyBlobService;
let writeOnlyBlobService;
let containerName;
let environment = global.CURRENT_ENVIRONMENT;
return thisObject;
function initialize(pDataOwner, callBackFunction, disableLogging) {
try {
containerName = pDataOwner.toLowerCase();
if (pDataOwner.environment !== undefined) { // This is use for data migration from one environment to the other.
environment = pDataOwner.environment;
let AZURE_STORAGE = require('./AzureBlobStorage')
let MINIO_STORAGE = require('./MinioBlobStorage')
switch (process.env.STORAGE_PROVIDER) {
case 'Azure': {
storage = AZURE_STORAGE.newAzureBlobBlobStorage(bot, logger);
break;
}
case 'Minio': {
storage = MINIO_STORAGE.newMinioBlobBlobStorage(bot, logger);
break;
}
default: {
if (ERROR_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[ERROR] newBlobStorage -> Storage Provider not supported -> process.env.STORAGE_PROVIDER = " + process.env.STORAGE_PROVIDER);
}
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> environment = " + environment); }
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> containerName = " + containerName); }
let readConnectionString = global.USER_PROFILE.storagePermissions.get(pDataOwner + "." + "READ");
if (readConnectionString !== undefined) {
readOnlyBlobService = storage.createBlobService(readConnectionString);
}
let writeConnectionString = global.USER_PROFILE.storagePermissions.get(pDataOwner + "." + "WRITE");
if (writeConnectionString !== undefined) {
writeOnlyBlobService = storage.createBlobService(writeConnectionString);
}
if (readOnlyBlobService !== undefined || writeOnlyBlobService !== undefined) {
callBackFunction(global.DEFAULT_OK_RESPONSE);
return;
} else {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> No READ or WRITE permissions found."); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> pDataOwner = " + pDataOwner); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function createFolder(pFolderPath, callBackFunction) {
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] createFolder -> Entering function."); }
try {
/* Folders do not need to be created when using blobs. */
callBackFunction(global.DEFAULT_OK_RESPONSE);
return;
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createFolder -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function createTextFile(pFolderPath, pFileName, pFileContent, callBackFunction) {
if (writeOnlyBlobService === undefined) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> initialize function not executed or failed. Can not process this request. Sorry."); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] createTextFile -> About to create a text file.");
logger.write(MODULE_NAME, "[INFO] createTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[INFO] createTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[INFO] createTextFile -> pFileName = " + pFileName);
}
writeOnlyBlobService.createBlockBlobFromText(containerName, pFolderPath + "/" + pFileName, pFileContent, onFileCreated);
function onFileCreated(err, result, response) {
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> Response from Azure received.");
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> err = " + JSON.stringify(err));
}
if (LOG_FILE_CONTENT === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> result = " + JSON.stringify(result));
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileReceived -> response = " + JSON.stringify(response));
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileReceived -> pFileContent = " + pFileContent);
}
if (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> err = " + JSON.stringify(err)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> result = " + JSON.stringify(result)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> response = " + JSON.stringify(response)); }
if (ERROR_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> Error trying to create this file.");
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> err.code = " + err.code);
}
if (
err.code === 'ECONNRESET' ||
err.code === 'ENOTFOUND' ||
err.code === 'ESOCKETTIMEDOUT' ||
err.code === 'ETIMEDOUT' ||
err.code === 'ECONNREFUSED' ||
err.code === 'EADDRINUSE' ||
err.code === 'EAI_AGAIN' ||
err.code === 'AuthenticationFailed' ||
err.code === 'OperationTimedOut' ||
err.code === 'ServerBusy'
)
{
setTimeout(secondTry, SECOND_TRY_WAIT_TIME);
return;
function secondTry() {
try {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> secondTry -> Retrying to create the file.");
writeOnlyBlobService.createBlockBlobFromText(containerName, pFolderPath + "/" + pFileName, pFileContent, onSecondTry);
function onSecondTry(err, result, response) {
try {
if (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> File not created. Giving Up."); }
if (ERROR_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> err.code = " + err.code);
}
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
} else {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> secondTry -> File succesfully created on second try.");
callBackFunction(global.DEFAULT_OK_RESPONSE);
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> secondTry -> onSecondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> secondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> Dont know what to do here. Cancelling operation. "); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
} else {
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> File Created."); }
callBackFunction(global.DEFAULT_OK_RESPONSE);
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function getTextFile(pFolderPath, pFileName, callBackFunction) {
if (readOnlyBlobService === undefined) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> initialize function not executed or failed. Can not process this request. Sorry."); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
return;
}
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] getTextFile -> About to get a text file.");
logger.write(MODULE_NAME, "[INFO] getTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[INFO] getTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[INFO] getTextFile -> pFileName = " + pFileName);
}
readOnlyBlobService.getBlobToText(containerName, pFolderPath + "/" + pFileName, onFileReceived);
function onFileReceived(err, text, response) {
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> Response from Azure received.");
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> err = " + JSON.stringify(err));
}
if (LOG_FILE_CONTENT === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> response = " + JSON.stringify(response));
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> text = " + text);
}
if (err) {
if (ERROR_LOG === true && logger !== undefined && err.code !== 'BlobNotFound') {
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> Error trying to get this file.");
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> err.code = " + err.code);
}
if (
err.code === 'ECONNRESET' ||
err.code === 'ENOTFOUND' ||
err.code === 'ESOCKETTIMEDOUT' ||
err.code === 'ETIMEDOUT' ||
err.code === 'ECONNREFUSED' ||
err.code === 'EADDRINUSE' ||
err.code === 'EAI_AGAIN' ||
err.code === 'AuthenticationFailed' ||
err.code === 'OperationTimedOut' ||
err.code === 'ServerBusy'
)
{
setTimeout(secondTry, SECOND_TRY_WAIT_TIME);
return;
function secondTry() {
try {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> secondTry -> Retrying to get the file.");
readOnlyBlobService.getBlobToText(containerName, pFolderPath + "/" + pFileName, onSecondTry);
function onSecondTry(err, text, response) {
try {
if (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> File not retrieved. Giving Up."); }
if (ERROR_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> err.code = " + err.code);
}
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
} else {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> secondTry -> File succesfully retrieved on second try.");
callBackFunction(global.DEFAULT_OK_RESPONSE, text);
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> secondTry -> onSecondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> secondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
if (err.code === "BlobNotFound") {
/* This is how Azure tell us the file does not exist. */
let customErr = {
result: global.CUSTOM_FAIL_RESPONSE.result,
message: "File does not exist."
};
logger.write(MODULE_NAME, "[WARN] getTextFile -> onFileReceived -> Custom Response -> message = " + customErr.message);
logger.write(MODULE_NAME, "[WARN] getTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFileName = " + pFileName);
callBackFunction(customErr);
return;
}
if (err.code === "ParentNotFound") {
/* This is how Azure tell us the folder where the file was supposed to be does not exist. We map this to our own response. */
let customErr = {
result: global.CUSTOM_FAIL_RESPONSE.result,
message: "Folder does not exist."
};
logger.write(MODULE_NAME, "[WARN] getTextFile -> onFileReceived -> Custom Response -> message = " + customErr.message);
logger.write(MODULE_NAME, "[WARN] getTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFileName = " + pFileName);
callBackFunction(customErr);
return;
}
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileCreated -> err = " + JSON.stringify(err)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileCreated -> response = " + JSON.stringify(response)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> Dont know what to do here. Cancelling operation. "); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
} else {
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> File retrieved."); }
callBackFunction(global.DEFAULT_OK_RESPONSE, text);
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
return storage;
};
Context.js
+3 -4
View File
@@ -490,12 +490,11 @@
thisObject.newHistoryRecord.messageRelevance,
thisObject.newHistoryRecord.messageTitle,
thisObject.newHistoryRecord.messageBody
thisObject.newHistoryRecord.messageBody,
thisObject.extraData
];
let recordWithExtraData = newRecord.concat(thisObject.extraData);
thisObject.executionHistory.push(recordWithExtraData);
thisObject.executionHistory.push(newRecord);
let fileContent = JSON.stringify(thisObject.executionHistory);
MinioBlobStorage.js
+401
View File
@@ -0,0 +1,401 @@
exports.newMinioBlobBlobStorage = function newMinioBlobBlobStorage(BOT, logger) {
let FULL_LOG = false;
let ERROR_LOG = true;
let LOG_FILE_CONTENT = false;
let bot = BOT;
const ROOT_DIR = './';
const MODULE_NAME = "Minio Blob Storage";
const SECOND_TRY_WAIT_TIME = 10000;
let thisObject = {
initialize: initialize,
createFolder: createFolder,
createTextFile: createTextFile,
getTextFile: getTextFile
};
let containerName;
let environment = global.CURRENT_ENVIRONMENT;
let Minio = require('minio');
let minioClient;
return thisObject;
function initialize(pDataOwner, callBackFunction, disableLogging) {
try {
minioClient = new Minio.Client({
endPoint: process.env.MINIO_END_POINT,
port: JSON.parse(process.env.MINIO_PORT),
useSSL: JSON.parse(process.env.MINIO_USE_SSL),
accessKey: process.env.MINIO_ACCESS_KEY,
secretKey: process.env.MINIO_SECRET_KEY
})
containerName = pDataOwner.toLowerCase();
if (pDataOwner.environment !== undefined) { // This is use for data migration from one environment to the other.
environment = pDataOwner.environment;
}
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> Entering function."); }
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> environment = " + environment); }
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] initialize -> containerName = " + containerName); }
callBackFunction(global.DEFAULT_OK_RESPONSE);
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] initialize -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function createFolder(pFolderPath, callBackFunction) {
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] createFolder -> Entering function."); }
try {
/* Folders do not need to be created when using blobs. */
callBackFunction(global.DEFAULT_OK_RESPONSE);
return;
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createFolder -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function createTextFile(pFolderPath, pFileName, pFileContent, callBackFunction) {
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] createTextFile -> About to create a text file.");
logger.write(MODULE_NAME, "[INFO] createTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[INFO] createTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[INFO] createTextFile -> pFileName = " + pFileName);
}
minioClient.putObject(containerName, pFolderPath + "/" + pFileName, pFileContent, 'text/plain', onFileCreated)
function onFileCreated(err, result, response) {
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> Response from Minio received.");
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> err = " + JSON.stringify(err));
}
if (LOG_FILE_CONTENT === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> result = " + JSON.stringify(result));
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileReceived -> response = " + JSON.stringify(response));
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileReceived -> pFileContent = " + pFileContent);
}
/* Console logging */
console.log(MODULE_NAME, "[INFO] 'createTextFile' -> onFileReceived -> File created at MINIO -> File = " + pFolderPath + "/" + pFileName);
if (err) {
console.log(MODULE_NAME, "[WARN] 'createTextFile' -> onFileReceived -> File created at MINIO -> err = " + err.message);
}
if (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> err = " + JSON.stringify(err)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> result = " + JSON.stringify(result)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> response = " + JSON.stringify(response)); }
if (ERROR_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> Error trying to create this file.");
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileReceived -> err.code = " + err.code);
}
if (
err.code === 'ECONNRESET' ||
err.code === 'ENOTFOUND' ||
err.code === 'ESOCKETTIMEDOUT' ||
err.code === 'ETIMEDOUT' ||
err.code === 'ECONNREFUSED' ||
err.code === 'EADDRINUSE' ||
err.code === 'EAI_AGAIN' ||
err.code === 'AuthenticationFailed' ||
err.code === 'OperationTimedOut' ||
err.code === 'ServerBusy'
) {
setTimeout(secondTry, SECOND_TRY_WAIT_TIME);
return;
function secondTry() {
try {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> secondTry -> Retrying to create the file.");
minioClient.putObject(containerName, pFolderPath + "/" + pFileName, pFileContent, 'text/plain', onSecondTry)
function onSecondTry(err, result, response) {
try {
if (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> File not created. Giving Up."); }
if (ERROR_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> secondTry -> err.code = " + err.code);
}
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
} else {
logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> secondTry -> File succesfully created on second try.");
callBackFunction(global.DEFAULT_OK_RESPONSE);
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> secondTry -> onSecondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> secondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] createTextFile -> onFileCreated -> Dont know what to do here. Cancelling operation. "); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
} else {
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] createTextFile -> onFileCreated -> File Created."); }
callBackFunction(global.DEFAULT_OK_RESPONSE);
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> onFileCreated -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'createTextFile' -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
function getTextFile(pFolderPath, pFileName, callBackFunction) {
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] getTextFile -> About to get a text file.");
logger.write(MODULE_NAME, "[INFO] getTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[INFO] getTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[INFO] getTextFile -> pFileName = " + pFileName);
}
minioClient.getObject(containerName, pFolderPath + "/" + pFileName, function (err, dataStream) {
let data = '';
if (err) {
onFileReceived(err, null, "Error retrieving file " + textFilename + " from bucket " + bucketName);
return
}
dataStream.on('data', function (chunk) {
data += chunk
})
dataStream.on('end', function () {
onFileReceived(null, data, "Data retrieved.");
})
dataStream.on('error', function (err) {
onFileReceived(err, null, "Error on data stream while retrieving file " + textFilename + " from bucket " + bucketName);
})
})
function onFileReceived(err, text, response) {
try {
if (FULL_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> Response from Minio received.");
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> err = " + JSON.stringify(err));
}
if (LOG_FILE_CONTENT === true && logger !== undefined) {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> response = " + JSON.stringify(response));
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> text = " + text);
}
/* Console logging */
console.log(MODULE_NAME, "[INFO] 'getTextFile' -> onFileReceived -> File read from MINIO -> File = " + pFolderPath + "/" + pFileName);
if (err) {
console.log(MODULE_NAME, "[WARN] 'getTextFile' -> onFileReceived -> File read from MINIO -> err = " + err.message);
}
if (err) {
if (ERROR_LOG === true && logger !== undefined && err.code !== 'BlobNotFound') {
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> Error trying to get this file.");
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> err.code = " + err.code);
}
if (
err.code === 'ECONNRESET' ||
err.code === 'ENOTFOUND' ||
err.code === 'ESOCKETTIMEDOUT' ||
err.code === 'ETIMEDOUT' ||
err.code === 'ECONNREFUSED' ||
err.code === 'EADDRINUSE' ||
err.code === 'EAI_AGAIN' ||
err.code === 'AuthenticationFailed' ||
err.code === 'OperationTimedOut' ||
err.code === 'ServerBusy'
) {
setTimeout(secondTry, SECOND_TRY_WAIT_TIME);
return;
function secondTry() {
try {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> secondTry -> Retrying to get the file.");
minioClient.getObject(containerName, pFolderPath + "/" + pFileName, function (err, dataStream) {
let data = '';
if (err) {
onSecondTry(err, null, "Error retrieving file " + textFilename + " from bucket " + bucketName);
return
}
dataStream.on('data', function (chunk) {
data += chunk
})
dataStream.on('end', function () {
onSecondTry(null, data, "Data retrieved.");
})
dataStream.on('error', function (err) {
onSecondTry(err, null, "Error on data stream while retrieving file " + textFilename + " from bucket " + bucketName);
})
})
function onSecondTry(err, text, response) {
try {
if (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> File not retrieved. Giving Up."); }
if (ERROR_LOG === true && logger !== undefined) {
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> containerName = " + containerName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> pFileName = " + pFileName);
logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> secondTry -> err.code = " + err.code);
}
callBackFunction(global.DEFAULT_RETRY_RESPONSE);
} else {
logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> secondTry -> File succesfully retrieved on second try.");
callBackFunction(global.DEFAULT_OK_RESPONSE, text);
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> secondTry -> onSecondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> secondTry -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
if (err.code === "NoSuchKey") {
/* This is how Minio tell us the file does not exist. */
let customErr = {
result: global.CUSTOM_FAIL_RESPONSE.result,
message: "File does not exist."
};
logger.write(MODULE_NAME, "[WARN] getTextFile -> onFileReceived -> Custom Response -> message = " + customErr.message);
logger.write(MODULE_NAME, "[WARN] getTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFileName = " + pFileName);
callBackFunction(customErr);
return;
}
if (err.code === "ParentNotFound") {
/* This is how Minio tell us the folder where the file was supposed to be does not exist. We map this to our own response. */
let customErr = {
result: global.CUSTOM_FAIL_RESPONSE.result,
message: "Folder does not exist."
};
logger.write(MODULE_NAME, "[WARN] getTextFile -> onFileReceived -> Custom Response -> message = " + customErr.message);
logger.write(MODULE_NAME, "[WARN] getTextFile -> containerName = " + containerName);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFolderPath = " + pFolderPath);
logger.write(MODULE_NAME, "[WARN] getTextFile -> pFileName = " + pFileName);
callBackFunction(customErr);
return;
}
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileCreated -> err = " + JSON.stringify(err)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileCreated -> response = " + JSON.stringify(response)); }
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] getTextFile -> onFileReceived -> Dont know what to do here. Cancelling operation. "); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
} else {
if (FULL_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[INFO] getTextFile -> onFileReceived -> File retrieved."); }
callBackFunction(global.DEFAULT_OK_RESPONSE, text);
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> onFileReceived -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
}
catch (err) {
if (ERROR_LOG === true && logger !== undefined) { logger.write(MODULE_NAME, "[ERROR] 'getTextFile' -> err = " + err.message); }
callBackFunction(global.DEFAULT_FAIL_RESPONSE);
}
}
};
Project.njsproj
+6
View File
@@ -33,12 +33,18 @@
<Compile Include="CloudRequire.js">
<SubType>Code</SubType>
</Compile>
<Compile Include="MinioBlobStorage.js">
<SubType>Code</SubType>
</Compile>
<Compile Include="MultiPeriodDaily.js">
<SubType>Code</SubType>
</Compile>
<Compile Include="MultiPeriodMarket.js">
<SubType>Code</SubType>
</Compile>
<Compile Include="AzureBlobStorage.js">
<SubType>Code</SubType>
</Compile>
<Compile Include="utils\auth.js">
<SubType>Code</SubType>
</Compile>
README.md
+11 -15
View File
@@ -1,28 +1,24 @@
# Clone Executor
Whithin the Superalgos project, we have the concept of Financial Beings, which are autonomous agents that have a financial life. These FBs entities have a master (imagine a class in programming) and clones (imagine an instance of a class running). The master is the one who aquires reputation after competitions. Every time we need to do a backtest, live trading or participating in a competition, a clone is extracted from the master and is put to run, either in backtest mode, live trading mode or competition mode. The clone lifespan is contrained by the purpose it was created. For instance, a clone created for a competition will live only during the period of such competition.
Whithin the Superalgos project, we have the concept of Financial Beings, which are autonomous agents that have a financial life. These FBs entities have a master (imagine a class in programming) and clones (imagine an instance of a class running). The master is the one who aquires reputation after competitions. Every time we need to do a backtest, live trading or participating in a competition, a clone is extracted from the master and is put to run, either in backtest mode, live trading mode or competition mode. The clone lifespan is constrained by the purpose it was created. For instance, a clone created for a competition will live only during the period of such competition.
The software needed to execute these clones is the Clone Executor.
## About the Superalgos Project
## Installation
Superalgos is built and maintained by the Superalgos Foundation and its Community.
To install the Clone Executor plase clone this repo and:
Our mission is to enable the fast-paced evolution of crypto trading bots within the context of trading bots competitions.
```
npm install
```
The project hosts competitions in which multidisciplinary teams (developers, traders, analysts, etc) create open source bots, a class of which end up competing against each other. Just like with any other open source project, developers are free to fork any bot's source code to add their own innovations. Forked bots then compete with the original ones, sparking a fierce around-the-clock evolutionary competition.
Then, ask for the .ENV and config files to the Core Team (This is a temporary measure until we pack everything into a Superalgos Node).
Both the AAPlatform and AA Bots are programed in JavaScript.
You can use then the included visual studio solution to start debugging right away, or:
## Get Involved
If you are interested in coding your own crypto trading bots and eventually participate in our competitions, this is how you can get started:
* [First Closed Doors Competition Documentation](https://github.com/AAArena/First-Closed-Doors-Competition/blob/master/README.md).
<!---
* [Introduction](https://github.com/AdvancedAlgos/Documentation/wiki)
* [Getting Started Guide](https://github.com/AdvancedAlgos/Documentation/wiki/Overview)
* [Best Practices](https://github.com/AdvancedAlgos/Documentation/wiki/Bot's-README-Files)--->
```
node.exe Run.js
```
## Copyright and License Notice
package-lock.json
Generated
+789 -631
View File
File diff suppressed because it is too large. Load diff
package.json
+1
View File
@@ -19,6 +19,7 @@
"graphql": "^14.0.2",
"jsonwebtoken": "^8.4.0",
"lodash": "^4.17.11",
"minio": "^7.0.5",
"nodemailer": "^4.6.8",
"nonce": "^1.0.4",
"querystring": "^0.2.0",