From 33c2ddb1506a418808a7e6a891639f5f423083f7 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Feb 2019 10:44:37 +0100 Subject: [PATCH 01/92] Initial commit --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..dfe077042 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto From e13507a18ae511b035249bd791782848c4810324 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Feb 2019 10:47:33 +0100 Subject: [PATCH 02/92] Initial Commit --- DailyFiles.js | 392 +++++++++++++++++++++++++++ FileCloud.js | 252 ++++++++++++++++++ FileCursor.js | 688 ++++++++++++++++++++++++++++++++++++++++++++++++ FileSequence.js | 385 +++++++++++++++++++++++++++ MarketFiles.js | 253 ++++++++++++++++++ README.md | 27 ++ SingleFile.js | 105 ++++++++ 7 files changed, 2102 insertions(+) create mode 100644 DailyFiles.js create mode 100644 FileCloud.js create mode 100644 FileCursor.js create mode 100644 FileSequence.js create mode 100644 MarketFiles.js create mode 100644 README.md create mode 100644 SingleFile.js diff --git a/DailyFiles.js b/DailyFiles.js new file mode 100644 index 000000000..c7cbf8314 --- /dev/null +++ b/DailyFiles.js @@ -0,0 +1,392 @@ + +function newDailyFiles() { + + const MODULE_NAME = "Daily Files"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + let thisObject = { + getFileCursor: getFileCursor, + setDatetime: setDatetime, + setTimePeriod: setTimePeriod, + getExpectedFiles: getExpectedFiles, + getFilesLoaded: getFilesLoaded, + initialize: initialize, + finalize: finalize + } + + let filesLoaded = 0; + let expectedFiles = 0; + + let fileCloud; + + let fileCursors = new Map; + + let callBackWhenFileReceived; + + return thisObject; + + function finalize() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } + + filesLoaded = undefined; + expectedFiles = undefined; + fileCloud = undefined; + + fileCursors.forEach(finalizeEach); + + function finalizeEach(item, key, mapObj) { + + if (INFO_LOG === true) { logger.write("[INFO] finalize -> finalizeEach -> Entering function."); } + + item.finalize(); + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } + } + } + + function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } + + callBackWhenFileReceived = callBackFunction; + + let exchange = ecosystem.getExchange(pProduct, pExchange); + + if (exchange === undefined) { + + throw "Exchange not supoorted by this product of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; + + } + + fileCloud = newFileCloud(); + fileCloud.initialize(pBot); + + /* First we will get the Data Range */ + + fileCloud.getFile(pDevTeam, pBot, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived); + + function onDataRangeReceived(err, pFile) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Entering function."); } + + let beginDateRange; + let endDateRange; + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received OK Response."); } + + beginDateRange = new Date(pFile.begin); + endDateRange = new Date(pFile.end); + + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> err.message = " + err.message); } + + if (err.message === "File does not exist.") { + + if (INFO_LOG === true) { logger.write("[WARN] initialize -> onFileReceived -> No Date Range file found. maxDate will be set to current datetime. "); } + + endDateRange = new Date(); + break; + } + + if (err.message === "Missing Configuration.") { + + if (ERROR_LOG === true) { logger.write("[WARN] initialize -> onFileReceived -> The needed configuration for the dateRange at the dataSet of the product was not found."); } + if (ERROR_LOG === true) { logger.write("[WARN] initialize -> onFileReceived -> maxDate will be set to current datetime."); } + + endDateRange = new Date(); + break; + } + + /* If none of the previous conditions are met, the we return the err to the caller. */ + + callBackFunction(err); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + /* Now we will get the daily files */ + + for (i = 0; i < dailyFilePeriods.length; i++) { + + let periodTime = dailyFilePeriods[i][0]; + let periodName = dailyFilePeriods[i][1]; + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> periodTime = " + periodTime); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> periodName = " + periodName); } + + if (pSet.validPeriods.includes(periodName) === true) { + + let fileCursor = newFileCursor(); + fileCursor.initialize(fileCloud, pDevTeam, pBot, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized); + + function onInitialized(err) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Received FAIL Response."); } + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Received Unexpected Response."); } + callBackWhenFileReceived(err); + return; + } + } + + fileCursors.set(periodTime, fileCursor); + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> expectedFiles = " + expectedFiles); } + + expectedFiles = expectedFiles + fileCursor.getExpectedFiles(); + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> expectedFiles = " + expectedFiles); } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> onInitialized -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + } + } + + loadThemAll(); + + function loadThemAll() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> loadThemAll -> Entering function."); } + + for (i = 0; i < dailyFilePeriods.length; i++) { + + let periodTime = dailyFilePeriods[i][0]; + let periodName = dailyFilePeriods[i][1]; + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> loadThemAll -> periodTime = " + periodTime); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> loadThemAll -> periodName = " + periodName); } + + if (pSet.validPeriods.includes(periodName) === true) { + + let fileCursor = fileCursors.get(periodTime); + fileCursor.reload(onFileReceived); + + } + } + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> loadThemAll -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function onFileReceived(err) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received FAIL Response."); } + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> err.message = " + err.message); } + + callBackWhenFileReceived(err); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received Unexpected Response."); } + callBackWhenFileReceived(err); + return; + } + } + + filesLoaded++; + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> filesLoaded = " + filesLoaded); } + + callBackWhenFileReceived(GLOBAL.DEFAULT_OK_RESPONSE, thisObject); // Note that the call back is called for every file loaded at each cursor. + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] onFileReceived -> err = " + err); } + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function getFileCursor(pPeriod) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFileCursor -> Entering function."); } + + return fileCursors.get(pPeriod); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFileCursor -> err = " + err); } + + } + } + + function setDatetime(pDatetime) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> Entering function."); } + + filesLoaded = 0; + expectedFiles = 0; + + fileCursors.forEach(setDatetimeToEach) + + function setDatetimeToEach(fileCursor, key, map) { + + fileCursor.setDatetime(pDatetime); + + if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> expectedFiles = " + expectedFiles); } + + expectedFiles = expectedFiles + fileCursor.getExpectedFiles(); + + if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> expectedFiles = " + expectedFiles); } + + fileCursor.reload(onFileReceived); + + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setDatetime -> err = " + err); } + + } + } + + function setTimePeriod(pTimePeriod, pDatetime) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Entering function."); } + + filesLoaded = 0; + expectedFiles = 0; + + fileCursors.forEach(setTimePeriodToEach) + + function setTimePeriodToEach(fileCursor, key, map) { + + fileCursor.setTimePeriod(pTimePeriod, pDatetime); + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> expectedFiles = " + expectedFiles); } + + expectedFiles = expectedFiles + fileCursor.getExpectedFiles(); + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> expectedFiles = " + expectedFiles); } + + fileCursor.reload(onFileReceived); + + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> err = " + err); } + + } + } + + function getExpectedFiles() { + + if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> expectedFiles = " + expectedFiles); } + + return expectedFiles; + + } + + function getFilesLoaded() { + + if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> filesLoaded = " + filesLoaded); } + + return filesLoaded; + + } +} \ No newline at end of file diff --git a/FileCloud.js b/FileCloud.js new file mode 100644 index 000000000..4173eb528 --- /dev/null +++ b/FileCloud.js @@ -0,0 +1,252 @@ + +function newFileCloud() { + + const MODULE_NAME = "File Cloud"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + /* + + This is the module in the system that actually connects to the cloud storage and grabs from there the needed files. + + */ + + var thisObject = { + getFile: getFile, + initialize: initialize + }; + + let blobService; + + return thisObject; + + function initialize(pBot) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> pBot = " + pBot.codeName); } + + blobService = AzureStorage.Blob.createBlobServiceWithSas(pBot.storage.fileUri, pBot.storage.sas); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + //callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); TODO> Handle this callback. + } + } + + function getFile(pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } + + const MAX_RETRIES = 3; + + getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + + function getFileRecursively(pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pSet.filePath + "-" + pSet.fileName); } + + let fileName; + let filePath; + + if (pDataRange === undefined) { + + fileName = pSet.fileName; + filePath = pSet.filePath; + + } else { + + if (pSet.dataRange !== undefined) { + + fileName = pSet.dataRange.fileName; + filePath = pSet.dataRange.filePath; + + } else { + + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: "Missing Configuration." + }; + + if (ERROR_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = " + customErr.message); } + if (ERROR_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found. "); } + + callBackFunction(customErr); + return; + } + } + + if (fileName === undefined) { + + logger.write("[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: "); + logger.write("[ERROR] getFile -> getFileRecursively -> pDevTeam = " + JSON.stringify(pDevTeam)); + logger.write("[ERROR] getFile -> getFileRecursively -> pBot = " + JSON.stringify(pBot)); + logger.write("[ERROR] getFile -> getFileRecursively -> pSet = " + JSON.stringify(pSet)); + logger.write("[ERROR] getFile -> getFileRecursively -> pExchange = " + JSON.stringify(pExchange)); + logger.write("[ERROR] getFile -> getFileRecursively -> pMarket = " + JSON.stringify(pMarket)); + logger.write("[ERROR] getFile -> getFileRecursively -> pPeriodName = " + JSON.stringify(pPeriodName)); + + throw ("Inconsistant data received."); + } + + if (pMarket !== undefined) { + + fileName = fileName.replace("@AssetA", pMarket.assetA); + fileName = fileName.replace("@AssetB", pMarket.assetB); + } + + if (pDevTeam !== undefined) { + + filePath = filePath.replace("@DevTeam", pDevTeam.codeName); + } + + if (pBot !== undefined) { + + filePath = filePath.replace("@Bot", pBot.codeName); + } + + if (pExchange !== undefined) { + + filePath = filePath.replace("@Exchange", pExchange.name); + } + + filePath = filePath.replace("@Period", pPeriodName); + + if (pDatetime !== undefined) { + + filePath = filePath.replace("@Year", pDatetime.getUTCFullYear()); + filePath = filePath.replace("@Month", pad(pDatetime.getUTCMonth() + 1, 2)); + filePath = filePath.replace("@Day", pad(pDatetime.getUTCDate(), 2)); + filePath = filePath.replace("@Hour", pad(pDatetime.getUTCHours(), 2)); + filePath = filePath.replace("@Minute", pad(pDatetime.getUTCMinutes(), 2)); + + } + + if (pSequence !== undefined) { + + fileName = fileName.replace("@Sequence", pSequence); + } + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> filePath = " + filePath); } + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> fileName = " + fileName); } + + let containerName; + + containerName = pDevTeam.codeName.toLowerCase(); + + blobService.getBlobToText(containerName, filePath + "/" + fileName, onFileReceived); + + function onFileReceived(err, text, response) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = " + filePath); } + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> fileName = " + fileName); } + + let data; + + if (err) { + + if (err.code === "BlobNotFound" | err.code === "FileNotFound" | err.code === "ParentNotFound") { + + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: "File does not exist." + }; + + if (INFO_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = " + customErr.message); } + + callBackFunction(customErr); + return; + } + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> containerName = " + containerName); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> filePath = " + filePath); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> fileName = " + fileName); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> Unexpected Error Ocurred."); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = " + err); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> text = " + text); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> response = " + response); } + + if (err.message === "XHR error") { + + if (pRetryCounter < MAX_RETRIES) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> Retrying to get this file. "); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = " + MAX_RETRIES); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = " + pRetryCounter); } + + getFileRecursively(pRetryCounter + 1, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + return; + + } else { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. "); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = " + MAX_RETRIES); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = " + pRetryCounter); } + + } + } + + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + + } else { + + try { + + data = JSON.parse(text); + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> Data received is valid JSON."); } + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data); + return; + + } catch (err) { + + if (INFO_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> err = " + err); } + + let customErr = { + result: GLOBAL.CUSTOM_OK_RESPONSE.result, + message: "Data not in JSON Format." + }; + + if (INFO_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = " + customErr.message); } + + callBackFunction(customErr, text); + return; + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } +} \ No newline at end of file diff --git a/FileCursor.js b/FileCursor.js new file mode 100644 index 000000000..9592c2c1f --- /dev/null +++ b/FileCursor.js @@ -0,0 +1,688 @@ + +function newFileCursor() { + + const MODULE_NAME = "File Cursor"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + let cursorDate; + + let thisObject = { + reload: reload, + setDatetime: setDatetime, + setTimePeriod: setTimePeriod, + files: new Map, + getExpectedFiles: getExpectedFiles, + initialize: initialize, + finalize: finalize + }; + + let minCursorSize = 10; + let maxCursorSize = 30; + + let market; + let exchange; + let fileCloud; + let devTeam; + let bot; + let thisSet; + let periodName; + let timePeriod; + let beginDateRange; + let endDateRange; + + let intervalHandle; + + return thisObject; + + function finalize() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } + + clearInterval(intervalHandle); + + thisObject.files = undefined; + cursorDate = undefined; + fileCloud = undefined; + periodName = undefined; + timePeriod = undefined; + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } + } + } + + function initialize(pFileCloud, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pSet.codeName + "-" + pPeriodName); } + + market = pMarket; + exchange = pExchange; + fileCloud = pFileCloud; + devTeam = pDevTeam; + bot = pBot; + thisSet = pSet; + periodName = pPeriodName; + cursorDate = pCursorDate; + timePeriod = pTimePeriod; + beginDateRange = pBeginDateRange; + endDateRange = pEndDateRange; + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> market = " + market); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> periodName = " + periodName); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> cursorDate = " + cursorDate); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> beginDateRange = " + beginDateRange); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> endDateRange = " + endDateRange); } + + setTimePeriod(pCurrentTimePeriod, pCursorDate); + + intervalHandle = setInterval(updateFiles , timePeriod); + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function updateFiles() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Entering function."); } + + /* + + In order to know if we need to update any file we follow this logic: + + 1. If the current date is already a file in the cursor, we need to update it. + 2. If the current date is one day ahead of a file in the cursor, we need to add this one to the cursor. + + */ + + let targetDate; + let dateString; + let file; + + /* Situation 1 */ + + targetDate = new Date(); + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); + + file = thisObject.files.get(dateString); + + if (file !== undefined) { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> File found at the cursor, proceeding to update."); } + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived); + + return; + } + + /* Situation 2 */ + + targetDate = new Date( (new Date()).valueOf() - ONE_DAY_IN_MILISECONDS); + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); + + file = thisObject.files.get(dateString); // This is from yesterday + + targetDate = new Date(); + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); + + if (file !== undefined) { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Previous day file found at the cursor, proceeding to add current date."); } + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived); + + return; + } + + function onFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received FAIL Response."); } + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received Unexpected Response."); } + return; + } + } + + thisObject.files.set(dateString, file); + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> File updated."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> devTeam = " + devTeam.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> bot = " + bot.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> thisSet = " + thisSet.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> dateString = " + dateString); } + + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> err = " + err); } + } + } + + function setTimePeriod(pTimePeriod, pDatetime) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Entering function."); } + + /* + + We are implementing here an algorithm designed to save bandwidth, memory and processing power at the browser. + We say there is a saving mode where the cursor is running at a minimum size. When the end user aproaches the time period the cursor + is set, then it should exit the saving mode and go to its actual size. + + To do this we are going to measure the distance from the Time Period received to the one the cursors was initialized with. + If these periods are consecutive, it means that the cursor should exit saving mode and load its full size. + + */ + + cursorDate = pDatetime; // Adjust the cursor date to the one received. + + let positionA; + + for (let i = 0; i < dailyFilePeriods.length; i++) { + + let period = dailyFilePeriods[i]; + + if (period[0] === pTimePeriod) { + + positionA = i; + + } + + if (period[0] === timePeriod) { + + positionB = i; + + } + } + + if (Math.abs(positionB - positionA) <= 1) { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Will EXIT Saving Mode."); } + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> pTimePeriod = " + convertTimePeriodToName(pTimePeriod)); } + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> timePeriod = " + convertTimePeriodToName(timePeriod)); } + + exitSavingMode(); + + } else { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Will ENTER Saving Mode."); } + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> pTimePeriod = " + convertTimePeriodToName(pTimePeriod)); } + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> timePeriod = " + convertTimePeriodToName(timePeriod)); } + + enterSavingMode(); + + } + + function enterSavingMode() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> enterSavingMode -> Entering function."); } + + switch (timePeriod) { + + case _45_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 30; + } + break; + case _40_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 20; + } + break; + case _30_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _20_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _15_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _10_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _5_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _4_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _3_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _2_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _1_MINUTE_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + default: + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> enterSavingMode -> err = " + err); } + } + } + + function exitSavingMode() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> exitSavingMode -> Entering function."); } + + switch (timePeriod) { + + case _45_MINUTES_IN_MILISECONDS: + { + minCursorSize = 15; + maxCursorSize = 30; + } + break; + case _40_MINUTES_IN_MILISECONDS: + { + minCursorSize = 13; + maxCursorSize = 20; + } + break; + case _30_MINUTES_IN_MILISECONDS: + { + minCursorSize = 11; + maxCursorSize = 15; + } + break; + case _20_MINUTES_IN_MILISECONDS: + { + minCursorSize = 9; + maxCursorSize = 15; + } + break; + case _15_MINUTES_IN_MILISECONDS: + { + minCursorSize = 7; + maxCursorSize = 15; + } + break; + case _10_MINUTES_IN_MILISECONDS: + { + minCursorSize = 7; + maxCursorSize = 15; + } + break; + case _5_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5; + maxCursorSize = 15; + } + break; + case _4_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5; + maxCursorSize = 15; + } + break; + case _3_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5; + maxCursorSize = 15; + } + break; + case _2_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5; + maxCursorSize = 15; + } + break; + case _1_MINUTE_IN_MILISECONDS: + { + minCursorSize = 5; + maxCursorSize = 15; + } + break; + default: + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> exitSavingMode -> err = " + err); } + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> err = " + err); } + } + } + + function setDatetime(pDatetime) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> Entering function."); } + + if (pDatetime === undefined) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setDatetime -> Received undefined datetime."); } + return; + } + + cursorDate = pDatetime; + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setDatetime -> err = " + err); } + } + } + + function reload(callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] reload -> Entering function."); } + + getFiles(callBackFunction); + + collectGarbage(callBackFunction); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] reload -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function getFiles(callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> Entering function."); } + + let i = 0; + let j = 0; + + let dateString; + + getNextFile(); + + function getNextFile() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> Entering function."); } + + let targetDate = new Date(cursorDate); + targetDate.setUTCDate(targetDate.getUTCDate() + j); + + /* Small algorith to allow load first the current date, then alternate between the most forwad and the most backwards ones. */ + if (j === 0) { j++; } + else { + if (j < 0) { + j = -j; + j++; + } + else { + j = -j; + } + } + + let compareTargetDate = new Date(targetDate.toDateString()); + + if (beginDateRange !== undefined) { + + let compareBeginDate = new Date(beginDateRange.toDateString()); + + if (compareTargetDate.valueOf() < compareBeginDate.valueOf()) { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> Not loading the file for this date."); } + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> compareTargetDate = " + compareTargetDate); } + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> compareBeginDate = " + compareBeginDate); } + + controlLoop(); + return; + } + } + + if (endDateRange !== undefined) { + + let compareEndDate = new Date(endDateRange.toDateString()); + + if (compareTargetDate.valueOf() > compareEndDate.valueOf()) { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> Not loading the file for this date."); } + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> compareTargetDate = " + compareTargetDate); } + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> compareEndDate = " + compareEndDate); } + + controlLoop(); + return; + } + } + + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); + + let currentDay = Math.trunc((new Date()).valueOf() / (24 * 60 * 60 * 1000)); + let targetDay = Math.trunc(targetDate.valueOf() / (24 * 60 * 60 * 1000)); + + if (targetDay > currentDay) { + + controlLoop(); + + } else { + + if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived); + + } else { + + controlLoop(); + } + + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> getNextFile -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function onFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> err.message = " + err.message); } + + if (err.message === "File does not exist.") { + + controlLoop(); + return; + + } else { + + callBackFunction(err); + return; + } + + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + thisObject.files.set(dateString, file); + + controlLoop(); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function controlLoop() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> controlLoop -> Entering function."); } + + if (callBackFunction !== undefined) { + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE); + + } + + i++; + + if (i < minCursorSize) { + + getNextFile(); + + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> controlLoop -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function collectGarbage(callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] collectGarbage -> Entering function."); } + + date = removeTime(cursorDate); + + let minDate = date.valueOf() - maxCursorSize * ONE_DAY_IN_MILISECONDS / 2; + let maxDate = date.valueOf() + maxCursorSize * ONE_DAY_IN_MILISECONDS / 2; + + for (let key of thisObject.files.keys()) { + + let keyDate = new Date(key); + + if (keyDate.valueOf() < minDate || keyDate.valueOf() > maxDate) { + + thisObject.files.delete(key); + + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] collectGarbage -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + + } + + function getExpectedFiles() { + + if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } + + return minCursorSize; + + } + +} \ No newline at end of file diff --git a/FileSequence.js b/FileSequence.js new file mode 100644 index 000000000..5028143d4 --- /dev/null +++ b/FileSequence.js @@ -0,0 +1,385 @@ + +function newFileSequence() { + + const MODULE_NAME = "File Sequence"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + let thisObject = { + eventHandler: undefined, + getFile: getFile, + getExpectedFiles: getExpectedFiles, + getFilesLoaded: getFilesLoaded, + initialize: initialize, + finalize: finalize + } + + let filesLoaded = 0; + let fileCloud; + let files = new Map; + let maxSequence = -1; // This is replaced by the content of the sequence file, which contains an index that starts on zero. In the case that the sequence file is not found the default value is -1 so when you add 1 it gives you the amount of files in the sequence, zero. + let market; + let devTeam; + let bot; + let thisSet; + let product; + + thisObject.eventHandler = newEventHandler(); + + let intervalHandle; + + return thisObject; + + function finalize() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] finalize -> devTeam = " + devTeam.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] finalize -> bot = " + bot.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] finalize -> thisSet = " + thisSet.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] finalize -> intervalHandle = " + intervalHandle); } + + clearInterval(intervalHandle); + + filesLoaded = undefined; + fileCloud = undefined; + files = undefined; + maxSequence = undefined; + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } + } + } + + function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } + + exchange = ecosystem.getExchange(pProduct, pExchange); + + if (exchange === undefined) { + + throw "Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; + + } + + intervalHandle = setInterval(updateFiles, _1_MINUTE_IN_MILISECONDS); + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> intervalHandle = " + intervalHandle); } + + market = pMarket; + devTeam = pDevTeam; + bot = pBot; + thisSet = pSet; + product = pProduct; + + fileCloud = newFileCloud(); + fileCloud.initialize(bot); + + /* First we will get the sequence max number */ + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, "Sequence", undefined, onSequenceFileReceived); + + function onSequenceFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> key = " + devTeam.codeName + "-" + bot.codeName + "-" + product.codeName); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received CUSTOM OK Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> err.message = " + err.message); } + + callBackFunction(err); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> err.message = " + err.message); } + + callBackFunction(err, thisObject); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + maxSequence = Number(file); + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> maxSequence = " + maxSequence); } + + /* Now we will get the sequence of files */ + + for (let i = 0; i <= maxSequence; i++) { + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived); + + function onFileReceived(err, file) { + + try { + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> err.message = " + err.message); } + + callBackFunction(err); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + files.set(i, file); + + filesLoaded++; + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject); // Note that the callback is called for every file loaded. + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onSequenceFileReceived -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onSequenceFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function updateFiles() { + + try { + + let updateFiles = 0; + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Entering function."); } + + /* + + To keep this data structure up-to-date we need to: + + 1. Re-read the sequence file. + 2. Re-load the current last file. + 3. Re-load all new files after our last one. + + */ + + let currentMaxSequence = maxSequence; + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> currentMaxSequence = " + currentMaxSequence); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> intervalHandle = " + intervalHandle); } + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> devTeam = " + devTeam.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> bot = " + bot.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> thisSet = " + thisSet.codeName); } + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, "Sequence", undefined, onSequenceFileReceived); + + function onSequenceFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> key = " + devTeam.codeName + "-" + bot.codeName + "-" + product.codeName); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received FAIL Response."); } + return; + } + + case GLOBAL.CUSTOM_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received CUSTOM OK Response."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> err.message = " + err.message); } + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> err.message = " + err.message); } + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received Unexpected Response."); } + return; + } + } + + maxSequence = Number(file); + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> maxSequence = " + maxSequence); } + + /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ + + for (let i = currentMaxSequence; i <= maxSequence; i++) { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> i = " + i); } + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived); + + function onFileReceived(err, file) { + + try { + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received FAIL Response."); } + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> err.message = " + err.message); } + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received Unexpected Response."); } + return; + } + } + + files.set(i, file); + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> File Updated."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> i = " + i); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> file.length = " + file.length); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> filesLoaded = " + filesLoaded); } + + if (i !== currentMaxSequence) { + + filesLoaded++; + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> filesLoaded = " + filesLoaded); } + + } + + if (filesLoaded === maxSequence + 1) { + + thisObject.eventHandler.raiseEvent("Files Updated", undefined); + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Files Updated event Raised."); } + + } + + } catch (err) { + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onSequenceFileReceived -> onFileReceived -> err = " + err); } + } + } + } + + } catch (err) { + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onSequenceFileReceived -> err = " + err); } + } + } + + } catch (err) { + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> err = " + err); } + } + } + + function getFile(pSequence) { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] getFile -> pSequence = " + pSequence); } + + return files.get(pSequence); + + } + + function getExpectedFiles() { + + if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } + + return maxSequence + 1; + + } + + function getFilesLoaded() { + + if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> Entering function."); } + + return filesLoaded; + + } + +} \ No newline at end of file diff --git a/MarketFiles.js b/MarketFiles.js new file mode 100644 index 000000000..0bbf6d116 --- /dev/null +++ b/MarketFiles.js @@ -0,0 +1,253 @@ + +function newMarketFiles() { + + const MODULE_NAME = "Market Files"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + let thisObject = { + eventHandler: undefined, + getFile: getFile, + getExpectedFiles: getExpectedFiles, + getFilesLoaded: getFilesLoaded, + initialize: initialize, + finalize: finalize + } + + let filesLoaded = 0; + + let fileCloud; + + let files = new Map; + + let market; + let exchange; + let devTeam; + let bot; + let thisSet; + let periodName; + + thisObject.eventHandler = newEventHandler(); + + let intervalHandle; + + return thisObject; + + function finalize() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } + + clearInterval(intervalHandle); + + filesLoaded = undefined; + files = undefined; + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } + } + } + + function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } + + exchange = ecosystem.getExchange(pProduct, pExchange); + + if (exchange === undefined) { + + throw "Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; + + } + + intervalHandle = setInterval(updateFiles, _10_MINUTES_IN_MILISECONDS); + + market = pMarket; + devTeam = pDevTeam; + bot = pBot; + thisSet = pSet; + + fileCloud = newFileCloud(); + fileCloud.initialize(pBot); + + /* Now we will get the market files */ + + for (let i = 0; i < marketFilesPeriods.length; i++) { + + let periodTime = marketFilesPeriods[i][0]; + let periodName = marketFilesPeriods[i][1]; + + if (thisSet.validPeriods.includes(periodName) === true) { + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived); + + function onFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> err.message = " + err.message); } + + callBackFunction(err); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + files.set(periodTime, file); + + filesLoaded++; + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject); // Note that the callback is called for every file loaded. + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + } + } + + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function updateFiles() { + + try { + + let updateFiles = 0; + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Entering function."); } + + /* Now we will get the market files */ + + for (let i = 0; i < marketFilesPeriods.length; i++) { + + let periodTime = marketFilesPeriods[i][0]; + let periodName = marketFilesPeriods[i][1]; + + if (thisSet.validPeriods.includes(periodName) === true) { + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived); + + function onFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received FAIL Response."); } + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> err.message = " + err.message); } + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received Unexpected Response."); } + return; + } + } + + files.set(periodTime, file); + updateFiles++; + + if (updateFiles === marketFilesPeriods.length) { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> All files received. "); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> devTeam = " + devTeam.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> bot = " + bot.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> thisSet = " + thisSet.codeName); } + + thisObject.eventHandler.raiseEvent("Files Updated", undefined); + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onFileReceived -> err = " + err); } + } + } + } + } + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> err = " + err); } + } + } + + function getFile(pPeriod) { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } + + return files.get(pPeriod); + + } + + function getExpectedFiles() { + + if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } + + return marketFilesPeriods.length; + + } + + function getFilesLoaded() { + + if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> Entering function."); } + + return filesLoaded; + + } + +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 000000000..b6abe364f --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +# Files Component + +The Files Component provides the functionality needed to load into the browser's memory data structures populated from plain text files grabbed from the cloud and optimized for different scenarios. Each of these scenarios are represented by a single js module as listed below. + +Once the information is initially loaded, it is automatically updated as times goes by or as the users changes its position on the timeline, both on the time dimension as in the time period (1m, 2m, 3m, 4m, 5m, 10m, ..., 1 hour, 2 hours, ..., 24 hs) dimension. + +## Single File + +As the name suggests, this is used to load a single file. + +## File Sequence + +This is a multi-file data structure in which is stored information of a sequence of files. Let's imagine that the App using this component needs to read File_1, File_2, File_3 and File_4, this data structure is capable of doing that. + +## Market File + +The system-wide concept of a Market File is defined as a file containing data corresponding to the whole market history. If the data were candles for instance, it would be containing all candles since the beginning on the market until the head of the market (the most recent information). + +Since for each market there are multiple Market Files (one for 1 hour time period, one for 2 hours, 4 hours, 6, 8, 12 and 24 hs.) this data structure loads all of these files and have them readily available in memory for consumption. + +## Daily Files + +The system-wide concept of a Daily File is defined as a file containing data corresponding only to one single day. If the data were candles for instance, it would be containing all candles calculated from trades that happened in a single day only. + +Since for each market there are multiple Daily Files (one for 1 min time period, one for 2 min, 3 min, 4, 5, 10, 15, 20, 30 and 45 minutes) this data structure loads all of these files and have them readily available in memory for consumption. + +Moreover, it uses the concept of a File Cursor to load more files of the previous days and the days following days in order to have them ready in memory if needed. diff --git a/SingleFile.js b/SingleFile.js new file mode 100644 index 000000000..65c8d8472 --- /dev/null +++ b/SingleFile.js @@ -0,0 +1,105 @@ + +function newSingleFile() { + + const MODULE_NAME = "Single File"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + let thisObject = { + getFile: getFile, + initialize: initialize + } + + let fileCloud; + let file; + + return thisObject; + + function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } + + let exchange = ecosystem.getExchange(pProduct, pExchange); + + if (pExchange !== undefined && exchange === undefined) { // We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product. + + throw "Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; + + } + + fileCloud = newFileCloud(); + fileCloud.initialize(pBot); + + /* Now we will get the file */ + + fileCloud.getFile(pDevTeam, pBot, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived); + + function onFileReceived(err, pFile) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> err.message = " + err.message); } + + callBackFunction(err); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + file = pFile; + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function getFile() { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } + + return file; + + } + +} \ No newline at end of file From 460b4db59ea43cb863897f2d0cc340a7f19188cf Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Feb 2019 10:51:20 +0100 Subject: [PATCH 03/92] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b6abe364f..a22ed6bfe 100644 --- a/README.md +++ b/README.md @@ -24,4 +24,4 @@ The system-wide concept of a Daily File is defined as a file containing data cor Since for each market there are multiple Daily Files (one for 1 min time period, one for 2 min, 3 min, 4, 5, 10, 15, 20, 30 and 45 minutes) this data structure loads all of these files and have them readily available in memory for consumption. -Moreover, it uses the concept of a File Cursor to load more files of the previous days and the days following days in order to have them ready in memory if needed. +Moreover, it uses the concept of a File Cursor to load more files of the previous days and the following days in order to have them ready in memory if needed. From edf3fd12a8f0d581fa65f4d042f9282dfa5b49b6 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Feb 2019 11:01:45 +0100 Subject: [PATCH 04/92] src folder introduced --- DailyFiles.js => src/DailyFiles.js | 0 FileCloud.js => src/FileCloud.js | 0 FileCursor.js => src/FileCursor.js | 0 FileSequence.js => src/FileSequence.js | 0 MarketFiles.js => src/MarketFiles.js | 0 SingleFile.js => src/SingleFile.js | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename DailyFiles.js => src/DailyFiles.js (100%) rename FileCloud.js => src/FileCloud.js (100%) rename FileCursor.js => src/FileCursor.js (100%) rename FileSequence.js => src/FileSequence.js (100%) rename MarketFiles.js => src/MarketFiles.js (100%) rename SingleFile.js => src/SingleFile.js (100%) diff --git a/DailyFiles.js b/src/DailyFiles.js similarity index 100% rename from DailyFiles.js rename to src/DailyFiles.js diff --git a/FileCloud.js b/src/FileCloud.js similarity index 100% rename from FileCloud.js rename to src/FileCloud.js diff --git a/FileCursor.js b/src/FileCursor.js similarity index 100% rename from FileCursor.js rename to src/FileCursor.js diff --git a/FileSequence.js b/src/FileSequence.js similarity index 100% rename from FileSequence.js rename to src/FileSequence.js diff --git a/MarketFiles.js b/src/MarketFiles.js similarity index 100% rename from MarketFiles.js rename to src/MarketFiles.js diff --git a/SingleFile.js b/src/SingleFile.js similarity index 100% rename from SingleFile.js rename to src/SingleFile.js From b250ff9d2497a97d3355733115255838f755c7d4 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 24 Feb 2019 12:40:09 +0100 Subject: [PATCH 05/92] Auto Formatter Applied --- src/FileCloud.js | 418 ++++++++++++++++++++------------------------- src/MarketFiles.js | 389 +++++++++++++++++++---------------------- 2 files changed, 360 insertions(+), 447 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index 4173eb528..bcd0d7605 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -1,252 +1,208 @@ - -function newFileCloud() { - const MODULE_NAME = "File Cloud"; - const INFO_LOG = false; - const ERROR_LOG = true; - const logger = newWebDebugLog(); - logger.fileName = MODULE_NAME; +function newFileCloud () { + const MODULE_NAME = 'File Cloud' + const INFO_LOG = false + const ERROR_LOG = true + const logger = newWebDebugLog() + logger.fileName = MODULE_NAME /* - This is the module in the system that actually connects to the cloud storage and grabs from there the needed files. + This is the module in the system that actually connects to the cloud storage and grabs from there the needed files. */ - var thisObject = { - getFile: getFile, - initialize: initialize - }; + var thisObject = { + getFile: getFile, + initialize: initialize + } - let blobService; + let blobService - return thisObject; + return thisObject - function initialize(pBot) { + function initialize (pBot) { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> pBot = ' + pBot.codeName) } - try { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> pBot = " + pBot.codeName); } - - blobService = AzureStorage.Blob.createBlobServiceWithSas(pBot.storage.fileUri, pBot.storage.sas); - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } - //callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); TODO> Handle this callback. - } + blobService = AzureStorage.Blob.createBlobServiceWithSas(pBot.storage.fileUri, pBot.storage.sas) + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + // callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); TODO> Handle this callback. } + } - function getFile(pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + try { + if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } + const MAX_RETRIES = 3 + + getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + + function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { - - if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } - - const MAX_RETRIES = 3; - - getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - - function getFileRecursively(pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pSet.filePath + "-" + pSet.fileName); } - - let fileName; - let filePath; - - if (pDataRange === undefined) { - - fileName = pSet.fileName; - filePath = pSet.filePath; - - } else { - - if (pSet.dataRange !== undefined) { - - fileName = pSet.dataRange.fileName; - filePath = pSet.dataRange.filePath; - - } else { - - let customErr = { - result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, - message: "Missing Configuration." - }; - - if (ERROR_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = " + customErr.message); } - if (ERROR_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found. "); } - - callBackFunction(customErr); - return; - } - } - - if (fileName === undefined) { - - logger.write("[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: "); - logger.write("[ERROR] getFile -> getFileRecursively -> pDevTeam = " + JSON.stringify(pDevTeam)); - logger.write("[ERROR] getFile -> getFileRecursively -> pBot = " + JSON.stringify(pBot)); - logger.write("[ERROR] getFile -> getFileRecursively -> pSet = " + JSON.stringify(pSet)); - logger.write("[ERROR] getFile -> getFileRecursively -> pExchange = " + JSON.stringify(pExchange)); - logger.write("[ERROR] getFile -> getFileRecursively -> pMarket = " + JSON.stringify(pMarket)); - logger.write("[ERROR] getFile -> getFileRecursively -> pPeriodName = " + JSON.stringify(pPeriodName)); - - throw ("Inconsistant data received."); - } - - if (pMarket !== undefined) { - - fileName = fileName.replace("@AssetA", pMarket.assetA); - fileName = fileName.replace("@AssetB", pMarket.assetB); - } - - if (pDevTeam !== undefined) { - - filePath = filePath.replace("@DevTeam", pDevTeam.codeName); - } - - if (pBot !== undefined) { - - filePath = filePath.replace("@Bot", pBot.codeName); - } - - if (pExchange !== undefined) { - - filePath = filePath.replace("@Exchange", pExchange.name); - } - - filePath = filePath.replace("@Period", pPeriodName); - - if (pDatetime !== undefined) { - - filePath = filePath.replace("@Year", pDatetime.getUTCFullYear()); - filePath = filePath.replace("@Month", pad(pDatetime.getUTCMonth() + 1, 2)); - filePath = filePath.replace("@Day", pad(pDatetime.getUTCDate(), 2)); - filePath = filePath.replace("@Hour", pad(pDatetime.getUTCHours(), 2)); - filePath = filePath.replace("@Minute", pad(pDatetime.getUTCMinutes(), 2)); - - } - - if (pSequence !== undefined) { - - fileName = fileName.replace("@Sequence", pSequence); - } - - if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> filePath = " + filePath); } - if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> fileName = " + fileName); } - - let containerName; - - containerName = pDevTeam.codeName.toLowerCase(); - - blobService.getBlobToText(containerName, filePath + "/" + fileName, onFileReceived); - - function onFileReceived(err, text, response) { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = " + filePath); } - if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> fileName = " + fileName); } - - let data; - - if (err) { - - if (err.code === "BlobNotFound" | err.code === "FileNotFound" | err.code === "ParentNotFound") { - - let customErr = { - result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, - message: "File does not exist." - }; - - if (INFO_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = " + customErr.message); } - - callBackFunction(customErr); - return; - } - - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> containerName = " + containerName); } - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> filePath = " + filePath); } - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> fileName = " + fileName); } - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> Unexpected Error Ocurred."); } - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = " + err); } - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> text = " + text); } - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> response = " + response); } - - if (err.message === "XHR error") { - - if (pRetryCounter < MAX_RETRIES) { - - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> Retrying to get this file. "); } - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = " + MAX_RETRIES); } - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = " + pRetryCounter); } - - getFileRecursively(pRetryCounter + 1, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - return; - - } else { - - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. "); } - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = " + MAX_RETRIES); } - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = " + pRetryCounter); } - - } - } - - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - return; - - } else { - - try { - - data = JSON.parse(text); - - if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> Data received is valid JSON."); } - - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data); - return; - - } catch (err) { - - if (INFO_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> err = " + err); } - - let customErr = { - result: GLOBAL.CUSTOM_OK_RESPONSE.result, - message: "Data not in JSON Format." - }; - - if (INFO_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = " + customErr.message); } - - callBackFunction(customErr, text); - return; - } - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } + + let fileName + let filePath + + if (pDataRange === undefined) { + fileName = pSet.fileName + filePath = pSet.filePath + } else { + if (pSet.dataRange !== undefined) { + fileName = pSet.dataRange.fileName + filePath = pSet.dataRange.filePath + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Missing Configuration.' + } + + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found. ') } + + callBackFunction(customErr) + return } + } + if (fileName === undefined) { + logger.write('[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: ') + logger.write('[ERROR] getFile -> getFileRecursively -> pDevTeam = ' + JSON.stringify(pDevTeam)) + logger.write('[ERROR] getFile -> getFileRecursively -> pBot = ' + JSON.stringify(pBot)) + logger.write('[ERROR] getFile -> getFileRecursively -> pSet = ' + JSON.stringify(pSet)) + logger.write('[ERROR] getFile -> getFileRecursively -> pExchange = ' + JSON.stringify(pExchange)) + logger.write('[ERROR] getFile -> getFileRecursively -> pMarket = ' + JSON.stringify(pMarket)) + logger.write('[ERROR] getFile -> getFileRecursively -> pPeriodName = ' + JSON.stringify(pPeriodName)) + throw ('Inconsistant data received.') + } + + if (pMarket !== undefined) { + fileName = fileName.replace('@AssetA', pMarket.assetA) + fileName = fileName.replace('@AssetB', pMarket.assetB) + } + + if (pDevTeam !== undefined) { + filePath = filePath.replace('@DevTeam', pDevTeam.codeName) + } + + if (pBot !== undefined) { + filePath = filePath.replace('@Bot', pBot.codeName) + } + + if (pExchange !== undefined) { + filePath = filePath.replace('@Exchange', pExchange.name) + } + + filePath = filePath.replace('@Period', pPeriodName) + + if (pDatetime !== undefined) { + filePath = filePath.replace('@Year', pDatetime.getUTCFullYear()) + filePath = filePath.replace('@Month', pad(pDatetime.getUTCMonth() + 1, 2)) + filePath = filePath.replace('@Day', pad(pDatetime.getUTCDate(), 2)) + filePath = filePath.replace('@Hour', pad(pDatetime.getUTCHours(), 2)) + filePath = filePath.replace('@Minute', pad(pDatetime.getUTCMinutes(), 2)) + } + + if (pSequence !== undefined) { + fileName = fileName.replace('@Sequence', pSequence) + } + + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> filePath = ' + filePath) } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> fileName = ' + fileName) } + + let containerName + + containerName = pDevTeam.codeName.toLowerCase() + + blobService.getBlobToText(containerName, filePath + '/' + fileName, onFileReceived) + + function onFileReceived (err, text, response) { + try { + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } + + let data + + if (err) { + if (err.code === 'BlobNotFound' | err.code === 'FileNotFound' | err.code === 'ParentNotFound') { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'File does not exist.' + filePath + '/' + fileName + } + + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + + callBackFunction(customErr) + return + } + + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> containerName = ' + containerName) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Unexpected Error Ocurred.') } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> text = ' + text) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> response = ' + response) } + + if (err.message === 'XHR error') { + if (pRetryCounter < MAX_RETRIES) { + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Retrying to get this file. ') } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = ' + MAX_RETRIES) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = ' + pRetryCounter) } + + getFileRecursively(pRetryCounter + 1, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + return + } else { + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. ') } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = ' + MAX_RETRIES) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = ' + pRetryCounter) } + } + } + + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } else { + try { + data = JSON.parse(text) + + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Data received is valid JSON.') } + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data) + return + } catch (err) { + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> err = ' + err) } + + let customErr = { + result: GLOBAL.CUSTOM_OK_RESPONSE.result, + message: 'Data not in JSON Format.' + filePath + '/' + fileName + } + + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + + callBackFunction(customErr, text) + return + } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + } + } } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } -} \ No newline at end of file + } +} diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 0bbf6d116..8ffc71a75 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -1,253 +1,210 @@ - -function newMarketFiles() { - const MODULE_NAME = "Market Files"; - const INFO_LOG = false; - const ERROR_LOG = true; - const logger = newWebDebugLog(); - logger.fileName = MODULE_NAME; +function newMarketFiles () { + const MODULE_NAME = 'Market Files' + const INFO_LOG = false + const ERROR_LOG = true + const logger = newWebDebugLog() + logger.fileName = MODULE_NAME - let thisObject = { - eventHandler: undefined, - getFile: getFile, - getExpectedFiles: getExpectedFiles, - getFilesLoaded: getFilesLoaded, - initialize: initialize, - finalize: finalize + let thisObject = { + eventHandler: undefined, + getFile: getFile, + getExpectedFiles: getExpectedFiles, + getFilesLoaded: getFilesLoaded, + initialize: initialize, + finalize: finalize + } + + let filesLoaded = 0 + + let fileCloud + + let files = new Map() + + let market + let exchange + let devTeam + let bot + let thisSet + let periodName + + thisObject.eventHandler = newEventHandler() + + let intervalHandle + + return thisObject + + function finalize () { + try { + if (INFO_LOG === true) { logger.write('[INFO] finalize -> Entering function.') } + + clearInterval(intervalHandle) + + filesLoaded = undefined + files = undefined + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err) } } + } - let filesLoaded = 0; + function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } - let fileCloud; + exchange = ecosystem.getExchange(pProduct, pExchange) - let files = new Map; + if (exchange === undefined) { + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + } - let market; - let exchange; - let devTeam; - let bot; - let thisSet; - let periodName; + intervalHandle = setInterval(updateFiles, _10_MINUTES_IN_MILISECONDS) - thisObject.eventHandler = newEventHandler(); + market = pMarket + devTeam = pDevTeam + bot = pBot + thisSet = pSet - let intervalHandle; - - return thisObject; - - function finalize() { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } - - clearInterval(intervalHandle); - - filesLoaded = undefined; - files = undefined; - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } - } - } - - function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } - - exchange = ecosystem.getExchange(pProduct, pExchange); - - if (exchange === undefined) { - - throw "Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; - - } - - intervalHandle = setInterval(updateFiles, _10_MINUTES_IN_MILISECONDS); - - market = pMarket; - devTeam = pDevTeam; - bot = pBot; - thisSet = pSet; - - fileCloud = newFileCloud(); - fileCloud.initialize(pBot); + fileCloud = newFileCloud() + fileCloud.initialize(pBot) /* Now we will get the market files */ - for (let i = 0; i < marketFilesPeriods.length; i++) { + for (let i = 0; i < marketFilesPeriods.length; i++) { + let periodTime = marketFilesPeriods[i][0] + let periodName = marketFilesPeriods[i][1] - let periodTime = marketFilesPeriods[i][0]; - let periodName = marketFilesPeriods[i][1]; + if (thisSet.validPeriods.includes(periodName) === true) { + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) - if (thisSet.validPeriods.includes(periodName) === true) { + function onFileReceived (err, file) { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Entering function.') } - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived); - - function onFileReceived(err, file) { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Entering function."); } - - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received OK Response."); } - break; - } - - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received FAIL Response."); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - return; - } - - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> err.message = " + err.message); } - - callBackFunction(err); - return; - } - - default: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received Unexpected Response."); } - callBackFunction(err); - return; - } - } - - files.set(periodTime, file); - - filesLoaded++; - - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject); // Note that the callback is called for every file loaded. - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } - } + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received OK Response.') } + break } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received FAIL Response.') } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> err.message = ' + err.message) } + + callBackFunction(err) + return + } + + default: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received Unexpected Response.') } + callBackFunction(err) + return + } + } + + files.set(periodTime, file) + + filesLoaded++ + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) // Note that the callback is called for every file loaded. + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } - - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } - function updateFiles() { + function updateFiles () { + try { + let updateFiles = 0 - try { - - let updateFiles = 0; - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Entering function."); } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Entering function.') } /* Now we will get the market files */ - for (let i = 0; i < marketFilesPeriods.length; i++) { + for (let i = 0; i < marketFilesPeriods.length; i++) { + let periodTime = marketFilesPeriods[i][0] + let periodName = marketFilesPeriods[i][1] - let periodTime = marketFilesPeriods[i][0]; - let periodName = marketFilesPeriods[i][1]; + if (thisSet.validPeriods.includes(periodName) === true) { + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) - if (thisSet.validPeriods.includes(periodName) === true) { + function onFileReceived (err, file) { + try { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Entering function.') } - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived); - - function onFileReceived(err, file) { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Entering function."); } - - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received OK Response."); } - break; - } - - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received FAIL Response."); } - return; - } - - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received CUSTOM FAIL Response."); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> err.message = " + err.message); } - return; - } - - default: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received Unexpected Response."); } - return; - } - } - - files.set(periodTime, file); - updateFiles++; - - if (updateFiles === marketFilesPeriods.length) { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> All files received. "); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> devTeam = " + devTeam.codeName); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> bot = " + bot.codeName); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> thisSet = " + thisSet.codeName); } - - thisObject.eventHandler.raiseEvent("Files Updated", undefined); - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onFileReceived -> err = " + err); } - } - } + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received OK Response.') } + break } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received FAIL Response.') } + return + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received CUSTOM FAIL Response.') } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> err.message = ' + err.message) } + return + } + + default: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received Unexpected Response.') } + return + } + } + + files.set(periodTime, file) + updateFiles++ + + if (updateFiles === marketFilesPeriods.length) { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> All files received. ') } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> devTeam = ' + devTeam.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> bot = ' + bot.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> thisSet = ' + thisSet.codeName) } + + thisObject.eventHandler.raiseEvent('Files Updated', undefined) + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onFileReceived -> err = ' + err) } } - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> err = " + err); } + } } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> err = ' + err) } } + } - function getFile(pPeriod) { + function getFile (pPeriod) { + if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } - if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } + return files.get(pPeriod) + } - return files.get(pPeriod); + function getExpectedFiles () { + if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } - } + return marketFilesPeriods.length + } - function getExpectedFiles() { + function getFilesLoaded () { + if (INFO_LOG === true) { logger.write('[INFO] getFilesLoaded -> Entering function.') } - if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } - - return marketFilesPeriods.length; - - } - - function getFilesLoaded() { - - if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> Entering function."); } - - return filesLoaded; - - } - -} \ No newline at end of file + return filesLoaded + } +} From 33522684b7f111b7e9e079757b8ba424484abfd8 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 1 Mar 2019 05:28:12 +0100 Subject: [PATCH 06/92] Code Formatting --- src/DailyFiles.js | 577 +++++++++++------------ src/FileCloud.js | 26 +- src/FileCursor.js | 1060 +++++++++++++++++++------------------------ src/FileSequence.js | 587 +++++++++++------------- src/MarketFiles.js | 2 +- src/SingleFile.js | 154 +++---- 6 files changed, 1094 insertions(+), 1312 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index c7cbf8314..cf94b1fe2 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -1,392 +1,327 @@ - -function newDailyFiles() { +  +function newDailyFiles () { + const MODULE_NAME = 'Daily Files' + const INFO_LOG = false + const ERROR_LOG = true + const logger = newWebDebugLog() + logger.fileName = MODULE_NAME - const MODULE_NAME = "Daily Files"; - const INFO_LOG = false; - const ERROR_LOG = true; - const logger = newWebDebugLog(); - logger.fileName = MODULE_NAME; + let thisObject = { + getFileCursor: getFileCursor, + setDatetime: setDatetime, + setTimePeriod: setTimePeriod, + getExpectedFiles: getExpectedFiles, + getFilesLoaded: getFilesLoaded, + initialize: initialize, + finalize: finalize + } - let thisObject = { - getFileCursor: getFileCursor, - setDatetime: setDatetime, - setTimePeriod: setTimePeriod, - getExpectedFiles: getExpectedFiles, - getFilesLoaded: getFilesLoaded, - initialize: initialize, - finalize: finalize + let filesLoaded = 0 + let expectedFiles = 0 + + let fileCloud + + let fileCursors = new Map() + + let callBackWhenFileReceived + + return thisObject + + function finalize () { + try { + if (INFO_LOG === true) { logger.write('[INFO] finalize -> Entering function.') } + + filesLoaded = undefined + expectedFiles = undefined + fileCloud = undefined + + fileCursors.forEach(finalizeEach) + + function finalizeEach (item, key, mapObj) { + if (INFO_LOG === true) { logger.write('[INFO] finalize -> finalizeEach -> Entering function.') } + + item.finalize() + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err) } } + } - let filesLoaded = 0; - let expectedFiles = 0; + function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } - let fileCloud; + callBackWhenFileReceived = callBackFunction - let fileCursors = new Map; + let exchange = ecosystem.getExchange(pProduct, pExchange) - let callBackWhenFileReceived; + if (exchange === undefined) { + throw 'Exchange not supoorted by this product of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + } - return thisObject; - - function finalize() { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } - - filesLoaded = undefined; - expectedFiles = undefined; - fileCloud = undefined; - - fileCursors.forEach(finalizeEach); - - function finalizeEach(item, key, mapObj) { - - if (INFO_LOG === true) { logger.write("[INFO] finalize -> finalizeEach -> Entering function."); } - - item.finalize(); - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } - } - } - - function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } - - callBackWhenFileReceived = callBackFunction; - - let exchange = ecosystem.getExchange(pProduct, pExchange); - - if (exchange === undefined) { - - throw "Exchange not supoorted by this product of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; - - } - - fileCloud = newFileCloud(); - fileCloud.initialize(pBot); + fileCloud = newFileCloud() + fileCloud.initialize(pBot) /* First we will get the Data Range */ - fileCloud.getFile(pDevTeam, pBot, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived); + fileCloud.getFile(pDevTeam, pBot, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) - function onDataRangeReceived(err, pFile) { + function onDataRangeReceived (err, pFile) { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Entering function.') } - try { + let beginDateRange + let endDateRange - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Entering function."); } + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received OK Response.') } - let beginDateRange; - let endDateRange; + beginDateRange = new Date(pFile.begin) + endDateRange = new Date(pFile.end) - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { + break + } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received OK Response."); } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received FAIL Response.') } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } - beginDateRange = new Date(pFile.begin); - endDateRange = new Date(pFile.end); + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> err.message = ' + err.message) } - break; - } + if (err.message === 'File does not exist.') { + if (INFO_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> No Date Range file found. maxDate will be set to current datetime. ') } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + endDateRange = new Date() + break + } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received FAIL Response."); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - return; - } + if (err.message === 'Missing Configuration.') { + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration for the dateRange at the dataSet of the product was not found.') } + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> maxDate will be set to current datetime.') } - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> err.message = " + err.message); } - - if (err.message === "File does not exist.") { - - if (INFO_LOG === true) { logger.write("[WARN] initialize -> onFileReceived -> No Date Range file found. maxDate will be set to current datetime. "); } - - endDateRange = new Date(); - break; - } - - if (err.message === "Missing Configuration.") { - - if (ERROR_LOG === true) { logger.write("[WARN] initialize -> onFileReceived -> The needed configuration for the dateRange at the dataSet of the product was not found."); } - if (ERROR_LOG === true) { logger.write("[WARN] initialize -> onFileReceived -> maxDate will be set to current datetime."); } - - endDateRange = new Date(); - break; - } + endDateRange = new Date() + break + } /* If none of the previous conditions are met, the we return the err to the caller. */ - callBackFunction(err); - return; - } + callBackFunction(err) + return + } - default: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received Unexpected Response."); } - callBackFunction(err); - return; - } - } + default: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received Unexpected Response.') } + callBackFunction(err) + return + } + } /* Now we will get the daily files */ - for (i = 0; i < dailyFilePeriods.length; i++) { + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] - let periodTime = dailyFilePeriods[i][0]; - let periodName = dailyFilePeriods[i][1]; + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> periodTime = ' + periodTime) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> periodName = ' + periodName) } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> periodTime = " + periodTime); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> periodName = " + periodName); } + if (pSet.validPeriods.includes(periodName) === true) { + let fileCursor = newFileCursor() + fileCursor.initialize(fileCloud, pDevTeam, pBot, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) - if (pSet.validPeriods.includes(periodName) === true) { + function onInitialized (err) { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Entering function.') } - let fileCursor = newFileCursor(); - fileCursor.initialize(fileCloud, pDevTeam, pBot, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized); + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { - function onInitialized(err) { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received OK Response.') } + break + } - try { + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Entering function."); } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received FAIL Response.') } + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + default: { - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received Unexpected Response.') } + callBackWhenFileReceived(err) + return + } + } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Received OK Response."); } - break; - } + fileCursors.set(periodTime, fileCursor) - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> expectedFiles = ' + expectedFiles) } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Received FAIL Response."); } - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE); - return; - } - default: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Received Unexpected Response."); } - callBackWhenFileReceived(err); - return; - } - } - - fileCursors.set(periodTime, fileCursor); - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> expectedFiles = " + expectedFiles); } - - expectedFiles = expectedFiles + fileCursor.getExpectedFiles(); - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> expectedFiles = " + expectedFiles); } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> onInitialized -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } - } - } - } - - loadThemAll(); - - function loadThemAll() { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> loadThemAll -> Entering function."); } - - for (i = 0; i < dailyFilePeriods.length; i++) { - - let periodTime = dailyFilePeriods[i][0]; - let periodName = dailyFilePeriods[i][1]; - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> loadThemAll -> periodTime = " + periodTime); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> loadThemAll -> periodName = " + periodName); } - - if (pSet.validPeriods.includes(periodName) === true) { - - let fileCursor = fileCursors.get(periodTime); - fileCursor.reload(onFileReceived); - - } - } - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> loadThemAll -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } - } + expectedFiles = expectedFiles + fileCursor.getExpectedFiles() + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> expectedFiles = ' + expectedFiles) } } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } } + } - } catch (err) { + loadThemAll() - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } - } + function loadThemAll () { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> loadThemAll -> Entering function.') } - function onFileReceived(err) { + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] - try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> loadThemAll -> periodTime = ' + periodTime) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> loadThemAll -> periodName = ' + periodName) } - if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Entering function."); } - - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received OK Response."); } - break; - } - - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received FAIL Response."); } - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE); - return; - } - - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received CUSTOM FAIL Response."); } - if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> err.message = " + err.message); } - - callBackWhenFileReceived(err); - return; - } - - default: { - - if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received Unexpected Response."); } - callBackWhenFileReceived(err); - return; + if (pSet.validPeriods.includes(periodName) === true) { + let fileCursor = fileCursors.get(periodTime) + fileCursor.reload(onFileReceived) } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> loadThemAll -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } - - filesLoaded++; - - if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> filesLoaded = " + filesLoaded); } - - callBackWhenFileReceived(GLOBAL.DEFAULT_OK_RESPONSE, thisObject); // Note that the call back is called for every file loaded at each cursor. - + } } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] onFileReceived -> err = " + err); } - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE); + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } - function getFileCursor(pPeriod) { + function onFileReceived (err) { + try { + if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> Entering function.') } - try { - - if (INFO_LOG === true) { logger.write("[INFO] getFileCursor -> Entering function."); } - - return fileCursors.get(pPeriod); - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] getFileCursor -> err = " + err); } - + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> Received OK Response.') } + break } - } - function setDatetime(pDatetime) { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> Entering function."); } - - filesLoaded = 0; - expectedFiles = 0; - - fileCursors.forEach(setDatetimeToEach) - - function setDatetimeToEach(fileCursor, key, map) { - - fileCursor.setDatetime(pDatetime); - - if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> expectedFiles = " + expectedFiles); } - - expectedFiles = expectedFiles + fileCursor.getExpectedFiles(); - - if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> expectedFiles = " + expectedFiles); } - - fileCursor.reload(onFileReceived); - - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] setDatetime -> err = " + err); } - + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> Received FAIL Response.') } + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) + return } - } - function setTimePeriod(pTimePeriod, pDatetime) { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Entering function."); } - - filesLoaded = 0; - expectedFiles = 0; - - fileCursors.forEach(setTimePeriodToEach) - - function setTimePeriodToEach(fileCursor, key, map) { - - fileCursor.setTimePeriod(pTimePeriod, pDatetime); - - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> expectedFiles = " + expectedFiles); } - - expectedFiles = expectedFiles + fileCursor.getExpectedFiles(); - - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> expectedFiles = " + expectedFiles); } - - fileCursor.reload(onFileReceived); - - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> err = " + err); } + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> Received CUSTOM FAIL Response.') } + if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> err.message = ' + err.message) } + callBackWhenFileReceived(err) + return } + + default: { + if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> Received Unexpected Response.') } + callBackWhenFileReceived(err) + return + } + } + + filesLoaded++ + + if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> filesLoaded = ' + filesLoaded) } + + callBackWhenFileReceived(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) // Note that the call back is called for every file loaded at each cursor. + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] onFileReceived -> err = ' + err) } + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } - function getExpectedFiles() { - - if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> expectedFiles = " + expectedFiles); } - - return expectedFiles; + function getFileCursor (pPeriod) { + try { + if (INFO_LOG === true) { logger.write('[INFO] getFileCursor -> Entering function.') } + return fileCursors.get(pPeriod) + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] getFileCursor -> err = ' + err) } } + } - function getFilesLoaded() { + function setDatetime (pDatetime) { + try { + if (INFO_LOG === true) { logger.write('[INFO] setDatetime -> Entering function.') } - if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> filesLoaded = " + filesLoaded); } + filesLoaded = 0 + expectedFiles = 0 - return filesLoaded; + fileCursors.forEach(setDatetimeToEach) + function setDatetimeToEach (fileCursor, key, map) { + fileCursor.setDatetime(pDatetime) + + if (INFO_LOG === true) { logger.write('[INFO] setDatetime -> expectedFiles = ' + expectedFiles) } + + expectedFiles = expectedFiles + fileCursor.getExpectedFiles() + + if (INFO_LOG === true) { logger.write('[INFO] setDatetime -> expectedFiles = ' + expectedFiles) } + + fileCursor.reload(onFileReceived) + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] setDatetime -> err = ' + err) } } -} \ No newline at end of file + } + + function setTimePeriod (pTimePeriod, pDatetime) { + try { + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> Entering function.') } + + filesLoaded = 0 + expectedFiles = 0 + + fileCursors.forEach(setTimePeriodToEach) + + function setTimePeriodToEach (fileCursor, key, map) { + fileCursor.setTimePeriod(pTimePeriod, pDatetime) + + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> expectedFiles = ' + expectedFiles) } + + expectedFiles = expectedFiles + fileCursor.getExpectedFiles() + + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> expectedFiles = ' + expectedFiles) } + + fileCursor.reload(onFileReceived) + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> err = ' + err) } + } + } + + function getExpectedFiles () { + if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> expectedFiles = ' + expectedFiles) } + + return expectedFiles + } + + function getFilesLoaded () { + if (INFO_LOG === true) { logger.write('[INFO] getFilesLoaded -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] getFilesLoaded -> filesLoaded = ' + filesLoaded) } + + return filesLoaded + } +} diff --git a/src/FileCloud.js b/src/FileCloud.js index bcd0d7605..49e5002c7 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -33,15 +33,19 @@ function newFileCloud () { } } - function getFile (pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction, pOperationsId) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } const MAX_RETRIES = 3 - getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + if (pOperationsId !== undefined) { + getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction, pOperationsId) + } else { + getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + } - function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction, pOperationsId) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } @@ -92,7 +96,11 @@ function newFileCloud () { } if (pBot !== undefined) { - filePath = filePath.replace('@Bot', pBot.codeName) + if (pOperationsId !== undefined) { + filePath = filePath.replace('@Bot', pBot.codeName + '-' + pOperationsId) + } else { + filePath = filePath.replace('@Bot', pBot.codeName) + } } if (pExchange !== undefined) { @@ -134,10 +142,10 @@ function newFileCloud () { if (err.code === 'BlobNotFound' | err.code === 'FileNotFound' | err.code === 'ParentNotFound') { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, - message: 'File does not exist.' + filePath + '/' + fileName + message: 'File does not exist.' } - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (INFO_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } callBackFunction(customErr) return @@ -177,14 +185,14 @@ function newFileCloud () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data) return } catch (err) { - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> err = ' + err) } + if (INFO_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> err = ' + err) } let customErr = { result: GLOBAL.CUSTOM_OK_RESPONSE.result, - message: 'Data not in JSON Format.' + filePath + '/' + fileName + message: 'Data not in JSON Format.' } - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (INFO_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } callBackFunction(customErr, text) return diff --git a/src/FileCursor.js b/src/FileCursor.js index 9592c2c1f..755cacc03 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -1,688 +1,586 @@ - -function newFileCursor() { +  +function newFileCursor () { + const MODULE_NAME = 'File Cursor' + const INFO_LOG = false + const ERROR_LOG = true + const logger = newWebDebugLog() + logger.fileName = MODULE_NAME - const MODULE_NAME = "File Cursor"; - const INFO_LOG = false; - const ERROR_LOG = true; - const logger = newWebDebugLog(); - logger.fileName = MODULE_NAME; + let cursorDate - let cursorDate; + let thisObject = { + reload: reload, + setDatetime: setDatetime, + setTimePeriod: setTimePeriod, + files: new Map(), + getExpectedFiles: getExpectedFiles, + initialize: initialize, + finalize: finalize + } - let thisObject = { - reload: reload, - setDatetime: setDatetime, - setTimePeriod: setTimePeriod, - files: new Map, - getExpectedFiles: getExpectedFiles, - initialize: initialize, - finalize: finalize - }; + let minCursorSize = 10 + let maxCursorSize = 30 - let minCursorSize = 10; - let maxCursorSize = 30; + let market + let exchange + let fileCloud + let devTeam + let bot + let thisSet + let periodName + let timePeriod + let beginDateRange + let endDateRange - let market; - let exchange; - let fileCloud; - let devTeam; - let bot; - let thisSet; - let periodName; - let timePeriod; - let beginDateRange; - let endDateRange; + let intervalHandle - let intervalHandle; + return thisObject - return thisObject; + function finalize () { + try { + if (INFO_LOG === true) { logger.write('[INFO] finalize -> Entering function.') } - function finalize() { + clearInterval(intervalHandle) - try { - - if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } - - clearInterval(intervalHandle); - - thisObject.files = undefined; - cursorDate = undefined; - fileCloud = undefined; - periodName = undefined; - timePeriod = undefined; - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } - } + thisObject.files = undefined + cursorDate = undefined + fileCloud = undefined + periodName = undefined + timePeriod = undefined + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err) } } + } - function initialize(pFileCloud, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize (pFileCloud, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.codeName + '-' + pPeriodName) } - try { + market = pMarket + exchange = pExchange + fileCloud = pFileCloud + devTeam = pDevTeam + bot = pBot + thisSet = pSet + periodName = pPeriodName + cursorDate = pCursorDate + timePeriod = pTimePeriod + beginDateRange = pBeginDateRange + endDateRange = pEndDateRange - if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pSet.codeName + "-" + pPeriodName); } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> market = ' + market) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> periodName = ' + periodName) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> cursorDate = ' + cursorDate) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> beginDateRange = ' + beginDateRange) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> endDateRange = ' + endDateRange) } - market = pMarket; - exchange = pExchange; - fileCloud = pFileCloud; - devTeam = pDevTeam; - bot = pBot; - thisSet = pSet; - periodName = pPeriodName; - cursorDate = pCursorDate; - timePeriod = pTimePeriod; - beginDateRange = pBeginDateRange; - endDateRange = pEndDateRange; + setTimePeriod(pCurrentTimePeriod, pCursorDate) - if (INFO_LOG === true) { logger.write("[INFO] initialize -> market = " + market); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> periodName = " + periodName); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> cursorDate = " + cursorDate); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> beginDateRange = " + beginDateRange); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> endDateRange = " + endDateRange); } + intervalHandle = setInterval(updateFiles, timePeriod) - setTimePeriod(pCurrentTimePeriod, pCursorDate); - - intervalHandle = setInterval(updateFiles , timePeriod); - - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE); - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } - function updateFiles() { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Entering function."); } + function updateFiles () { + try { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Entering function.') } /* In order to know if we need to update any file we follow this logic: 1. If the current date is already a file in the cursor, we need to update it. - 2. If the current date is one day ahead of a file in the cursor, we need to add this one to the cursor. + 2. If the current date is one day ahead of a file in the cursor, we need to add this one to the cursor. */ - let targetDate; - let dateString; - let file; + let targetDate + let dateString + let file /* Situation 1 */ - targetDate = new Date(); - dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); + targetDate = new Date() + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - file = thisObject.files.get(dateString); + file = thisObject.files.get(dateString) - if (file !== undefined) { + if (file !== undefined) { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> File found at the cursor, proceeding to update.') } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> File found at the cursor, proceeding to update."); } + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived); - - return; - } + return + } /* Situation 2 */ - targetDate = new Date( (new Date()).valueOf() - ONE_DAY_IN_MILISECONDS); - dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); + targetDate = new Date((new Date()).valueOf() - ONE_DAY_IN_MILISECONDS) + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - file = thisObject.files.get(dateString); // This is from yesterday + file = thisObject.files.get(dateString) // This is from yesterday - targetDate = new Date(); - dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); + targetDate = new Date() + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - if (file !== undefined) { + if (file !== undefined) { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Previous day file found at the cursor, proceeding to add current date.') } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Previous day file found at the cursor, proceeding to add current date."); } + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived); - - return; - } - - function onFileReceived(err, file) { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Entering function."); } - - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received OK Response."); } - break; - } - - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received FAIL Response."); } - return; - } - - default: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received Unexpected Response."); } - return; - } - } - - thisObject.files.set(dateString, file); - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> File updated."); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> devTeam = " + devTeam.codeName); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> bot = " + bot.codeName); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> thisSet = " + thisSet.codeName); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> dateString = " + dateString); } - - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onFileReceived -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } - } - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> err = " + err); } - } - } - - function setTimePeriod(pTimePeriod, pDatetime) { + return + } + function onFileReceived (err, file) { try { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Entering function.') } - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Entering function."); } + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received OK Response.') } + break + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received FAIL Response.') } + return + } + + default: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received Unexpected Response.') } + return + } + } + + thisObject.files.set(dateString, file) + + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> File updated.') } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> devTeam = ' + devTeam.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> bot = ' + bot.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> thisSet = ' + thisSet.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> dateString = ' + dateString) } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onFileReceived -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> err = ' + err) } + } + } + + function setTimePeriod (pTimePeriod, pDatetime) { + try { + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> Entering function.') } /* - + We are implementing here an algorithm designed to save bandwidth, memory and processing power at the browser. We say there is a saving mode where the cursor is running at a minimum size. When the end user aproaches the time period the cursor is set, then it should exit the saving mode and go to its actual size. - + To do this we are going to measure the distance from the Time Period received to the one the cursors was initialized with. If these periods are consecutive, it means that the cursor should exit saving mode and load its full size. - + */ - cursorDate = pDatetime; // Adjust the cursor date to the one received. + cursorDate = pDatetime // Adjust the cursor date to the one received. - let positionA; + let positionA - for (let i = 0; i < dailyFilePeriods.length; i++) { + for (let i = 0; i < dailyFilePeriods.length; i++) { + let period = dailyFilePeriods[i] - let period = dailyFilePeriods[i]; - - if (period[0] === pTimePeriod) { - - positionA = i; - - } - - if (period[0] === timePeriod) { - - positionB = i; - - } - } - - if (Math.abs(positionB - positionA) <= 1) { - - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Will EXIT Saving Mode."); } - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> pTimePeriod = " + convertTimePeriodToName(pTimePeriod)); } - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> timePeriod = " + convertTimePeriodToName(timePeriod)); } - - exitSavingMode(); - - } else { - - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Will ENTER Saving Mode."); } - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> pTimePeriod = " + convertTimePeriodToName(pTimePeriod)); } - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> timePeriod = " + convertTimePeriodToName(timePeriod)); } - - enterSavingMode(); - - } - - function enterSavingMode() { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> enterSavingMode -> Entering function."); } - - switch (timePeriod) { - - case _45_MINUTES_IN_MILISECONDS: - { - minCursorSize = 3; - maxCursorSize = 30; - } - break; - case _40_MINUTES_IN_MILISECONDS: - { - minCursorSize = 3; - maxCursorSize = 20; - } - break; - case _30_MINUTES_IN_MILISECONDS: - { - minCursorSize = 3; - maxCursorSize = 15; - } - break; - case _20_MINUTES_IN_MILISECONDS: - { - minCursorSize = 3; - maxCursorSize = 15; - } - break; - case _15_MINUTES_IN_MILISECONDS: - { - minCursorSize = 3; - maxCursorSize = 15; - } - break; - case _10_MINUTES_IN_MILISECONDS: - { - minCursorSize = 3; - maxCursorSize = 15; - } - break; - case _5_MINUTES_IN_MILISECONDS: - { - minCursorSize = 3; - maxCursorSize = 15; - } - break; - case _4_MINUTES_IN_MILISECONDS: - { - minCursorSize = 3; - maxCursorSize = 15; - } - break; - case _3_MINUTES_IN_MILISECONDS: - { - minCursorSize = 3; - maxCursorSize = 15; - } - break; - case _2_MINUTES_IN_MILISECONDS: - { - minCursorSize = 3; - maxCursorSize = 15; - } - break; - case _1_MINUTE_IN_MILISECONDS: - { - minCursorSize = 3; - maxCursorSize = 15; - } - break; - default: - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> enterSavingMode -> err = " + err); } - } - } - - function exitSavingMode() { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> exitSavingMode -> Entering function."); } - - switch (timePeriod) { - - case _45_MINUTES_IN_MILISECONDS: - { - minCursorSize = 15; - maxCursorSize = 30; - } - break; - case _40_MINUTES_IN_MILISECONDS: - { - minCursorSize = 13; - maxCursorSize = 20; - } - break; - case _30_MINUTES_IN_MILISECONDS: - { - minCursorSize = 11; - maxCursorSize = 15; - } - break; - case _20_MINUTES_IN_MILISECONDS: - { - minCursorSize = 9; - maxCursorSize = 15; - } - break; - case _15_MINUTES_IN_MILISECONDS: - { - minCursorSize = 7; - maxCursorSize = 15; - } - break; - case _10_MINUTES_IN_MILISECONDS: - { - minCursorSize = 7; - maxCursorSize = 15; - } - break; - case _5_MINUTES_IN_MILISECONDS: - { - minCursorSize = 5; - maxCursorSize = 15; - } - break; - case _4_MINUTES_IN_MILISECONDS: - { - minCursorSize = 5; - maxCursorSize = 15; - } - break; - case _3_MINUTES_IN_MILISECONDS: - { - minCursorSize = 5; - maxCursorSize = 15; - } - break; - case _2_MINUTES_IN_MILISECONDS: - { - minCursorSize = 5; - maxCursorSize = 15; - } - break; - case _1_MINUTE_IN_MILISECONDS: - { - minCursorSize = 5; - maxCursorSize = 15; - } - break; - default: - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> exitSavingMode -> err = " + err); } - } - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> err = " + err); } + if (period[0] === pTimePeriod) { + positionA = i } - } - function setDatetime(pDatetime) { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> Entering function."); } - - if (pDatetime === undefined) { - - if (ERROR_LOG === true) { logger.write("[ERROR] setDatetime -> Received undefined datetime."); } - return; - } - - cursorDate = pDatetime; - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] setDatetime -> err = " + err); } + if (period[0] === timePeriod) { + positionB = i } - } + } - function reload(callBackFunction) { + if (Math.abs(positionB - positionA) <= 1) { + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> Will EXIT Saving Mode.') } + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> pTimePeriod = ' + convertTimePeriodToName(pTimePeriod)) } + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> timePeriod = ' + convertTimePeriodToName(timePeriod)) } + exitSavingMode() + } else { + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> Will ENTER Saving Mode.') } + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> pTimePeriod = ' + convertTimePeriodToName(pTimePeriod)) } + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> timePeriod = ' + convertTimePeriodToName(timePeriod)) } + + enterSavingMode() + } + + function enterSavingMode () { try { + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> enterSavingMode -> Entering function.') } - if (INFO_LOG === true) { logger.write("[INFO] reload -> Entering function."); } - - getFiles(callBackFunction); - - collectGarbage(callBackFunction); + switch (timePeriod) { + case _45_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3 + maxCursorSize = 30 + } + break + case _40_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3 + maxCursorSize = 20 + } + break + case _30_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3 + maxCursorSize = 15 + } + break + case _20_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3 + maxCursorSize = 15 + } + break + case _15_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3 + maxCursorSize = 15 + } + break + case _10_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3 + maxCursorSize = 15 + } + break + case _5_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3 + maxCursorSize = 15 + } + break + case _4_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3 + maxCursorSize = 15 + } + break + case _3_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3 + maxCursorSize = 15 + } + break + case _2_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3 + maxCursorSize = 15 + } + break + case _1_MINUTE_IN_MILISECONDS: + { + minCursorSize = 3 + maxCursorSize = 15 + } + break + default: + } } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] reload -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> enterSavingMode -> err = ' + err) } } - } - - function getFiles(callBackFunction) { + } + function exitSavingMode () { try { + if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> exitSavingMode -> Entering function.') } - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> Entering function."); } + switch (timePeriod) { - let i = 0; - let j = 0; + case _45_MINUTES_IN_MILISECONDS: + { + minCursorSize = 15 + maxCursorSize = 30 + } + break + case _40_MINUTES_IN_MILISECONDS: + { + minCursorSize = 13 + maxCursorSize = 20 + } + break + case _30_MINUTES_IN_MILISECONDS: + { + minCursorSize = 11 + maxCursorSize = 15 + } + break + case _20_MINUTES_IN_MILISECONDS: + { + minCursorSize = 9 + maxCursorSize = 15 + } + break + case _15_MINUTES_IN_MILISECONDS: + { + minCursorSize = 7 + maxCursorSize = 15 + } + break + case _10_MINUTES_IN_MILISECONDS: + { + minCursorSize = 7 + maxCursorSize = 15 + } + break + case _5_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5 + maxCursorSize = 15 + } + break + case _4_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5 + maxCursorSize = 15 + } + break + case _3_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5 + maxCursorSize = 15 + } + break + case _2_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5 + maxCursorSize = 15 + } + break + case _1_MINUTE_IN_MILISECONDS: + { + minCursorSize = 5 + maxCursorSize = 15 + } + break + default: + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> exitSavingMode -> err = ' + err) } + } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> err = ' + err) } + } + } - let dateString; + function setDatetime (pDatetime) { + try { + if (INFO_LOG === true) { logger.write('[INFO] setDatetime -> Entering function.') } - getNextFile(); + if (pDatetime === undefined) { + if (ERROR_LOG === true) { logger.write('[ERROR] setDatetime -> Received undefined datetime.') } + return + } - function getNextFile() { + cursorDate = pDatetime + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] setDatetime -> err = ' + err) } + } + } - try { + function reload (callBackFunction) { + try { + if (INFO_LOG === true) { logger.write('[INFO] reload -> Entering function.') } - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> Entering function."); } + getFiles(callBackFunction) - let targetDate = new Date(cursorDate); - targetDate.setUTCDate(targetDate.getUTCDate() + j); + collectGarbage(callBackFunction) + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] reload -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + } + } + + function getFiles (callBackFunction) { + try { + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> Entering function.') } + + let i = 0 + let j = 0 + + let dateString + + getNextFile() + + function getNextFile () { + try { + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> Entering function.') } + + let targetDate = new Date(cursorDate) + targetDate.setUTCDate(targetDate.getUTCDate() + j) /* Small algorith to allow load first the current date, then alternate between the most forwad and the most backwards ones. */ - if (j === 0) { j++; } - else { - if (j < 0) { - j = -j; - j++; - } - else { - j = -j; - } - } - - let compareTargetDate = new Date(targetDate.toDateString()); - - if (beginDateRange !== undefined) { - - let compareBeginDate = new Date(beginDateRange.toDateString()); - - if (compareTargetDate.valueOf() < compareBeginDate.valueOf()) { - - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> Not loading the file for this date."); } - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> compareTargetDate = " + compareTargetDate); } - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> compareBeginDate = " + compareBeginDate); } - - controlLoop(); - return; - } - } - - if (endDateRange !== undefined) { - - let compareEndDate = new Date(endDateRange.toDateString()); - - if (compareTargetDate.valueOf() > compareEndDate.valueOf()) { - - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> Not loading the file for this date."); } - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> compareTargetDate = " + compareTargetDate); } - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> compareEndDate = " + compareEndDate); } - - controlLoop(); - return; - } - } - - dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); - - let currentDay = Math.trunc((new Date()).valueOf() / (24 * 60 * 60 * 1000)); - let targetDay = Math.trunc(targetDate.valueOf() / (24 * 60 * 60 * 1000)); - - if (targetDay > currentDay) { - - controlLoop(); - - } else { - - if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived); - - } else { - - controlLoop(); - } - - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> getNextFile -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } + if (j === 0) { j++ } else { + if (j < 0) { + j = -j + j++ + } else { + j = -j } + } - function onFileReceived(err, file) { + let compareTargetDate = new Date(targetDate.toDateString()) - try { + if (beginDateRange !== undefined) { + let compareBeginDate = new Date(beginDateRange.toDateString()) - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Entering function."); } + if (compareTargetDate.valueOf() < compareBeginDate.valueOf()) { + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> Not loading the file for this date.') } + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> compareTargetDate = ' + compareTargetDate) } + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> compareBeginDate = ' + compareBeginDate) } - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received OK Response."); } - break; - } - - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received FAIL Response."); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - return; - } - - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received CUSTOM FAIL Response."); } - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> err.message = " + err.message); } - - if (err.message === "File does not exist.") { - - controlLoop(); - return; - - } else { - - callBackFunction(err); - return; - } - - } - - default: { - - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received Unexpected Response."); } - callBackFunction(err); - return; - } - } - - thisObject.files.set(dateString, file); - - controlLoop(); - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> onFileReceived -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } + controlLoop() + return } + } - function controlLoop() { + if (endDateRange !== undefined) { + let compareEndDate = new Date(endDateRange.toDateString()) - try { + if (compareTargetDate.valueOf() > compareEndDate.valueOf()) { + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> Not loading the file for this date.') } + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> compareTargetDate = ' + compareTargetDate) } + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> compareEndDate = ' + compareEndDate) } - if (INFO_LOG === true) { logger.write("[INFO] getFiles -> controlLoop -> Entering function."); } - - if (callBackFunction !== undefined) { - - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE); - - } - - i++; - - if (i < minCursorSize) { - - getNextFile(); - - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> controlLoop -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } + controlLoop() + return } + } + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) + + let currentDay = Math.trunc((new Date()).valueOf() / (24 * 60 * 60 * 1000)) + let targetDay = Math.trunc(targetDate.valueOf() / (24 * 60 * 60 * 1000)) + + if (targetDay > currentDay) { + controlLoop() + } else { + if (thisObject.files.get(dateString) === undefined) { + // We dont reload files we already have. + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + } else { + controlLoop() + } + } } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> getNextFile -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } - } - - function collectGarbage(callBackFunction) { + } + function onFileReceived (err, file) { try { + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> onFileReceived -> Entering function.') } - if (INFO_LOG === true) { logger.write("[INFO] collectGarbage -> Entering function."); } - - date = removeTime(cursorDate); - - let minDate = date.valueOf() - maxCursorSize * ONE_DAY_IN_MILISECONDS / 2; - let maxDate = date.valueOf() + maxCursorSize * ONE_DAY_IN_MILISECONDS / 2; - - for (let key of thisObject.files.keys()) { - - let keyDate = new Date(key); - - if (keyDate.valueOf() < minDate || keyDate.valueOf() > maxDate) { - - thisObject.files.delete(key); - - } + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> onFileReceived -> Received OK Response.') } + break } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> onFileReceived -> Received FAIL Response.') } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> onFileReceived -> Received CUSTOM FAIL Response.') } + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> onFileReceived -> err.message = ' + err.message) } + + if (err.message === 'File does not exist.') { + controlLoop() + return + } else { + callBackFunction(err) + return + } + } + + default: { + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> onFileReceived -> Received Unexpected Response.') } + callBackFunction(err) + return + } + } + + thisObject.files.set(dateString, file) + + controlLoop() } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] collectGarbage -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> onFileReceived -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } + function controlLoop () { + try { + if (INFO_LOG === true) { logger.write('[INFO] getFiles -> controlLoop -> Entering function.') } + + if (callBackFunction !== undefined) { + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) + } + + i++ + + if (i < minCursorSize) { + getNextFile() + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> controlLoop -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } - function getExpectedFiles() { + function collectGarbage (callBackFunction) { + try { + if (INFO_LOG === true) { logger.write('[INFO] collectGarbage -> Entering function.') } - if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } + date = removeTime(cursorDate) - return minCursorSize; + let minDate = date.valueOf() - maxCursorSize * ONE_DAY_IN_MILISECONDS / 2 + let maxDate = date.valueOf() + maxCursorSize * ONE_DAY_IN_MILISECONDS / 2 + for (let key of thisObject.files.keys()) { + let keyDate = new Date(key) + + if (keyDate.valueOf() < minDate || keyDate.valueOf() > maxDate) { + thisObject.files.delete(key) + } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] collectGarbage -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } -} \ No newline at end of file + function getExpectedFiles () { + if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } + + return minCursorSize + } +} diff --git a/src/FileSequence.js b/src/FileSequence.js index 5028143d4..330af1e55 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -1,219 +1,196 @@ - -function newFileSequence() { - const MODULE_NAME = "File Sequence"; - const INFO_LOG = false; - const ERROR_LOG = true; - const logger = newWebDebugLog(); - logger.fileName = MODULE_NAME; +function newFileSequence () { + const MODULE_NAME = 'File Sequence' + const INFO_LOG = false + const ERROR_LOG = true + const logger = newWebDebugLog() + logger.fileName = MODULE_NAME - let thisObject = { - eventHandler: undefined, - getFile: getFile, - getExpectedFiles: getExpectedFiles, - getFilesLoaded: getFilesLoaded, - initialize: initialize, - finalize: finalize + let thisObject = { + eventHandler: undefined, + getFile: getFile, + getExpectedFiles: getExpectedFiles, + getFilesLoaded: getFilesLoaded, + initialize: initialize, + finalize: finalize + } + + let filesLoaded = 0 + let fileCloud + let files = new Map() + let maxSequence = -1 // This is replaced by the content of the sequence file, which contains an index that starts on zero. In the case that the sequence file is not found the default value is -1 so when you add 1 it gives you the amount of files in the sequence, zero. + let market + let devTeam + let bot + let thisSet + let product + + thisObject.eventHandler = newEventHandler() + + let intervalHandle + + return thisObject + + function finalize () { + try { + if (INFO_LOG === true) { logger.write('[INFO] finalize -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] finalize -> devTeam = ' + devTeam.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] finalize -> bot = ' + bot.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] finalize -> thisSet = ' + thisSet.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] finalize -> intervalHandle = ' + intervalHandle) } + + clearInterval(intervalHandle) + + filesLoaded = undefined + fileCloud = undefined + files = undefined + maxSequence = undefined + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err) } } + } - let filesLoaded = 0; - let fileCloud; - let files = new Map; - let maxSequence = -1; // This is replaced by the content of the sequence file, which contains an index that starts on zero. In the case that the sequence file is not found the default value is -1 so when you add 1 it gives you the amount of files in the sequence, zero. - let market; - let devTeam; - let bot; - let thisSet; - let product; + function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction, pOperationsId) { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } - thisObject.eventHandler = newEventHandler(); + exchange = ecosystem.getExchange(pProduct, pExchange) - let intervalHandle; + if (exchange === undefined) { + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + } - return thisObject; + intervalHandle = setInterval(updateFiles, _1_MINUTE_IN_MILISECONDS) - function finalize() { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> intervalHandle = ' + intervalHandle) } - try { + market = pMarket + devTeam = pDevTeam + bot = pBot + thisSet = pSet + product = pProduct - if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] finalize -> devTeam = " + devTeam.codeName); } - if (INFO_LOG === true) { logger.write("[INFO] finalize -> bot = " + bot.codeName); } - if (INFO_LOG === true) { logger.write("[INFO] finalize -> thisSet = " + thisSet.codeName); } - if (INFO_LOG === true) { logger.write("[INFO] finalize -> intervalHandle = " + intervalHandle); } - - clearInterval(intervalHandle); - - filesLoaded = undefined; - fileCloud = undefined; - files = undefined; - maxSequence = undefined; - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } - } - } - - function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { - - try { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } - - exchange = ecosystem.getExchange(pProduct, pExchange); - - if (exchange === undefined) { - - throw "Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; - - } - - intervalHandle = setInterval(updateFiles, _1_MINUTE_IN_MILISECONDS); - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> intervalHandle = " + intervalHandle); } - - market = pMarket; - devTeam = pDevTeam; - bot = pBot; - thisSet = pSet; - product = pProduct; - - fileCloud = newFileCloud(); - fileCloud.initialize(bot); + fileCloud = newFileCloud() + fileCloud.initialize(bot) /* First we will get the sequence max number */ - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, "Sequence", undefined, onSequenceFileReceived); + if (pOperationsId !== undefined) { + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived, pOperationsId) + } else { + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + } - function onSequenceFileReceived(err, file) { + function onSequenceFileReceived (err, file) { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> key = ' + devTeam.codeName + '-' + bot.codeName + '-' + product.codeName) } - try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Received OK Response.') } + break + } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> key = " + devTeam.codeName + "-" + bot.codeName + "-" + product.codeName); } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Received FAIL Response.') } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { + case GLOBAL.CUSTOM_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Received CUSTOM OK Response.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> err.message = ' + err.message) } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received OK Response."); } - break; - } + callBackFunction(err) + return + } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Received CUSTOM FAIL Response.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> err.message = ' + err.message) } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received FAIL Response."); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - return; - } + callBackFunction(err, thisObject) + return + } - case GLOBAL.CUSTOM_OK_RESPONSE.result: { + default: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Received Unexpected Response.') } + callBackFunction(err) + return + } + } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received CUSTOM OK Response."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> err.message = " + err.message); } + maxSequence = Number(file) - callBackFunction(err); - return; - } - - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received CUSTOM FAIL Response."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> err.message = " + err.message); } - - callBackFunction(err, thisObject); - return; - } - - default: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received Unexpected Response."); } - callBackFunction(err); - return; - } - } - - maxSequence = Number(file); - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> maxSequence = " + maxSequence); } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> maxSequence = ' + maxSequence) } /* Now we will get the sequence of files */ - for (let i = 0; i <= maxSequence; i++) { - - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived); - - function onFileReceived(err, file) { - - try { - - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received OK Response."); } - break; - } - - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received FAIL Response."); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - return; - } - - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received CUSTOM FAIL Response."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> err.message = " + err.message); } - - callBackFunction(err); - return; - } - - default: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received Unexpected Response."); } - callBackFunction(err); - return; - } - } - - files.set(i, file); - - filesLoaded++; - - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject); // Note that the callback is called for every file loaded. - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onSequenceFileReceived -> onFileReceived -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } - } - } - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onSequenceFileReceived -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } + for (let i = 0; i <= maxSequence; i++) { + if (pOperationsId !== undefined) { + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived, pOperationsId) + } else { + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) } + function onFileReceived (err, file) { + try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received OK Response.') } + break + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received FAIL Response.') } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received CUSTOM FAIL Response.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> err.message = ' + err.message) } + + callBackFunction(err) + return + } + + default: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received Unexpected Response.') } + callBackFunction(err) + return + } + } + + files.set(i, file) + + filesLoaded++ + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) // Note that the callback is called for every file loaded. + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onSequenceFileReceived -> onFileReceived -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + } + } + } } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onSequenceFileReceived -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } - function updateFiles() { + function updateFiles () { + try { + let updateFiles = 0 - try { - - let updateFiles = 0; - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Entering function."); } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Entering function.') } /* @@ -225,161 +202,141 @@ function newFileSequence() { */ - let currentMaxSequence = maxSequence; + let currentMaxSequence = maxSequence - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> currentMaxSequence = " + currentMaxSequence); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> intervalHandle = " + intervalHandle); } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> currentMaxSequence = ' + currentMaxSequence) } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> intervalHandle = ' + intervalHandle) } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> devTeam = " + devTeam.codeName); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> bot = " + bot.codeName); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> thisSet = " + thisSet.codeName); } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> devTeam = ' + devTeam.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> bot = ' + bot.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> thisSet = ' + thisSet.codeName) } - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, "Sequence", undefined, onSequenceFileReceived); + if (pOperationsId !== undefined) { + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived, pOperationsId) + } else { + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + } - function onSequenceFileReceived(err, file) { + function onSequenceFileReceived (err, file) { + try { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> key = ' + devTeam.codeName + '-' + bot.codeName + '-' + product.codeName) } - try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received OK Response.') } + break + } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> key = " + devTeam.codeName + "-" + bot.codeName + "-" + product.codeName); } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received FAIL Response.') } + return + } - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { + case GLOBAL.CUSTOM_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received CUSTOM OK Response.') } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> err.message = ' + err.message) } + return + } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received OK Response."); } - break; - } + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received CUSTOM FAIL Response.') } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> err.message = ' + err.message) } + return + } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + default: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received Unexpected Response.') } + return + } + } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received FAIL Response."); } - return; - } + maxSequence = Number(file) - case GLOBAL.CUSTOM_OK_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received CUSTOM OK Response."); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> err.message = " + err.message); } - return; - } - - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received CUSTOM FAIL Response."); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> err.message = " + err.message); } - return; - } - - default: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received Unexpected Response."); } - return; - } - } - - maxSequence = Number(file); - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> maxSequence = " + maxSequence); } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> maxSequence = ' + maxSequence) } /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ - for (let i = currentMaxSequence; i <= maxSequence; i++) { + for (let i = currentMaxSequence; i <= maxSequence; i++) { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> i = ' + i) } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> i = " + i); } - - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived); - - function onFileReceived(err, file) { - - try { - - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received OK Response."); } - break; - } - - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received FAIL Response."); } - return; - } - - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received CUSTOM FAIL Response."); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> err.message = " + err.message); } - return; - } - - default: { - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received Unexpected Response."); } - return; - } - } - - files.set(i, file); - - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> File Updated."); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> i = " + i); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> file.length = " + file.length); } - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> filesLoaded = " + filesLoaded); } - - if (i !== currentMaxSequence) { - - filesLoaded++; - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> filesLoaded = " + filesLoaded); } - - } - - if (filesLoaded === maxSequence + 1) { - - thisObject.eventHandler.raiseEvent("Files Updated", undefined); - if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Files Updated event Raised."); } - - } - - } catch (err) { - if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onSequenceFileReceived -> onFileReceived -> err = " + err); } - } - } - } - - } catch (err) { - if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onSequenceFileReceived -> err = " + err); } - } + if (pOperationsId !== undefined) { + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived, pOperationsId) + } else { + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) } + function onFileReceived (err, file) { + try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received OK Response.') } + break + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received FAIL Response.') } + return + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received CUSTOM FAIL Response.') } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> err.message = ' + err.message) } + return + } + + default: { + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received Unexpected Response.') } + return + } + } + + files.set(i, file) + + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> File Updated.') } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> i = ' + i) } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> file.length = ' + file.length) } + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> filesLoaded = ' + filesLoaded) } + + if (i !== currentMaxSequence) { + filesLoaded++ + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> filesLoaded = ' + filesLoaded) } + } + + if (filesLoaded === maxSequence + 1) { + thisObject.eventHandler.raiseEvent('Files Updated', undefined) + if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Files Updated event Raised.') } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onSequenceFileReceived -> onFileReceived -> err = ' + err) } + } + } + } } catch (err) { - if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> err = " + err); } + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onSequenceFileReceived -> err = ' + err) } } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> err = ' + err) } } + } - function getFile(pSequence) { + function getFile (pSequence) { + if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> pSequence = ' + pSequence) } - if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] getFile -> pSequence = " + pSequence); } + return files.get(pSequence) + } - return files.get(pSequence); + function getExpectedFiles () { + if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } - } + return maxSequence + 1 + } - function getExpectedFiles() { + function getFilesLoaded () { + if (INFO_LOG === true) { logger.write('[INFO] getFilesLoaded -> Entering function.') } - if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } - - return maxSequence + 1; - - } - - function getFilesLoaded() { - - if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> Entering function."); } - - return filesLoaded; - - } - -} \ No newline at end of file + return filesLoaded + } +} diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 8ffc71a75..324a2c1f7 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -1,4 +1,4 @@ - +  function newMarketFiles () { const MODULE_NAME = 'Market Files' const INFO_LOG = false diff --git a/src/SingleFile.js b/src/SingleFile.js index 65c8d8472..fe4c22914 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -1,105 +1,89 @@ - -function newSingleFile() { +  +function newSingleFile () { + const MODULE_NAME = 'Single File' + const INFO_LOG = false + const ERROR_LOG = true + const logger = newWebDebugLog() + logger.fileName = MODULE_NAME - const MODULE_NAME = "Single File"; - const INFO_LOG = false; - const ERROR_LOG = true; - const logger = newWebDebugLog(); - logger.fileName = MODULE_NAME; + let thisObject = { + getFile: getFile, + initialize: initialize + } - let thisObject = { - getFile: getFile, - initialize: initialize - } + let fileCloud + let file - let fileCloud; - let file; + return thisObject - return thisObject; + function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } - function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + let exchange = ecosystem.getExchange(pProduct, pExchange) - try { + if (pExchange !== undefined && exchange === undefined) { + // We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product. - if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + } - let exchange = ecosystem.getExchange(pProduct, pExchange); - - if (pExchange !== undefined && exchange === undefined) { // We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product. - - throw "Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; - - } - - fileCloud = newFileCloud(); - fileCloud.initialize(pBot); + fileCloud = newFileCloud() + fileCloud.initialize(pBot) /* Now we will get the file */ - fileCloud.getFile(pDevTeam, pBot, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived); + fileCloud.getFile(pDevTeam, pBot, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) - function onFileReceived(err, pFile) { + function onFileReceived (err, pFile) { + try { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Entering function.') } - try { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Entering function."); } - - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received OK Response."); } - break; - } - - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received FAIL Response."); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - return; - } - - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response."); } - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> err.message = " + err.message); } - - callBackFunction(err); - return; - } - - default: { - - if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received Unexpected Response."); } - callBackFunction(err); - return; - } - } - - file = pFile; - - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE); - - } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); - } + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received OK Response.') } + break } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received FAIL Response.') } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response.') } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> err.message = ' + err.message) } + + callBackFunction(err) + return + } + + default: { + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received Unexpected Response.') } + callBackFunction(err) + return + } + } + + file = pFile + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) } catch (err) { - - if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + } - function getFile() { + function getFile () { + if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } - if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } - - return file; - - } - -} \ No newline at end of file + return file + } +} From 83e5aff52f8cf810aa970f6ce5d72c8090ef44dd Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 5 Mar 2019 09:27:12 +0100 Subject: [PATCH 07/92] Last Nikola Update --- src/FileSequence.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/FileSequence.js b/src/FileSequence.js index 330af1e55..9bf99f601 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -61,7 +61,12 @@ function newFileSequence () { throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } - intervalHandle = setInterval(updateFiles, _1_MINUTE_IN_MILISECONDS) + + if (pOperationsId !== undefined) { + intervalHandle = setInterval(function () { updateFiles(pOperationsId) }, _1_MINUTE_IN_MILISECONDS) + } else { + intervalHandle = setInterval(updateFiles, _1_MINUTE_IN_MILISECONDS) + } if (INFO_LOG === true) { logger.write('[INFO] initialize -> intervalHandle = ' + intervalHandle) } @@ -186,7 +191,7 @@ function newFileSequence () { } } - function updateFiles () { + function updateFiles (pOperationsId) { try { let updateFiles = 0 From 9cd3126c7a28b38ac5498fa8bceb347336c34e8e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 19 Mar 2019 15:00:22 +0100 Subject: [PATCH 08/92] LOG ERROR when file is not in valid JSON format --- src/FileCloud.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index 49e5002c7..1acddf35c 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -185,14 +185,17 @@ function newFileCloud () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data) return } catch (err) { - if (INFO_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> containerName = ' + containerName) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } let customErr = { result: GLOBAL.CUSTOM_OK_RESPONSE.result, message: 'Data not in JSON Format.' } - if (INFO_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } callBackFunction(customErr, text) return From 03253296bc5ad251c4f8ccc0ca81359721e8509d Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 23 Mar 2019 10:17:30 +0100 Subject: [PATCH 09/92] Clone Id not as a parameter --- src/FileCloud.js | 14 +++++--------- src/FileSequence.js | 35 +++++++---------------------------- 2 files changed, 12 insertions(+), 37 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index 1acddf35c..763eb92ed 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -33,19 +33,15 @@ function newFileCloud () { } } - function getFile (pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction, pOperationsId) { + function getFile (pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } const MAX_RETRIES = 3 - if (pOperationsId !== undefined) { - getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction, pOperationsId) - } else { - getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - } + getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction, pOperationsId) { + function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } @@ -96,8 +92,8 @@ function newFileCloud () { } if (pBot !== undefined) { - if (pOperationsId !== undefined) { - filePath = filePath.replace('@Bot', pBot.codeName + '-' + pOperationsId) + if (pBot.cloneId !== undefined) { + filePath = filePath.replace('@Bot', pBot.codeName + '-' + pBot.cloneId) } else { filePath = filePath.replace('@Bot', pBot.codeName) } diff --git a/src/FileSequence.js b/src/FileSequence.js index 9bf99f601..ff68362ee 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -50,7 +50,7 @@ function newFileSequence () { } } - function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction, pOperationsId) { + function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } @@ -61,12 +61,7 @@ function newFileSequence () { throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } - - if (pOperationsId !== undefined) { - intervalHandle = setInterval(function () { updateFiles(pOperationsId) }, _1_MINUTE_IN_MILISECONDS) - } else { - intervalHandle = setInterval(updateFiles, _1_MINUTE_IN_MILISECONDS) - } + intervalHandle = setInterval(updateFiles(), _1_MINUTE_IN_MILISECONDS) if (INFO_LOG === true) { logger.write('[INFO] initialize -> intervalHandle = ' + intervalHandle) } @@ -81,11 +76,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - if (pOperationsId !== undefined) { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived, pOperationsId) - } else { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) - } + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -134,11 +125,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - if (pOperationsId !== undefined) { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived, pOperationsId) - } else { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) - } + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -191,7 +178,7 @@ function newFileSequence () { } } - function updateFiles (pOperationsId) { + function updateFiles () { try { let updateFiles = 0 @@ -216,11 +203,7 @@ function newFileSequence () { if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> bot = ' + bot.codeName) } if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> thisSet = ' + thisSet.codeName) } - if (pOperationsId !== undefined) { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived, pOperationsId) - } else { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) - } + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -265,11 +248,7 @@ function newFileSequence () { for (let i = currentMaxSequence; i <= maxSequence; i++) { if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> i = ' + i) } - if (pOperationsId !== undefined) { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived, pOperationsId) - } else { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) - } + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { From 7c2b7ed19012b34b8438ed39002d81e240b2b66b Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 27 Mar 2019 22:37:41 +0100 Subject: [PATCH 10/92] Refinements to prevents unnecesary updates --- src/FileCursor.js | 3 +++ src/FileSequence.js | 7 +++++++ src/MarketFiles.js | 3 +++ 3 files changed, 13 insertions(+) diff --git a/src/FileCursor.js b/src/FileCursor.js index 755cacc03..e25dc3a3d 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -33,6 +33,7 @@ function newFileCursor () { let endDateRange let intervalHandle + let finalized = false return thisObject @@ -47,6 +48,7 @@ function newFileCursor () { fileCloud = undefined periodName = undefined timePeriod = undefined + finalized = true } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err) } } @@ -88,6 +90,7 @@ function newFileCursor () { function updateFiles () { try { + if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Entering function.') } /* diff --git a/src/FileSequence.js b/src/FileSequence.js index ff68362ee..615ea4225 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -24,6 +24,8 @@ function newFileSequence () { let bot let thisSet let product + let finalized = false + let initialized = false thisObject.eventHandler = newEventHandler() @@ -45,6 +47,7 @@ function newFileSequence () { fileCloud = undefined files = undefined maxSequence = undefined + finalized = true } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err) } } @@ -83,6 +86,8 @@ function newFileSequence () { if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> key = ' + devTeam.codeName + '-' + bot.codeName + '-' + product.codeName) } + initialized = true + switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Received OK Response.') } @@ -180,6 +185,8 @@ function newFileSequence () { function updateFiles () { try { + if (finalized === true || initialized === false) { return } + let updateFiles = 0 if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Entering function.') } diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 324a2c1f7..7bd256cc9 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -31,6 +31,7 @@ function newMarketFiles () { thisObject.eventHandler = newEventHandler() let intervalHandle + let finalized = false return thisObject @@ -42,6 +43,7 @@ function newMarketFiles () { filesLoaded = undefined files = undefined + finalized = true } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err) } } @@ -128,6 +130,7 @@ function newMarketFiles () { function updateFiles () { try { + if (finalized === true) { return } let updateFiles = 0 if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Entering function.') } From 9279d9e5e12d386aea3eb72cb1b34547d6f4956d Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 6 Apr 2019 21:24:59 +0200 Subject: [PATCH 11/92] Working with Azyre and AAWeb as Storage Providers --- src/FileCloud.js | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index 763eb92ed..3eb6b92dd 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -1,16 +1,16 @@ -function newFileCloud () { +function newFileCloud() { const MODULE_NAME = 'File Cloud' const INFO_LOG = false const ERROR_LOG = true const logger = newWebDebugLog() logger.fileName = MODULE_NAME - /* + /* - This is the module in the system that actually connects to the cloud storage and grabs from there the needed files. + This is the module in the system that actually connects to the cloud storage and grabs from there the needed files. - */ + */ var thisObject = { getFile: getFile, @@ -21,19 +21,33 @@ function newFileCloud () { return thisObject - function initialize (pBot) { + function initialize(pBot) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] initialize -> pBot = ' + pBot.codeName) } - blobService = AzureStorage.Blob.createBlobServiceWithSas(pBot.storage.fileUri, pBot.storage.sas) + switch (STORAGE_PROVIDER) { + case 'Azure': { + blobService = AzureStorage.Blob.createBlobServiceWithSas(pBot.storage.fileUri, pBot.storage.sas) + break; + } + case 'AAWeb': { + blobService = newFileStorage(); + break; + } + default: { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> Storage Provider not supported -> process.env.STORAGE_PROVIDER = ' + STORAGE_PROVIDER) } + return; + } + } + } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } - // callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); TODO> Handle this callback. + // callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); TODO> Handle this callback. } } - function getFile (pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile(pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } @@ -41,7 +55,7 @@ function newFileCloud () { getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively(pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } @@ -126,7 +140,7 @@ function newFileCloud () { blobService.getBlobToText(containerName, filePath + '/' + fileName, onFileReceived) - function onFileReceived (err, text, response) { + function onFileReceived(err, text, response) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } @@ -134,7 +148,8 @@ function newFileCloud () { let data - if (err) { + if (err && err.result !== GLOBAL.DEFAULT_OK_RESPONSE.result) { + if (err.code === 'BlobNotFound' | err.code === 'FileNotFound' | err.code === 'ParentNotFound') { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, From af0a1c56f31f2abae75b2d14a572c251b84314d3 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 7 Apr 2019 19:50:25 +0200 Subject: [PATCH 12/92] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index a22ed6bfe..71f275c98 100644 --- a/README.md +++ b/README.md @@ -25,3 +25,9 @@ The system-wide concept of a Daily File is defined as a file containing data cor Since for each market there are multiple Daily Files (one for 1 min time period, one for 2 min, 3 min, 4, 5, 10, 15, 20, 30 and 45 minutes) this data structure loads all of these files and have them readily available in memory for consumption. Moreover, it uses the concept of a File Cursor to load more files of the previous days and the following days in order to have them ready in memory if needed. + +## Installation + +The Files Component is part of the Canvas App. Please continue here to learn how to install it: + +https://github.com/Superalgos/CanvasApp From 1b7f55add5bba52397f77a6837a9b652698289dd Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 9 Apr 2019 15:58:30 +0200 Subject: [PATCH 13/92] Storage Gateway final Implementation --- FileStorage | 54 ++++++++++++++++++++++++++++++++++++++++++++++ src/FileStorage.js | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 FileStorage create mode 100644 src/FileStorage.js diff --git a/FileStorage b/FileStorage new file mode 100644 index 000000000..c6c12d006 --- /dev/null +++ b/FileStorage @@ -0,0 +1,54 @@ +function newFileStorage() { + + const MODULE_NAME = 'File Cloud' + const INFO_LOG = false + const ERROR_LOG = true + const logger = newWebDebugLog() + logger.fileName = MODULE_NAME + + thisObject = { + getBlobToText: getBlobToText + }; + + return thisObject; + + function getBlobToText(pContainerName, pPath, callBackFunction) { + + let xhttp = new XMLHttpRequest(); + xhttp.onreadystatechange = function () { + if (this.readyState === 4 && this.status === 200) { + + try { + let response = JSON.parse(xhttp.responseText); + callBackFunction(response.err, response.text, ""); + } catch (err) { + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. "); } + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] AppPreLoader -> getBlobToText -> response.text = " & response.text); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, "", ""); + } + } + }; + + try { + + let request = { + conatinerName: pContainerName, + path: pPath + }; + + request = JSON.stringify(request); + + let path = "FileService"; + + let blob = new Blob([request], { type: 'text/plain' }); + + xhttp.open("GET", path, true); + xhttp.send(blob); + + } catch (err) { + + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] getBlobToText -> err.message = " & err.message); } + + } + } +} \ No newline at end of file diff --git a/src/FileStorage.js b/src/FileStorage.js new file mode 100644 index 000000000..85bad1b01 --- /dev/null +++ b/src/FileStorage.js @@ -0,0 +1,54 @@ +function newFileStorage() { + + const MODULE_NAME = 'File Cloud' + const INFO_LOG = false + const ERROR_LOG = true + const logger = newWebDebugLog() + logger.fileName = MODULE_NAME + + thisObject = { + getBlobToText: getBlobToText + }; + + return thisObject; + + function getBlobToText(pContainerName, pPath, callBackFunction) { + + try { + if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } + + let xhttp = new XMLHttpRequest(); + xhttp.onreadystatechange = function () { + if (this.readyState === 4 && this.status === 200) { + + try { + let response = JSON.parse(xhttp.responseText); + callBackFunction(response.err, response.text, ""); + } catch (err) { + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. "); } + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] AppPreLoader -> getBlobToText -> response.text = " & response.text); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, "", ""); + } + } + }; + + + let request = { + conatinerName: pContainerName, + path: pPath + }; + + request = JSON.stringify(request); + + let path = window.canvasApp.urlPrefix + "FileService"; + + let blob = new Blob([request], { type: 'text/plain' }); + + xhttp.open("POST", path, true); + xhttp.send(blob); + + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] getBlobToText -> err = ' + err) } + } + } +} \ No newline at end of file From 08cbc07bb5aa12cbdcec663878c2af503171e127 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 10 Apr 2019 13:51:50 +0200 Subject: [PATCH 14/92] Error Logging Improved --- src/DailyFiles.js | 18 +++++++++--------- src/FileCloud.js | 12 ++++++------ src/FileCursor.js | 28 ++++++++++++++-------------- src/FileSequence.js | 14 +++++++------- src/FileStorage.js | 2 +- src/MarketFiles.js | 10 +++++----- src/SingleFile.js | 4 ++-- 7 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index cf94b1fe2..d67671a38 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -43,7 +43,7 @@ function newDailyFiles () { item.finalize() } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err.stack) } } } @@ -168,7 +168,7 @@ function newDailyFiles () { if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> expectedFiles = ' + expectedFiles) } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } @@ -194,17 +194,17 @@ function newDailyFiles () { } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> loadThemAll -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> loadThemAll -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } @@ -246,7 +246,7 @@ function newDailyFiles () { callBackWhenFileReceived(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) // Note that the call back is called for every file loaded at each cursor. } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] onFileReceived -> err = ' + err.stack) } callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) } } @@ -257,7 +257,7 @@ function newDailyFiles () { return fileCursors.get(pPeriod) } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] getFileCursor -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFileCursor -> err = ' + err.stack) } } } @@ -282,7 +282,7 @@ function newDailyFiles () { fileCursor.reload(onFileReceived) } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setDatetime -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] setDatetime -> err = ' + err.stack) } } } @@ -307,7 +307,7 @@ function newDailyFiles () { fileCursor.reload(onFileReceived) } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> err = ' + err.stack) } } } diff --git a/src/FileCloud.js b/src/FileCloud.js index 3eb6b92dd..e527fec6a 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -42,7 +42,7 @@ function newFileCloud() { } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } // callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); TODO> Handle this callback. } } @@ -166,7 +166,7 @@ function newFileCloud() { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Unexpected Error Ocurred.') } - if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = ' + err.stack) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> text = ' + text) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> response = ' + response) } @@ -196,7 +196,7 @@ function newFileCloud() { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data) return } catch (err) { - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> err = ' + err.stack) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> containerName = ' + containerName) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } @@ -213,17 +213,17 @@ function newFileCloud() { } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } diff --git a/src/FileCursor.js b/src/FileCursor.js index e25dc3a3d..cb1a360d6 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -50,7 +50,7 @@ function newFileCursor () { timePeriod = undefined finalized = true } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err.stack) } } } @@ -83,7 +83,7 @@ function newFileCursor () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } @@ -168,12 +168,12 @@ function newFileCursor () { if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> thisSet = ' + thisSet.codeName) } if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> dateString = ' + dateString) } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onFileReceived -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> err = ' + err.stack) } } } @@ -297,7 +297,7 @@ function newFileCursor () { default: } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> enterSavingMode -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> enterSavingMode -> err = ' + err.stack) } } } @@ -376,11 +376,11 @@ function newFileCursor () { default: } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> exitSavingMode -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> exitSavingMode -> err = ' + err.stack) } } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> err = ' + err.stack) } } } @@ -395,7 +395,7 @@ function newFileCursor () { cursorDate = pDatetime } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setDatetime -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] setDatetime -> err = ' + err.stack) } } } @@ -407,7 +407,7 @@ function newFileCursor () { collectGarbage(callBackFunction) } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] reload -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] reload -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } @@ -485,7 +485,7 @@ function newFileCursor () { } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> getNextFile -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> getNextFile -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } @@ -530,7 +530,7 @@ function newFileCursor () { controlLoop() } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> onFileReceived -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } @@ -549,12 +549,12 @@ function newFileCursor () { getNextFile() } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> controlLoop -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> controlLoop -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getFiles -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } @@ -576,7 +576,7 @@ function newFileCursor () { } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] collectGarbage -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] collectGarbage -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } diff --git a/src/FileSequence.js b/src/FileSequence.js index 615ea4225..a78a18625 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -49,7 +49,7 @@ function newFileSequence () { maxSequence = undefined finalized = true } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err.stack) } } } @@ -167,18 +167,18 @@ function newFileSequence () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) // Note that the callback is called for every file loaded. } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onSequenceFileReceived -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onSequenceFileReceived -> onFileReceived -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onSequenceFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onSequenceFileReceived -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } @@ -299,16 +299,16 @@ function newFileSequence () { if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Files Updated event Raised.') } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onSequenceFileReceived -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onSequenceFileReceived -> onFileReceived -> err = ' + err.stack) } } } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onSequenceFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onSequenceFileReceived -> err = ' + err.stack) } } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> err = ' + err.stack) } } } diff --git a/src/FileStorage.js b/src/FileStorage.js index 85bad1b01..5f0879e1a 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -48,7 +48,7 @@ function newFileStorage() { xhttp.send(blob); } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] getBlobToText -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] getBlobToText -> err = ' + err.stack) } } } } \ No newline at end of file diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 7bd256cc9..fa20c30f5 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -45,7 +45,7 @@ function newMarketFiles () { files = undefined finalized = true } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err.stack) } } } @@ -116,14 +116,14 @@ function newMarketFiles () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) // Note that the callback is called for every file loaded. } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } @@ -183,13 +183,13 @@ function newMarketFiles () { thisObject.eventHandler.raiseEvent('Files Updated', undefined) } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onFileReceived -> err = ' + err.stack) } } } } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> err = ' + err.stack) } } } diff --git a/src/SingleFile.js b/src/SingleFile.js index fe4c22914..af741c69a 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -71,12 +71,12 @@ function newSingleFile () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } From dbf279d77c8d869635f26023e48eb1f1dba79f9b Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 11 Apr 2019 21:32:53 +0200 Subject: [PATCH 15/92] Prevention to use Cursor after finalized --- src/FileCursor.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/FileCursor.js b/src/FileCursor.js index cb1a360d6..87a653a8d 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -141,6 +141,7 @@ function newFileCursor () { function onFileReceived (err, file) { try { + if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Entering function.') } switch (err.result) { @@ -179,6 +180,7 @@ function newFileCursor () { function setTimePeriod (pTimePeriod, pDatetime) { try { + if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> Entering function.') } /* @@ -386,6 +388,7 @@ function newFileCursor () { function setDatetime (pDatetime) { try { + if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] setDatetime -> Entering function.') } if (pDatetime === undefined) { @@ -401,6 +404,7 @@ function newFileCursor () { function reload (callBackFunction) { try { + if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] reload -> Entering function.') } getFiles(callBackFunction) @@ -414,6 +418,7 @@ function newFileCursor () { function getFiles (callBackFunction) { try { + if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] getFiles -> Entering function.') } let i = 0 @@ -561,6 +566,7 @@ function newFileCursor () { function collectGarbage (callBackFunction) { try { + if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] collectGarbage -> Entering function.') } date = removeTime(cursorDate) @@ -582,6 +588,7 @@ function newFileCursor () { } function getExpectedFiles () { + if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } return minCursorSize From cbf76e33b2d49eee89cb7c4d8f2873e95ab6ed64 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 21 Apr 2019 19:05:37 +0200 Subject: [PATCH 16/92] If a file is missing, wont halt initialization --- src/MarketFiles.js | 54 ++++++++++++++++------------------------------ 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index fa20c30f5..6e1797655 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -11,11 +11,13 @@ function newMarketFiles () { getFile: getFile, getExpectedFiles: getExpectedFiles, getFilesLoaded: getFilesLoaded, + getFilesNotLoaded: getFilesNotLoaded, initialize: initialize, finalize: finalize } let filesLoaded = 0 + let filesNotLoaded = 0 let fileCloud @@ -83,38 +85,20 @@ function newMarketFiles () { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Entering function.') } - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received OK Response.') } - break - } - - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received FAIL Response.') } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) - return - } - - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> err.message = ' + err.message) } - - callBackFunction(err) - return - } - - default: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received Unexpected Response.') } - callBackFunction(err) - return - } + if (err.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { + files.set(periodTime, file) + filesLoaded++ + } else { + filesNotLoaded++ } - files.set(periodTime, file) - - filesLoaded++ - - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) // Note that the callback is called for every file loaded. + if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { + if (filesLoaded > filesNotLoaded) { + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) + } else { + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, thisObject) + } + } } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) @@ -194,20 +178,18 @@ function newMarketFiles () { } function getFile (pPeriod) { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } - return files.get(pPeriod) } function getExpectedFiles () { - if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } - return marketFilesPeriods.length } function getFilesLoaded () { - if (INFO_LOG === true) { logger.write('[INFO] getFilesLoaded -> Entering function.') } - return filesLoaded } + + function getFilesNotLoaded () { + return filesNotLoaded + } } From 5b9c586f141cb84d62dc1604de0f4c96e3d870bc Mon Sep 17 00:00:00 2001 From: Julian Molina Date: Wed, 24 Apr 2019 11:09:44 +0200 Subject: [PATCH 17/92] Legal --- CODE_OF_CONDUCT.md | 46 +++++++++++ CONTRIBUTING.md | 9 ++ LICENSE | 201 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 256 insertions(+) create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..4b37b0e16 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,46 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team using the online form in the contacts page of the superalgos.org website. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..1d1736260 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,9 @@ +# Contributing + +We are thrilled you are interested in contributing to this repository. Your help may be essential to the project's success and we want you in! + +Superalgos is a collective project, currently at a very early stage. We not only welcome contributions but may even choose to incentivize relevant contributions via the project's native crypto token. + +Please read the [Join the Dev Team]( https://superalgos.org/developers-join-dev-team.shtml) page in our website to learn more about how you may become a relevant member of the Community and even make a living out of contributing code to the project. + +Contributions are deemed _voluntary work_, irrespective of the incentives that may be in place, and are subject to the conditions described in the LICENSE \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From e2f8c52c5879f90aa11d7caf83918ef3b42088ca Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 24 Apr 2019 20:27:03 +0200 Subject: [PATCH 18/92] BUG thisObject without LET solved --- src/FileCloud.js | 22 +++++------ src/FileStorage.js | 91 ++++++++++++++++++++++------------------------ 2 files changed, 53 insertions(+), 60 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index e527fec6a..f979610da 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -1,5 +1,5 @@ -function newFileCloud() { +function newFileCloud () { const MODULE_NAME = 'File Cloud' const INFO_LOG = false const ERROR_LOG = true @@ -12,7 +12,7 @@ function newFileCloud() { */ - var thisObject = { + let thisObject = { getFile: getFile, initialize: initialize } @@ -21,7 +21,7 @@ function newFileCloud() { return thisObject - function initialize(pBot) { + function initialize (pBot) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] initialize -> pBot = ' + pBot.codeName) } @@ -29,25 +29,24 @@ function newFileCloud() { switch (STORAGE_PROVIDER) { case 'Azure': { blobService = AzureStorage.Blob.createBlobServiceWithSas(pBot.storage.fileUri, pBot.storage.sas) - break; + break } case 'AAWeb': { - blobService = newFileStorage(); - break; + blobService = newFileStorage() + break } default: { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> Storage Provider not supported -> process.env.STORAGE_PROVIDER = ' + STORAGE_PROVIDER) } - return; + return } } - } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } // callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); TODO> Handle this callback. } } - function getFile(pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } @@ -55,7 +54,7 @@ function newFileCloud() { getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - function getFileRecursively(pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } @@ -140,7 +139,7 @@ function newFileCloud() { blobService.getBlobToText(containerName, filePath + '/' + fileName, onFileReceived) - function onFileReceived(err, text, response) { + function onFileReceived (err, text, response) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } @@ -149,7 +148,6 @@ function newFileCloud() { let data if (err && err.result !== GLOBAL.DEFAULT_OK_RESPONSE.result) { - if (err.code === 'BlobNotFound' | err.code === 'FileNotFound' | err.code === 'ParentNotFound') { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, diff --git a/src/FileStorage.js b/src/FileStorage.js index 5f0879e1a..d398a0ffc 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -1,54 +1,49 @@ -function newFileStorage() { +function newFileStorage () { + const MODULE_NAME = 'File Cloud' + const INFO_LOG = false + const ERROR_LOG = true + const logger = newWebDebugLog() + logger.fileName = MODULE_NAME - const MODULE_NAME = 'File Cloud' - const INFO_LOG = false - const ERROR_LOG = true - const logger = newWebDebugLog() - logger.fileName = MODULE_NAME + let thisObject = { + getBlobToText: getBlobToText + } - thisObject = { - getBlobToText: getBlobToText - }; + return thisObject - return thisObject; + function getBlobToText (pContainerName, pPath, callBackFunction) { + try { + if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } - function getBlobToText(pContainerName, pPath, callBackFunction) { - - try { - if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } - - let xhttp = new XMLHttpRequest(); - xhttp.onreadystatechange = function () { - if (this.readyState === 4 && this.status === 200) { - - try { - let response = JSON.parse(xhttp.responseText); - callBackFunction(response.err, response.text, ""); - } catch (err) { - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. "); } - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] AppPreLoader -> getBlobToText -> response.text = " & response.text); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, "", ""); - } - } - }; - - - let request = { - conatinerName: pContainerName, - path: pPath - }; - - request = JSON.stringify(request); - - let path = window.canvasApp.urlPrefix + "FileService"; - - let blob = new Blob([request], { type: 'text/plain' }); - - xhttp.open("POST", path, true); - xhttp.send(blob); - - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] getBlobToText -> err = ' + err.stack) } + let xhttp = new XMLHttpRequest() + xhttp.onreadystatechange = function () { + if (this.readyState === 4 && this.status === 200) { + try { + let response = JSON.parse(xhttp.responseText) + callBackFunction(response.err, response.text, '') + } catch (err) { + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. ') } + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> response.text = ' & response.text) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, '', '') + } } + } + + let request = { + conatinerName: pContainerName, + path: pPath + } + + request = JSON.stringify(request) + + let path = window.canvasApp.urlPrefix + 'FileService' + + let blob = new Blob([request], { type: 'text/plain' }) + + xhttp.open('POST', path, true) + xhttp.send(blob) + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] getBlobToText -> err = ' + err.stack) } } -} \ No newline at end of file + } +} From 5bad7f7be1efd4fea66834096ddf610999169200 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 30 Apr 2019 18:07:21 +0200 Subject: [PATCH 19/92] BUG Fixed: reading one day before the authorized datarange --- src/FileCursor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index 87a653a8d..842a22f1b 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -396,7 +396,7 @@ function newFileCursor () { return } - cursorDate = pDatetime + cursorDate = removeTime(pDatetime) } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] setDatetime -> err = ' + err.stack) } } From 5feb1b693cccd4de24595e26f7e73e3eb3e7b4fb Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 30 Apr 2019 19:40:29 +0200 Subject: [PATCH 20/92] BUG FIXED: Dates as string --- src/DailyFiles.js | 29 +++++++++++++---------------- src/FileCursor.js | 22 +++++++--------------- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index d67671a38..57825c527 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -78,8 +78,8 @@ function newDailyFiles () { case GLOBAL.DEFAULT_OK_RESPONSE.result: { if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received OK Response.') } - beginDateRange = new Date(pFile.begin) - endDateRange = new Date(pFile.end) + beginDateRange = pFile.begin + endDateRange = pFile.end break } @@ -141,23 +141,20 @@ function newDailyFiles () { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received OK Response.') } - break - } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received OK Response.') } + break + } case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received FAIL Response.') } - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) - return - } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received FAIL Response.') } + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } default: { - - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received Unexpected Response.') } - callBackWhenFileReceived(err) - return - } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received Unexpected Response.') } + callBackWhenFileReceived(err) + return + } } fileCursors.set(periodTime, fileCursor) diff --git a/src/FileCursor.js b/src/FileCursor.js index 842a22f1b..febf86343 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -66,7 +66,7 @@ function newFileCursor () { bot = pBot thisSet = pSet periodName = pPeriodName - cursorDate = pCursorDate + cursorDate = removeTime(pCursorDate) timePeriod = pTimePeriod beginDateRange = pBeginDateRange endDateRange = pEndDateRange @@ -194,7 +194,7 @@ function newFileCursor () { */ - cursorDate = pDatetime // Adjust the cursor date to the one received. + cursorDate = removeTime(pDatetime) // Adjust the cursor date to the one received. let positionA @@ -445,29 +445,21 @@ function newFileCursor () { } } - let compareTargetDate = new Date(targetDate.toDateString()) + let compareTargetDate = new Date(targetDate.valueOf()) if (beginDateRange !== undefined) { - let compareBeginDate = new Date(beginDateRange.toDateString()) - - if (compareTargetDate.valueOf() < compareBeginDate.valueOf()) { - if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> Not loading the file for this date.') } - if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> compareTargetDate = ' + compareTargetDate) } - if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> compareBeginDate = ' + compareBeginDate) } + let compareBeginDate = beginDateRange + if (compareTargetDate.valueOf() < compareBeginDate) { controlLoop() return } } if (endDateRange !== undefined) { - let compareEndDate = new Date(endDateRange.toDateString()) - - if (compareTargetDate.valueOf() > compareEndDate.valueOf()) { - if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> Not loading the file for this date.') } - if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> compareTargetDate = ' + compareTargetDate) } - if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> compareEndDate = ' + compareEndDate) } + let compareEndDate = endDateRange + if (compareTargetDate.valueOf() > compareEndDate) { controlLoop() return } From b7bdbc6e1d14dbf7999e05d9365019cdc9d3a25b Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 18 May 2019 19:32:35 +0200 Subject: [PATCH 21/92] Cleaning and allowing that daily or market dont block plotter initialization when they are not there --- src/DailyFiles.js | 93 ++-------------------------------------------- src/MarketFiles.js | 46 +---------------------- 2 files changed, 4 insertions(+), 135 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 57825c527..fe6eefe52 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -1,7 +1,6 @@  function newDailyFiles () { const MODULE_NAME = 'Daily Files' - const INFO_LOG = false const ERROR_LOG = true const logger = newWebDebugLog() logger.fileName = MODULE_NAME @@ -29,8 +28,6 @@ function newDailyFiles () { function finalize () { try { - if (INFO_LOG === true) { logger.write('[INFO] finalize -> Entering function.') } - filesLoaded = undefined expectedFiles = undefined fileCloud = undefined @@ -38,8 +35,6 @@ function newDailyFiles () { fileCursors.forEach(finalizeEach) function finalizeEach (item, key, mapObj) { - if (INFO_LOG === true) { logger.write('[INFO] finalize -> finalizeEach -> Entering function.') } - item.finalize() } } catch (err) { @@ -49,9 +44,6 @@ function newDailyFiles () { function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } - callBackWhenFileReceived = callBackFunction let exchange = ecosystem.getExchange(pProduct, pExchange) @@ -69,15 +61,11 @@ function newDailyFiles () { function onDataRangeReceived (err, pFile) { try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Entering function.') } - let beginDateRange let endDateRange switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received OK Response.') } - beginDateRange = pFile.begin endDateRange = pFile.end @@ -85,22 +73,16 @@ function newDailyFiles () { } case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received FAIL Response.') } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) return } case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> err.message = ' + err.message) } - if (err.message === 'File does not exist.') { - if (INFO_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> No Date Range file found. maxDate will be set to current datetime. ') } - - endDateRange = new Date() - break + err.message = 'Dataset Unavailable.' + callBackFunction(err) + return } - if (err.message === 'Missing Configuration.') { if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration for the dateRange at the dataSet of the product was not found.') } if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> maxDate will be set to current datetime.') } @@ -108,62 +90,40 @@ function newDailyFiles () { endDateRange = new Date() break } - /* If none of the previous conditions are met, the we return the err to the caller. */ - callBackFunction(err) return } - default: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Received Unexpected Response.') } callBackFunction(err) return } } - /* Now we will get the daily files */ - for (i = 0; i < dailyFilePeriods.length; i++) { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> periodTime = ' + periodTime) } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> periodName = ' + periodName) } - if (pSet.validPeriods.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.initialize(fileCloud, pDevTeam, pBot, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) - function onInitialized (err) { try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Entering function.') } - switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received OK Response.') } break } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received FAIL Response.') } callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) return } default: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> Received Unexpected Response.') } callBackWhenFileReceived(err) return } } - fileCursors.set(periodTime, fileCursor) - - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> expectedFiles = ' + expectedFiles) } - expectedFiles = expectedFiles + fileCursor.getExpectedFiles() - - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> onInitialized -> expectedFiles = ' + expectedFiles) } } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) @@ -171,20 +131,14 @@ function newDailyFiles () { } } } - loadThemAll() function loadThemAll () { try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> loadThemAll -> Entering function.') } - for (i = 0; i < dailyFilePeriods.length; i++) { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> loadThemAll -> periodTime = ' + periodTime) } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> loadThemAll -> periodName = ' + periodName) } - if (pSet.validPeriods.includes(periodName) === true) { let fileCursor = fileCursors.get(periodTime) fileCursor.reload(onFileReceived) @@ -208,39 +162,24 @@ function newDailyFiles () { function onFileReceived (err) { try { - if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> Entering function.') } - switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> Received OK Response.') } break } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> Received FAIL Response.') } callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) return } - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> Received CUSTOM FAIL Response.') } - if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> err.message = ' + err.message) } - callBackWhenFileReceived(err) return } - default: { - if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> Received Unexpected Response.') } callBackWhenFileReceived(err) return } } - filesLoaded++ - - if (INFO_LOG === true) { logger.write('[INFO] onFileReceived -> filesLoaded = ' + filesLoaded) } - callBackWhenFileReceived(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) // Note that the call back is called for every file loaded at each cursor. } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] onFileReceived -> err = ' + err.stack) } @@ -250,8 +189,6 @@ function newDailyFiles () { function getFileCursor (pPeriod) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFileCursor -> Entering function.') } - return fileCursors.get(pPeriod) } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] getFileCursor -> err = ' + err.stack) } @@ -260,22 +197,13 @@ function newDailyFiles () { function setDatetime (pDatetime) { try { - if (INFO_LOG === true) { logger.write('[INFO] setDatetime -> Entering function.') } - filesLoaded = 0 expectedFiles = 0 - fileCursors.forEach(setDatetimeToEach) function setDatetimeToEach (fileCursor, key, map) { fileCursor.setDatetime(pDatetime) - - if (INFO_LOG === true) { logger.write('[INFO] setDatetime -> expectedFiles = ' + expectedFiles) } - expectedFiles = expectedFiles + fileCursor.getExpectedFiles() - - if (INFO_LOG === true) { logger.write('[INFO] setDatetime -> expectedFiles = ' + expectedFiles) } - fileCursor.reload(onFileReceived) } } catch (err) { @@ -285,22 +213,13 @@ function newDailyFiles () { function setTimePeriod (pTimePeriod, pDatetime) { try { - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> Entering function.') } - filesLoaded = 0 expectedFiles = 0 - fileCursors.forEach(setTimePeriodToEach) function setTimePeriodToEach (fileCursor, key, map) { fileCursor.setTimePeriod(pTimePeriod, pDatetime) - - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> expectedFiles = ' + expectedFiles) } - expectedFiles = expectedFiles + fileCursor.getExpectedFiles() - - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> expectedFiles = ' + expectedFiles) } - fileCursor.reload(onFileReceived) } } catch (err) { @@ -309,16 +228,10 @@ function newDailyFiles () { } function getExpectedFiles () { - if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> expectedFiles = ' + expectedFiles) } - return expectedFiles } function getFilesLoaded () { - if (INFO_LOG === true) { logger.write('[INFO] getFilesLoaded -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFilesLoaded -> filesLoaded = ' + filesLoaded) } - return filesLoaded } } diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 6e1797655..b0496d89a 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -1,7 +1,6 @@  function newMarketFiles () { const MODULE_NAME = 'Market Files' - const INFO_LOG = false const ERROR_LOG = true const logger = newWebDebugLog() logger.fileName = MODULE_NAME @@ -39,8 +38,6 @@ function newMarketFiles () { function finalize () { try { - if (INFO_LOG === true) { logger.write('[INFO] finalize -> Entering function.') } - clearInterval(intervalHandle) filesLoaded = undefined @@ -53,9 +50,6 @@ function newMarketFiles () { function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } - exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { @@ -83,8 +77,6 @@ function newMarketFiles () { function onFileReceived (err, file) { try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onFileReceived -> Entering function.') } - if (err.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { files.set(periodTime, file) filesLoaded++ @@ -93,11 +85,7 @@ function newMarketFiles () { } if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { - if (filesLoaded > filesNotLoaded) { - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) - } else { - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, thisObject) - } + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) } } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) } @@ -117,8 +105,6 @@ function newMarketFiles () { if (finalized === true) { return } let updateFiles = 0 - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Entering function.') } - /* Now we will get the market files */ for (let i = 0; i < marketFilesPeriods.length; i++) { @@ -130,40 +116,10 @@ function newMarketFiles () { function onFileReceived (err, file) { try { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Entering function.') } - - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received OK Response.') } - break - } - - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received FAIL Response.') } - return - } - - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received CUSTOM FAIL Response.') } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> err.message = ' + err.message) } - return - } - - default: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received Unexpected Response.') } - return - } - } - files.set(periodTime, file) updateFiles++ if (updateFiles === marketFilesPeriods.length) { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> All files received. ') } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> devTeam = ' + devTeam.codeName) } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> bot = ' + bot.codeName) } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> thisSet = ' + thisSet.codeName) } - thisObject.eventHandler.raiseEvent('Files Updated', undefined) } } catch (err) { From b2a05da11bfc746311d05802f1692acf21985cee Mon Sep 17 00:00:00 2001 From: Matias Date: Fri, 24 May 2019 20:14:07 -0300 Subject: [PATCH 22/92] Replacing filestorage client to saweb-api --- src/FileCloud.js | 2 +- src/FileStorage.js | 61 +++++++++++++++++++++++++++------------------- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index f979610da..2853541f4 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -137,7 +137,7 @@ function newFileCloud () { containerName = pDevTeam.codeName.toLowerCase() - blobService.getBlobToText(containerName, filePath + '/' + fileName, onFileReceived) + blobService.getBlobToText(containerName, filePath + '/' + fileName, pDevTeam.host, onFileReceived) function onFileReceived (err, text, response) { try { diff --git a/src/FileStorage.js b/src/FileStorage.js index d398a0ffc..a848df296 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -1,4 +1,4 @@ -function newFileStorage () { +function newFileStorage() { const MODULE_NAME = 'File Cloud' const INFO_LOG = false const ERROR_LOG = true @@ -11,37 +11,48 @@ function newFileStorage () { return thisObject - function getBlobToText (pContainerName, pPath, callBackFunction) { + function getBlobToText(container, filePath, host, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } - let xhttp = new XMLHttpRequest() - xhttp.onreadystatechange = function () { - if (this.readyState === 4 && this.status === 200) { - try { - let response = JSON.parse(xhttp.responseText) - callBackFunction(response.err, response.text, '') - } catch (err) { - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. ') } - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> response.text = ' & response.text) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, '', '') - } + let headers + let accessToken = window.localStorage.getItem('access_token') + if(accessToken !== null){ + headers={ + authorization: 'Bearer ' +accessToken } } - let request = { - conatinerName: pContainerName, - path: pPath - } + axios({ + url: host.url + 'graphql', + method: 'post', + data: { + query: ` + query web_FileContent($container: String!, $filePath: String!, $accessKey: String!, $storage: String!){ + web_FileContent(file: { container: $container, filePath: $filePath, accessKey: $accessKey, storage: $storage }) + } + `, + variables: { + container: container, + filePath: filePath, + storage: host.storage, + accessKey: host.accessKey + } + }, + headers: headers + }).then(res => { + if(res.data.errors){ + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> response.text = ' + JSON.stringify(res.data.errors)) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, '', '') + } + let response = res.data.data.web_FileContent + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response, '') + }).catch(error => { + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. ') } + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> response.text = ' + error.message) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, '', '') + }) - request = JSON.stringify(request) - - let path = window.canvasApp.urlPrefix + 'FileService' - - let blob = new Blob([request], { type: 'text/plain' }) - - xhttp.open('POST', path, true) - xhttp.send(blob) } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] getBlobToText -> err = ' + err.stack) } } From 3890ed226e434c067b946dbcfb227511c7bf5434 Mon Sep 17 00:00:00 2001 From: Matias Date: Tue, 18 Jun 2019 18:59:52 -0300 Subject: [PATCH 23/92] Node update --- src/FileCloud.js | 2 +- src/FileStorage.js | 37 ++++++++++++++++++++++--------------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index 2853541f4..e7330ff7b 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -148,7 +148,7 @@ function newFileCloud () { let data if (err && err.result !== GLOBAL.DEFAULT_OK_RESPONSE.result) { - if (err.code === 'BlobNotFound' | err.code === 'FileNotFound' | err.code === 'ParentNotFound') { + if (err.code === 'BlobNotFound' | err.code === 'FileNotFound' | err.code === 'ParentNotFound' | err.code === 'The specified key does not exist.') { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, message: 'File does not exist.' diff --git a/src/FileStorage.js b/src/FileStorage.js index a848df296..dec3f07fe 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -17,9 +17,9 @@ function newFileStorage() { let headers let accessToken = window.localStorage.getItem('access_token') - if(accessToken !== null){ - headers={ - authorization: 'Bearer ' +accessToken + if (accessToken !== null) { + headers = { + authorization: 'Bearer ' + accessToken } } @@ -28,31 +28,38 @@ function newFileStorage() { method: 'post', data: { query: ` - query web_FileContent($container: String!, $filePath: String!, $accessKey: String!, $storage: String!){ - web_FileContent(file: { container: $container, filePath: $filePath, accessKey: $accessKey, storage: $storage }) + query web_FileContent($file: web_FileInput){ + web_FileContent(file: $file) } `, variables: { - container: container, - filePath: filePath, - storage: host.storage, - accessKey: host.accessKey + file: { + container: container, + filePath: filePath, + storage: host.storage, + accessKey: host.accessKey + } } }, headers: headers }).then(res => { - if(res.data.errors){ - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> response.text = ' + JSON.stringify(res.data.errors)) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, '', '') + if (res.data.errors) { + let error = { + code: res.data.errors[0] + } + callBackFunction(error) + return } let response = res.data.data.web_FileContent - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response, '') + if (response) + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response) + else + callBackFunction(GLOBAL.CUSTOM_FAIL_RESPONSE) }).catch(error => { if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. ') } if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> response.text = ' + error.message) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, '', '') + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) }) - } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] getBlobToText -> err = ' + err.stack) } } From cf7f5f398601688c3868ae8755a9889c7d40f769 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 26 Jun 2019 13:03:09 +0200 Subject: [PATCH 24/92] removing time part of data range dates --- src/FileCursor.js | 71 ++++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index febf86343..7f4414972 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -1,5 +1,5 @@ -  -function newFileCursor () { + +function newFileCursor() { const MODULE_NAME = 'File Cursor' const INFO_LOG = false const ERROR_LOG = true @@ -37,7 +37,7 @@ function newFileCursor () { return thisObject - function finalize () { + function finalize() { try { if (INFO_LOG === true) { logger.write('[INFO] finalize -> Entering function.') } @@ -54,7 +54,7 @@ function newFileCursor () { } } - function initialize (pFileCloud, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize(pFileCloud, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.codeName + '-' + pPeriodName) } @@ -88,25 +88,25 @@ function newFileCursor () { } } - function updateFiles () { + function updateFiles() { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Entering function.') } - /* + /* - In order to know if we need to update any file we follow this logic: + In order to know if we need to update any file we follow this logic: - 1. If the current date is already a file in the cursor, we need to update it. - 2. If the current date is one day ahead of a file in the cursor, we need to add this one to the cursor. + 1. If the current date is already a file in the cursor, we need to update it. + 2. If the current date is one day ahead of a file in the cursor, we need to add this one to the cursor. - */ + */ let targetDate let dateString let file - /* Situation 1 */ + /* Situation 1 */ targetDate = new Date() dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) @@ -121,7 +121,7 @@ function newFileCursor () { return } - /* Situation 2 */ + /* Situation 2 */ targetDate = new Date((new Date()).valueOf() - ONE_DAY_IN_MILISECONDS) dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) @@ -139,7 +139,7 @@ function newFileCursor () { return } - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Entering function.') } @@ -178,21 +178,21 @@ function newFileCursor () { } } - function setTimePeriod (pTimePeriod, pDatetime) { + function setTimePeriod(pTimePeriod, pDatetime) { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> Entering function.') } - /* + /* - We are implementing here an algorithm designed to save bandwidth, memory and processing power at the browser. - We say there is a saving mode where the cursor is running at a minimum size. When the end user aproaches the time period the cursor - is set, then it should exit the saving mode and go to its actual size. + We are implementing here an algorithm designed to save bandwidth, memory and processing power at the browser. + We say there is a saving mode where the cursor is running at a minimum size. When the end user aproaches the time period the cursor + is set, then it should exit the saving mode and go to its actual size. - To do this we are going to measure the distance from the Time Period received to the one the cursors was initialized with. - If these periods are consecutive, it means that the cursor should exit saving mode and load its full size. + To do this we are going to measure the distance from the Time Period received to the one the cursors was initialized with. + If these periods are consecutive, it means that the cursor should exit saving mode and load its full size. - */ + */ cursorDate = removeTime(pDatetime) // Adjust the cursor date to the one received. @@ -224,7 +224,7 @@ function newFileCursor () { enterSavingMode() } - function enterSavingMode () { + function enterSavingMode() { try { if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> enterSavingMode -> Entering function.') } @@ -303,7 +303,7 @@ function newFileCursor () { } } - function exitSavingMode () { + function exitSavingMode() { try { if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> exitSavingMode -> Entering function.') } @@ -386,7 +386,7 @@ function newFileCursor () { } } - function setDatetime (pDatetime) { + function setDatetime(pDatetime) { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] setDatetime -> Entering function.') } @@ -402,7 +402,7 @@ function newFileCursor () { } } - function reload (callBackFunction) { + function reload(callBackFunction) { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] reload -> Entering function.') } @@ -416,7 +416,7 @@ function newFileCursor () { } } - function getFiles (callBackFunction) { + function getFiles(callBackFunction) { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] getFiles -> Entering function.') } @@ -428,14 +428,14 @@ function newFileCursor () { getNextFile() - function getNextFile () { + function getNextFile() { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> Entering function.') } let targetDate = new Date(cursorDate) targetDate.setUTCDate(targetDate.getUTCDate() + j) - /* Small algorith to allow load first the current date, then alternate between the most forwad and the most backwards ones. */ + /* Small algorith to allow load first the current date, then alternate between the most forwad and the most backwards ones. */ if (j === 0) { j++ } else { if (j < 0) { j = -j @@ -448,7 +448,7 @@ function newFileCursor () { let compareTargetDate = new Date(targetDate.valueOf()) if (beginDateRange !== undefined) { - let compareBeginDate = beginDateRange + let compareBeginDate = removeTime(beginDateRange) if (compareTargetDate.valueOf() < compareBeginDate) { controlLoop() @@ -457,7 +457,7 @@ function newFileCursor () { } if (endDateRange !== undefined) { - let compareEndDate = endDateRange + let compareEndDate = removeTime(endDateRange) if (compareTargetDate.valueOf() > compareEndDate) { controlLoop() @@ -474,7 +474,7 @@ function newFileCursor () { controlLoop() } else { if (thisObject.files.get(dateString) === undefined) { - // We dont reload files we already have. + // We dont reload files we already have. fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) } else { @@ -487,7 +487,7 @@ function newFileCursor () { } } - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> onFileReceived -> Entering function.') } @@ -532,7 +532,7 @@ function newFileCursor () { } } - function controlLoop () { + function controlLoop() { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> controlLoop -> Entering function.') } @@ -556,7 +556,7 @@ function newFileCursor () { } } - function collectGarbage (callBackFunction) { + function collectGarbage(callBackFunction) { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] collectGarbage -> Entering function.') } @@ -579,10 +579,11 @@ function newFileCursor () { } } - function getExpectedFiles () { + function getExpectedFiles() { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } return minCursorSize } } + From 2269a7b800c055d8e99e8a6df7a397570b57039f Mon Sep 17 00:00:00 2001 From: Matias Date: Mon, 1 Jul 2019 16:32:43 -0300 Subject: [PATCH 25/92] File does not exist issue --- src/FileStorage.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/FileStorage.js b/src/FileStorage.js index dec3f07fe..1afbcb8a9 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -44,9 +44,7 @@ function newFileStorage() { headers: headers }).then(res => { if (res.data.errors) { - let error = { - code: res.data.errors[0] - } + let error = { code: res.data.errors[0] } callBackFunction(error) return } @@ -56,9 +54,14 @@ function newFileStorage() { else callBackFunction(GLOBAL.CUSTOM_FAIL_RESPONSE) }).catch(error => { - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. ') } - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> response.text = ' + error.message) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + if (error.message === 'Request aborted' || error.message === 'Network Error') { + let error = { code: 'The specified key does not exist.' } + callBackFunction(error) + } else { + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. ') } + if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> response.text = ' + error.message) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + } }) } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] getBlobToText -> err = ' + err.stack) } From 0fbe8e4077258d9fe1d074585af9274c2391fba7 Mon Sep 17 00:00:00 2001 From: Matias Date: Sun, 7 Jul 2019 21:02:39 -0300 Subject: [PATCH 26/92] File does not exist fix --- src/FileStorage.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/FileStorage.js b/src/FileStorage.js index 1afbcb8a9..c14f50da7 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -49,10 +49,12 @@ function newFileStorage() { return } let response = res.data.data.web_FileContent - if (response) + if (response){ callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response) - else - callBackFunction(GLOBAL.CUSTOM_FAIL_RESPONSE) + } else { + let error = { code: 'The specified key does not exist.' } + callBackFunction(error) + } }).catch(error => { if (error.message === 'Request aborted' || error.message === 'Network Error') { let error = { code: 'The specified key does not exist.' } From c95dd3bc63f066fc95e631b2214db4dbbd305ac6 Mon Sep 17 00:00:00 2001 From: Matias Date: Sun, 7 Jul 2019 21:02:39 -0300 Subject: [PATCH 27/92] File does not exist fix --- src/FileStorage.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/FileStorage.js b/src/FileStorage.js index 1afbcb8a9..c14f50da7 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -49,10 +49,12 @@ function newFileStorage() { return } let response = res.data.data.web_FileContent - if (response) + if (response){ callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response) - else - callBackFunction(GLOBAL.CUSTOM_FAIL_RESPONSE) + } else { + let error = { code: 'The specified key does not exist.' } + callBackFunction(error) + } }).catch(error => { if (error.message === 'Request aborted' || error.message === 'Network Error') { let error = { code: 'The specified key does not exist.' } From 1bd581ffbd1a0ee5f6593b99488b705926c6037d Mon Sep 17 00:00:00 2001 From: Matias Date: Wed, 17 Jul 2019 08:36:26 -0300 Subject: [PATCH 28/92] Improving get file --- src/FileStorage.js | 60 ++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/src/FileStorage.js b/src/FileStorage.js index c14f50da7..dca34455c 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -1,17 +1,19 @@ function newFileStorage() { const MODULE_NAME = 'File Cloud' const INFO_LOG = false - const ERROR_LOG = true const logger = newWebDebugLog() logger.fileName = MODULE_NAME + const MAX_RETRY = 2 + let currentRetry = 0 + let thisObject = { getBlobToText: getBlobToText } return thisObject - function getBlobToText(container, filePath, host, callBackFunction) { + async function getBlobToText(container, filePath, host, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } @@ -23,7 +25,7 @@ function newFileStorage() { } } - axios({ + let response = await axios({ url: host.url + 'graphql', method: 'post', data: { @@ -34,39 +36,41 @@ function newFileStorage() { `, variables: { file: { - container: container, - filePath: filePath, + container: container.toLowerCase(), + filePath, storage: host.storage, accessKey: host.accessKey } } }, headers: headers - }).then(res => { - if (res.data.errors) { - let error = { code: res.data.errors[0] } - callBackFunction(error) - return - } - let response = res.data.data.web_FileContent - if (response){ - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response) - } else { - let error = { code: 'The specified key does not exist.' } - callBackFunction(error) - } - }).catch(error => { - if (error.message === 'Request aborted' || error.message === 'Network Error') { - let error = { code: 'The specified key does not exist.' } - callBackFunction(error) - } else { - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. ') } - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + ' : ' + '[ERROR] AppPreLoader -> getBlobToText -> response.text = ' + error.message) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) - } }) + + currentRetry = 0 + if (response.data.errors) { + let error = { code: response.data.errors[0] } + callBackFunction(error) + return + } + + if (response.data.data.web_FileContent) { + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response.data.data.web_FileContent) + } else { + callBackFunction({ code: 'The specified key does not exist.' }) + } + } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] getBlobToText -> err = ' + err.stack) } + if ((err.code === 'ETIMEDOUT' || err.code === 'ECONNRESET') && currentRetry < MAX_RETRY) { + currentRetry++ + if (INFO_LOG === true) { console.log('[INFO] getTextFile -> Retrying connection to the server because received error: ' + err.code + '. Retry #: ' + currentRetry) } + getBlobToText(container, filePath, host, callBackFunction) + }else if (err.message === 'Request aborted') { + let err = { code: 'The specified key does not exist.' } + callBackFunction(err) + } else { + console.log('Error geting the file from the serever:', err) + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + } } } } From b5e3918f66032e90e683a7ebbc8bfafd60df5dc7 Mon Sep 17 00:00:00 2001 From: Matias Date: Sat, 20 Jul 2019 07:28:05 -0300 Subject: [PATCH 29/92] Adding recoverable errors --- src/FileStorage.js | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/FileStorage.js b/src/FileStorage.js index dca34455c..e0616a5d0 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -7,6 +7,22 @@ function newFileStorage() { const MAX_RETRY = 2 let currentRetry = 0 + const recoverableErrors = [ + 'SOCKETTIMEDOUT', + 'TIMEDOUT', + 'CONNRESET', + 'CONNREFUSED', + 'NOTFOUND', + 'ENOTFOUND', + 'ECONNREFUSED', + 'CONNREFUSED', + 'NOTFOUND', + 'ESOCKETTIMEDOUT', + 'ECONNRESET', + 'ETIMEDOUT', + 'EAI_AGAIN' + ] + let thisObject = { getBlobToText: getBlobToText } @@ -46,7 +62,6 @@ function newFileStorage() { headers: headers }) - currentRetry = 0 if (response.data.errors) { let error = { code: response.data.errors[0] } callBackFunction(error) @@ -60,7 +75,7 @@ function newFileStorage() { } } catch (err) { - if ((err.code === 'ETIMEDOUT' || err.code === 'ECONNRESET') && currentRetry < MAX_RETRY) { + if (verifyRetry(err.code) && currentRetry < MAX_RETRY) { currentRetry++ if (INFO_LOG === true) { console.log('[INFO] getTextFile -> Retrying connection to the server because received error: ' + err.code + '. Retry #: ' + currentRetry) } getBlobToText(container, filePath, host, callBackFunction) @@ -73,4 +88,14 @@ function newFileStorage() { } } } + + function verifyRetry(errorCode) { + for (let i = 0; i < recoverableErrors.length; i++) { + const error = recoverableErrors[i]; + if (error === errorCode) { + return true + } + } + return false + } } From 118c8b1ddfa78a2d23c93c2c32807f71392efdbe Mon Sep 17 00:00:00 2001 From: Matias Date: Sat, 27 Jul 2019 09:11:06 -0300 Subject: [PATCH 30/92] adding local file support --- src/FileStorage.js | 72 ++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/src/FileStorage.js b/src/FileStorage.js index e0616a5d0..e19f33a90 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -33,45 +33,55 @@ function newFileStorage() { try { if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } - let headers - let accessToken = window.localStorage.getItem('access_token') - if (accessToken !== null) { - headers = { - authorization: 'Bearer ' + accessToken + if (host.url.indexOf('localhost') !== -1) { + callServer(undefined, 'Storage/' + container + '/' + filePath, (response, fileContent) => { + if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) + } else { + callBackFunction(response) + } + }) + } else { + let headers + let accessToken = window.localStorage.getItem('access_token') + if (accessToken !== null) { + headers = { + authorization: 'Bearer ' + accessToken + } } - } - let response = await axios({ - url: host.url + 'graphql', - method: 'post', - data: { - query: ` + let response = await axios({ + url: host.url + 'graphql', + method: 'post', + data: { + query: ` query web_FileContent($file: web_FileInput){ web_FileContent(file: $file) } `, - variables: { - file: { - container: container.toLowerCase(), - filePath, - storage: host.storage, - accessKey: host.accessKey + variables: { + file: { + container: container.toLowerCase(), + filePath, + storage: host.storage, + accessKey: host.accessKey + } } - } - }, - headers: headers - }) + }, + headers: headers + }) - if (response.data.errors) { - let error = { code: response.data.errors[0] } - callBackFunction(error) - return - } + if (response.data.errors) { + let error = { code: response.data.errors[0] } + callBackFunction(error) + return + } - if (response.data.data.web_FileContent) { - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response.data.data.web_FileContent) - } else { - callBackFunction({ code: 'The specified key does not exist.' }) + if (response.data.data.web_FileContent) { + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response.data.data.web_FileContent) + } else { + callBackFunction({ code: 'The specified key does not exist.' }) + } } } catch (err) { @@ -79,7 +89,7 @@ function newFileStorage() { currentRetry++ if (INFO_LOG === true) { console.log('[INFO] getTextFile -> Retrying connection to the server because received error: ' + err.code + '. Retry #: ' + currentRetry) } getBlobToText(container, filePath, host, callBackFunction) - }else if (err.message === 'Request aborted') { + } else if (err.message === 'Request aborted') { let err = { code: 'The specified key does not exist.' } callBackFunction(err) } else { From 53debdbe7f554350c3bbcfaf6ef5f8c184facc71 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 17 Aug 2019 15:47:43 +0200 Subject: [PATCH 31/92] Some Cleaning and UPDATES of Sequence Files --- src/FileCursor.js | 41 +++++++++-------------- src/FileSequence.js | 82 +++++---------------------------------------- src/MarketFiles.js | 6 ++-- 3 files changed, 28 insertions(+), 101 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index 7f4414972..6b9f0c487 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -1,5 +1,5 @@ -function newFileCursor() { +function newFileCursor () { const MODULE_NAME = 'File Cursor' const INFO_LOG = false const ERROR_LOG = true @@ -37,7 +37,7 @@ function newFileCursor() { return thisObject - function finalize() { + function finalize () { try { if (INFO_LOG === true) { logger.write('[INFO] finalize -> Entering function.') } @@ -54,7 +54,7 @@ function newFileCursor() { } } - function initialize(pFileCloud, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize (pFileCloud, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.codeName + '-' + pPeriodName) } @@ -88,7 +88,7 @@ function newFileCursor() { } } - function updateFiles() { + function updateFiles () { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Entering function.') } @@ -114,8 +114,6 @@ function newFileCursor() { file = thisObject.files.get(dateString) if (file !== undefined) { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> File found at the cursor, proceeding to update.') } - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) return @@ -132,31 +130,25 @@ function newFileCursor() { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) if (file !== undefined) { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Previous day file found at the cursor, proceeding to add current date.') } - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) return } - function onFileReceived(err, file) { + function onFileReceived (err, file) { try { if (finalized === true) { return } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Entering function.') } switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received OK Response.') } break } case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received FAIL Response.') } return } default: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> Received Unexpected Response.') } return } } @@ -178,10 +170,9 @@ function newFileCursor() { } } - function setTimePeriod(pTimePeriod, pDatetime) { + function setTimePeriod (pTimePeriod, pDatetime) { try { if (finalized === true) { return } - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> Entering function.') } /* @@ -224,7 +215,7 @@ function newFileCursor() { enterSavingMode() } - function enterSavingMode() { + function enterSavingMode () { try { if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> enterSavingMode -> Entering function.') } @@ -303,7 +294,7 @@ function newFileCursor() { } } - function exitSavingMode() { + function exitSavingMode () { try { if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> exitSavingMode -> Entering function.') } @@ -386,7 +377,7 @@ function newFileCursor() { } } - function setDatetime(pDatetime) { + function setDatetime (pDatetime) { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] setDatetime -> Entering function.') } @@ -402,7 +393,7 @@ function newFileCursor() { } } - function reload(callBackFunction) { + function reload (callBackFunction) { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] reload -> Entering function.') } @@ -416,7 +407,7 @@ function newFileCursor() { } } - function getFiles(callBackFunction) { + function getFiles (callBackFunction) { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] getFiles -> Entering function.') } @@ -428,7 +419,7 @@ function newFileCursor() { getNextFile() - function getNextFile() { + function getNextFile () { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> Entering function.') } @@ -487,7 +478,7 @@ function newFileCursor() { } } - function onFileReceived(err, file) { + function onFileReceived (err, file) { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> onFileReceived -> Entering function.') } @@ -532,7 +523,7 @@ function newFileCursor() { } } - function controlLoop() { + function controlLoop () { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> controlLoop -> Entering function.') } @@ -556,7 +547,7 @@ function newFileCursor() { } } - function collectGarbage(callBackFunction) { + function collectGarbage (callBackFunction) { try { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] collectGarbage -> Entering function.') } @@ -579,7 +570,7 @@ function newFileCursor() { } } - function getExpectedFiles() { + function getExpectedFiles () { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } diff --git a/src/FileSequence.js b/src/FileSequence.js index a78a18625..24ebd4033 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -35,12 +35,6 @@ function newFileSequence () { function finalize () { try { - if (INFO_LOG === true) { logger.write('[INFO] finalize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] finalize -> devTeam = ' + devTeam.codeName) } - if (INFO_LOG === true) { logger.write('[INFO] finalize -> bot = ' + bot.codeName) } - if (INFO_LOG === true) { logger.write('[INFO] finalize -> thisSet = ' + thisSet.codeName) } - if (INFO_LOG === true) { logger.write('[INFO] finalize -> intervalHandle = ' + intervalHandle) } - clearInterval(intervalHandle) filesLoaded = undefined @@ -55,18 +49,13 @@ function newFileSequence () { function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } - exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } - intervalHandle = setInterval(updateFiles(), _1_MINUTE_IN_MILISECONDS) - - if (INFO_LOG === true) { logger.write('[INFO] initialize -> intervalHandle = ' + intervalHandle) } + intervalHandle = setInterval(updateFiles, _1_MINUTE_IN_MILISECONDS) market = pMarket devTeam = pDevTeam @@ -83,41 +72,33 @@ function newFileSequence () { function onSequenceFileReceived (err, file) { try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> key = ' + devTeam.codeName + '-' + bot.codeName + '-' + product.codeName) } - initialized = true switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Received OK Response.') } break } case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Received FAIL Response.') } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) return } case GLOBAL.CUSTOM_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Received CUSTOM OK Response.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> err.message = ' + err.message) } + if (ERROR_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> err.message = ' + err.message) } callBackFunction(err) return } case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Received CUSTOM FAIL Response.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> err.message = ' + err.message) } + if (ERROR_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> err.message = ' + err.message) } callBackFunction(err, thisObject) return } default: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> Received Unexpected Response.') } callBackFunction(err) return } @@ -125,8 +106,6 @@ function newFileSequence () { maxSequence = Number(file) - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> maxSequence = ' + maxSequence) } - /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { @@ -136,26 +115,21 @@ function newFileSequence () { try { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received OK Response.') } break } case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received FAIL Response.') } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) return } case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received CUSTOM FAIL Response.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> err.message = ' + err.message) } - + if (ERROR_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> err.message = ' + err.message) } callBackFunction(err) return } default: { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received Unexpected Response.') } callBackFunction(err) return } @@ -187,9 +161,7 @@ function newFileSequence () { try { if (finalized === true || initialized === false) { return } - let updateFiles = 0 - - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Entering function.') } + logger.write('[INFO] updateFiles -> Entering function.') /* @@ -203,100 +175,71 @@ function newFileSequence () { let currentMaxSequence = maxSequence - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> currentMaxSequence = ' + currentMaxSequence) } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> intervalHandle = ' + intervalHandle) } - - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> devTeam = ' + devTeam.codeName) } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> bot = ' + bot.codeName) } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> thisSet = ' + thisSet.codeName) } - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> key = ' + devTeam.codeName + '-' + bot.codeName + '-' + product.codeName) } - switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received OK Response.') } break } case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received FAIL Response.') } return } case GLOBAL.CUSTOM_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received CUSTOM OK Response.') } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> err.message = ' + err.message) } + if (ERROR_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> err.message = ' + err.message) } return } case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received CUSTOM FAIL Response.') } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> err.message = ' + err.message) } + if (ERROR_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> err.message = ' + err.message) } return } default: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received Unexpected Response.') } + if (ERROR_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received Unexpected Response.') } return } } maxSequence = Number(file) - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> maxSequence = ' + maxSequence) } - /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> i = ' + i) } - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received OK Response.') } break } case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received FAIL Response.') } return } case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received CUSTOM FAIL Response.') } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> err.message = ' + err.message) } + if (ERROR_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> err.message = ' + err.message) } return } default: { - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received Unexpected Response.') } return } } files.set(i, file) - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> File Updated.') } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> i = ' + i) } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> file.length = ' + file.length) } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> filesLoaded = ' + filesLoaded) } - if (i !== currentMaxSequence) { filesLoaded++ - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> filesLoaded = ' + filesLoaded) } } if (filesLoaded === maxSequence + 1) { thisObject.eventHandler.raiseEvent('Files Updated', undefined) - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Files Updated event Raised.') } } } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onSequenceFileReceived -> onFileReceived -> err = ' + err.stack) } @@ -313,21 +256,14 @@ function newFileSequence () { } function getFile (pSequence) { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> pSequence = ' + pSequence) } - return files.get(pSequence) } function getExpectedFiles () { - if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } - return maxSequence + 1 } function getFilesLoaded () { - if (INFO_LOG === true) { logger.write('[INFO] getFilesLoaded -> Entering function.') } - return filesLoaded } } diff --git a/src/MarketFiles.js b/src/MarketFiles.js index b0496d89a..a682c491c 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -103,7 +103,7 @@ function newMarketFiles () { function updateFiles () { try { if (finalized === true) { return } - let updateFiles = 0 + let updatedFiles = 0 /* Now we will get the market files */ @@ -117,9 +117,9 @@ function newMarketFiles () { function onFileReceived (err, file) { try { files.set(periodTime, file) - updateFiles++ + updatedFiles++ - if (updateFiles === marketFilesPeriods.length) { + if (updatedFiles === marketFilesPeriods.length) { thisObject.eventHandler.raiseEvent('Files Updated', undefined) } } catch (err) { From 228529a276378b01b8a0a825c527f54a85930bcc Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 22 Sep 2019 11:29:40 +0200 Subject: [PATCH 32/92] Daily Files Updated Events now Handled --- src/DailyFiles.js | 10 +++++++--- src/FileCursor.js | 10 ++++------ src/MarketFiles.js | 5 ++++- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index fe6eefe52..2bbda5c6a 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -6,6 +6,7 @@ function newDailyFiles () { logger.fileName = MODULE_NAME let thisObject = { + eventHandler: undefined, getFileCursor: getFileCursor, setDatetime: setDatetime, setTimePeriod: setTimePeriod, @@ -17,17 +18,19 @@ function newDailyFiles () { let filesLoaded = 0 let expectedFiles = 0 - let fileCloud - let fileCursors = new Map() - let callBackWhenFileReceived + thisObject.eventHandler = newEventHandler() + return thisObject function finalize () { try { + thisObject.eventHandler.finalize() + thisObject.eventHandler = undefined + filesLoaded = undefined expectedFiles = undefined fileCloud = undefined @@ -106,6 +109,7 @@ function newDailyFiles () { if (pSet.validPeriods.includes(periodName) === true) { let fileCursor = newFileCursor() + fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s fileCursor.initialize(fileCloud, pDevTeam, pBot, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) function onInitialized (err) { try { diff --git a/src/FileCursor.js b/src/FileCursor.js index 6b9f0c487..caa066abf 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -9,6 +9,7 @@ function newFileCursor () { let cursorDate let thisObject = { + eventHandler: undefined, // received from parent reload: reload, setDatetime: setDatetime, setTimePeriod: setTimePeriod, @@ -41,6 +42,8 @@ function newFileCursor () { try { if (INFO_LOG === true) { logger.write('[INFO] finalize -> Entering function.') } + thisObject.eventHandler = undefined + clearInterval(intervalHandle) thisObject.files = undefined @@ -154,12 +157,7 @@ function newFileCursor () { } thisObject.files.set(dateString, file) - - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> File updated.') } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> devTeam = ' + devTeam.codeName) } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> bot = ' + bot.codeName) } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> thisSet = ' + thisSet.codeName) } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> onFileReceived -> dateString = ' + dateString) } + thisObject.eventHandler.raiseEvent('Files Updated') } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onFileReceived -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index a682c491c..e574e3893 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -43,6 +43,9 @@ function newMarketFiles () { filesLoaded = undefined files = undefined finalized = true + + thisObject.eventHandler.finalize() + thisObject.eventHandler = undefined } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err.stack) } } @@ -120,7 +123,7 @@ function newMarketFiles () { updatedFiles++ if (updatedFiles === marketFilesPeriods.length) { - thisObject.eventHandler.raiseEvent('Files Updated', undefined) + thisObject.eventHandler.raiseEvent('Files Updated') } } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onFileReceived -> err = ' + err.stack) } From a7a6b4c7f9af1e8d7ed31ff9d06bd9d231b37e54 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 22 Sep 2019 13:36:52 +0200 Subject: [PATCH 33/92] Unnecesary INFO logs removed --- src/FileCursor.js | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index caa066abf..54725ba3e 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -40,8 +40,6 @@ function newFileCursor () { function finalize () { try { - if (INFO_LOG === true) { logger.write('[INFO] finalize -> Entering function.') } - thisObject.eventHandler = undefined clearInterval(intervalHandle) @@ -59,9 +57,6 @@ function newFileCursor () { function initialize (pFileCloud, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.codeName + '-' + pPeriodName) } - market = pMarket exchange = pExchange fileCloud = pFileCloud @@ -74,12 +69,6 @@ function newFileCursor () { beginDateRange = pBeginDateRange endDateRange = pEndDateRange - if (INFO_LOG === true) { logger.write('[INFO] initialize -> market = ' + market) } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> periodName = ' + periodName) } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> cursorDate = ' + cursorDate) } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> beginDateRange = ' + beginDateRange) } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> endDateRange = ' + endDateRange) } - setTimePeriod(pCurrentTimePeriod, pCursorDate) intervalHandle = setInterval(updateFiles, timePeriod) @@ -94,7 +83,6 @@ function newFileCursor () { function updateFiles () { try { if (finalized === true) { return } - if (INFO_LOG === true) { logger.write('[INFO] updateFiles -> Entering function.') } /* @@ -200,23 +188,13 @@ function newFileCursor () { } if (Math.abs(positionB - positionA) <= 1) { - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> Will EXIT Saving Mode.') } - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> pTimePeriod = ' + convertTimePeriodToName(pTimePeriod)) } - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> timePeriod = ' + convertTimePeriodToName(timePeriod)) } - exitSavingMode() } else { - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> Will ENTER Saving Mode.') } - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> pTimePeriod = ' + convertTimePeriodToName(pTimePeriod)) } - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> timePeriod = ' + convertTimePeriodToName(timePeriod)) } - enterSavingMode() } function enterSavingMode () { try { - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> enterSavingMode -> Entering function.') } - switch (timePeriod) { case _45_MINUTES_IN_MILISECONDS: @@ -294,8 +272,6 @@ function newFileCursor () { function exitSavingMode () { try { - if (INFO_LOG === true) { logger.write('[INFO] setTimePeriod -> exitSavingMode -> Entering function.') } - switch (timePeriod) { case _45_MINUTES_IN_MILISECONDS: @@ -378,7 +354,6 @@ function newFileCursor () { function setDatetime (pDatetime) { try { if (finalized === true) { return } - if (INFO_LOG === true) { logger.write('[INFO] setDatetime -> Entering function.') } if (pDatetime === undefined) { if (ERROR_LOG === true) { logger.write('[ERROR] setDatetime -> Received undefined datetime.') } @@ -394,7 +369,6 @@ function newFileCursor () { function reload (callBackFunction) { try { if (finalized === true) { return } - if (INFO_LOG === true) { logger.write('[INFO] reload -> Entering function.') } getFiles(callBackFunction) @@ -408,7 +382,6 @@ function newFileCursor () { function getFiles (callBackFunction) { try { if (finalized === true) { return } - if (INFO_LOG === true) { logger.write('[INFO] getFiles -> Entering function.') } let i = 0 let j = 0 @@ -419,8 +392,6 @@ function newFileCursor () { function getNextFile () { try { - if (INFO_LOG === true) { logger.write('[INFO] getFiles -> getNextFile -> Entering function.') } - let targetDate = new Date(cursorDate) targetDate.setUTCDate(targetDate.getUTCDate() + j) @@ -548,7 +519,6 @@ function newFileCursor () { function collectGarbage (callBackFunction) { try { if (finalized === true) { return } - if (INFO_LOG === true) { logger.write('[INFO] collectGarbage -> Entering function.') } date = removeTime(cursorDate) @@ -575,4 +545,3 @@ function newFileCursor () { return minCursorSize } } - From 53106e88f46322db1d401317f74ed666f596851f Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 22 Sep 2019 15:32:47 +0200 Subject: [PATCH 34/92] Removed Updating files base on setInterval and now is based on events when the status report is saved. --- src/DailyFiles.js | 2 +- src/FileCursor.js | 29 +++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 2bbda5c6a..14d77eeac 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -110,7 +110,7 @@ function newDailyFiles () { if (pSet.validPeriods.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDevTeam, pBot, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) + fileCursor.initialize(fileCloud, pDevTeam, pBot, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) function onInitialized (err) { try { switch (err.result) { diff --git a/src/FileCursor.js b/src/FileCursor.js index 54725ba3e..19dde9d7e 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -27,22 +27,35 @@ function newFileCursor () { let fileCloud let devTeam let bot + let product let thisSet let periodName let timePeriod let beginDateRange let endDateRange - let intervalHandle let finalized = false + let eventSubscriptionIdDatasetUpdated + return thisObject function finalize () { try { + systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) thisObject.eventHandler = undefined - clearInterval(intervalHandle) + market = undefined + exchange = undefined + fileCloud = undefined + devTeam = undefined + bot = undefined + product = undefined + thisSet = undefined + periodName = undefined + timePeriod = undefined + beginDateRange = undefined + endDateRange = undefined thisObject.files = undefined cursorDate = undefined @@ -55,13 +68,14 @@ function newFileCursor () { } } - function initialize (pFileCloud, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize (pFileCloud, pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { try { market = pMarket exchange = pExchange fileCloud = pFileCloud devTeam = pDevTeam bot = pBot + product = pProduct thisSet = pSet periodName = pPeriodName cursorDate = removeTime(pCursorDate) @@ -69,11 +83,14 @@ function newFileCursor () { beginDateRange = pBeginDateRange endDateRange = pEndDateRange - setTimePeriod(pCurrentTimePeriod, pCursorDate) - - intervalHandle = setInterval(updateFiles, timePeriod) + let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) + + function onResponse (message) { + eventSubscriptionIdDatasetUpdated = message.eventSubscriptionId + } } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) From dbb272222366276f67808726db7c1c29f598176b Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 23 Sep 2019 11:28:59 +0200 Subject: [PATCH 35/92] Updates are now triggered by events at the Datasets --- src/DailyFiles.js | 6 ++--- src/FileCursor.js | 58 ++++++--------------------------------------- src/FileSequence.js | 26 ++++++++++++++++---- src/MarketFiles.js | 34 ++++++++++++++++++++------ 4 files changed, 59 insertions(+), 65 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 14d77eeac..8e58e1e52 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -82,9 +82,9 @@ function newDailyFiles () { case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { if (err.message === 'File does not exist.') { - err.message = 'Dataset Unavailable.' - callBackFunction(err) - return + beginDateRange = new Date() + endDateRange = new Date() + break } if (err.message === 'Missing Configuration.') { if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration for the dateRange at the dataSet of the product was not found.') } diff --git a/src/FileCursor.js b/src/FileCursor.js index 19dde9d7e..e856afedd 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -97,72 +97,28 @@ function newFileCursor () { } } - function updateFiles () { + function updateFiles (message) { try { if (finalized === true) { return } - /* + if (message.event === undefined || message.event.lastFile === undefined) { return } - In order to know if we need to update any file we follow this logic: - - 1. If the current date is already a file in the cursor, we need to update it. - 2. If the current date is one day ahead of a file in the cursor, we need to add this one to the cursor. - - */ - - let targetDate + let targetDate = new Date(message.event.lastFile) let dateString let file - /* Situation 1 */ - - targetDate = new Date() dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - file = thisObject.files.get(dateString) - - if (file !== undefined) { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) - - return - } - - /* Situation 2 */ - - targetDate = new Date((new Date()).valueOf() - ONE_DAY_IN_MILISECONDS) - dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - - file = thisObject.files.get(dateString) // This is from yesterday - - targetDate = new Date() - dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - - if (file !== undefined) { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) - - return - } + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { if (finalized === true) { return } - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - break - } - - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - return - } - - default: { - return - } + if (err.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { + thisObject.files.set(dateString, file) + thisObject.eventHandler.raiseEvent('Files Updated') } - - thisObject.files.set(dateString, file) - thisObject.eventHandler.raiseEvent('Files Updated') } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onFileReceived -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) diff --git a/src/FileSequence.js b/src/FileSequence.js index 24ebd4033..cfca4992d 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -29,18 +29,27 @@ function newFileSequence () { thisObject.eventHandler = newEventHandler() - let intervalHandle + let eventSubscriptionIdDatasetUpdated return thisObject function finalize () { try { - clearInterval(intervalHandle) + systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) + + thisObject.eventHandler.finalize() + thisObject.eventHandler = undefined filesLoaded = undefined fileCloud = undefined files = undefined maxSequence = undefined + market = undefined + devTeam = undefined + bot = undefined + thisSet = undefined + product = undefined + finalized = true } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err.stack) } @@ -55,8 +64,6 @@ function newFileSequence () { throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } - intervalHandle = setInterval(updateFiles, _1_MINUTE_IN_MILISECONDS) - market = pMarket devTeam = pDevTeam bot = pBot @@ -66,12 +73,20 @@ function newFileSequence () { fileCloud = newFileCloud() fileCloud.initialize(bot) + let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key, onResponse, updateFiles) + + function onResponse (message) { + eventSubscriptionIdDatasetUpdated = message.eventSubscriptionId + } + /* First we will get the sequence max number */ fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { + if (finalized === true) { return } initialized = true switch (err.result) { @@ -113,6 +128,7 @@ function newFileSequence () { function onFileReceived (err, file) { try { + if (finalized === true) { return } switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { break @@ -179,6 +195,7 @@ function newFileSequence () { function onSequenceFileReceived (err, file) { try { + if (finalized === true) { return } switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { break @@ -213,6 +230,7 @@ function newFileSequence () { function onFileReceived (err, file) { try { + if (finalized === true) { return } switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { break diff --git a/src/MarketFiles.js b/src/MarketFiles.js index e574e3893..05aa887ba 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -27,6 +27,7 @@ function newMarketFiles () { let devTeam let bot let thisSet + let product let periodName thisObject.eventHandler = newEventHandler() @@ -34,18 +35,29 @@ function newMarketFiles () { let intervalHandle let finalized = false + let eventSubscriptionIdDatasetUpdated + return thisObject function finalize () { try { - clearInterval(intervalHandle) - - filesLoaded = undefined - files = undefined - finalized = true + systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) thisObject.eventHandler.finalize() thisObject.eventHandler = undefined + + filesLoaded = undefined + files = undefined + + market = undefined + exchange = undefined + devTeam = undefined + bot = undefined + thisSet = undefined + product = undefined + periodName = undefined + + finalized = true } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err.stack) } } @@ -59,12 +71,11 @@ function newMarketFiles () { throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } - intervalHandle = setInterval(updateFiles, _10_MINUTES_IN_MILISECONDS) - market = pMarket devTeam = pDevTeam bot = pBot thisSet = pSet + product = pProduct fileCloud = newFileCloud() fileCloud.initialize(pBot) @@ -80,6 +91,7 @@ function newMarketFiles () { function onFileReceived (err, file) { try { + if (finalized === true) { return } if (err.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { files.set(periodTime, file) filesLoaded++ @@ -88,7 +100,14 @@ function newMarketFiles () { } if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { + let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) + + function onResponse (message) { + eventSubscriptionIdDatasetUpdated = message.eventSubscriptionId + } } } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) } @@ -119,6 +138,7 @@ function newMarketFiles () { function onFileReceived (err, file) { try { + if (finalized === true) { return } files.set(periodTime, file) updatedFiles++ From d761dc4a7c9b15f5302e0eb68a6223a083b873e1 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 23 Sep 2019 14:54:22 +0200 Subject: [PATCH 36/92] Sequence Files Initialization Fixed --- src/FileSequence.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/FileSequence.js b/src/FileSequence.js index cfca4992d..9d194024c 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -87,7 +87,6 @@ function newFileSequence () { function onSequenceFileReceived (err, file) { try { if (finalized === true) { return } - initialized = true switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { @@ -101,14 +100,18 @@ function newFileSequence () { case GLOBAL.CUSTOM_OK_RESPONSE.result: { if (ERROR_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> err.message = ' + err.message) } - + initialized = true callBackFunction(err) return } case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (err.message === 'File does not exist.') { // We will assume that the process which generates the files has never been started, that does not imply that we can wait for data to come later. + initialized = true + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) + return + } if (ERROR_LOG === true) { logger.write('[INFO] initialize -> onSequenceFileReceived -> err.message = ' + err.message) } - callBackFunction(err, thisObject) return } @@ -118,7 +121,7 @@ function newFileSequence () { return } } - + initialized = true maxSequence = Number(file) /* Now we will get the sequence of files */ From b5d85a82e09a4c7973effbf7e917aae62cdb6c30 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 23 Sep 2019 17:21:40 +0200 Subject: [PATCH 37/92] FileSequence connected by events --- src/FileSequence.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/FileSequence.js b/src/FileSequence.js index 9d194024c..cfd0cfaf4 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -73,8 +73,10 @@ function newFileSequence () { fileCloud = newFileCloud() fileCloud.initialize(bot) + let callerId = newUniqueId() + let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName - systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key, onResponse, updateFiles) + systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { eventSubscriptionIdDatasetUpdated = message.eventSubscriptionId From 74915e23ce82ae38faf353c8ed202872e74ef9f0 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 23 Sep 2019 18:04:33 +0200 Subject: [PATCH 38/92] File Sequence Fixed --- src/FileSequence.js | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/FileSequence.js b/src/FileSequence.js index cfd0cfaf4..a42bb9e7d 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -30,6 +30,7 @@ function newFileSequence () { thisObject.eventHandler = newEventHandler() let eventSubscriptionIdDatasetUpdated + let callerId return thisObject @@ -73,7 +74,7 @@ function newFileSequence () { fileCloud = newFileCloud() fileCloud.initialize(bot) - let callerId = newUniqueId() + callerId = newUniqueId() let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) @@ -198,7 +199,7 @@ function newFileSequence () { fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) - function onSequenceFileReceived (err, file) { + function onSequenceFileReceived (err, sequenceFile) { try { if (finalized === true) { return } switch (err.result) { @@ -226,7 +227,8 @@ function newFileSequence () { } } - maxSequence = Number(file) + maxSequence = Number(sequenceFile) + filesLoaded = 0 /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ @@ -240,26 +242,18 @@ function newFileSequence () { case GLOBAL.DEFAULT_OK_RESPONSE.result: { break } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { return } - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { if (ERROR_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> err.message = ' + err.message) } return } - - default: { - return - } } files.set(i, file) - if (i !== currentMaxSequence) { - filesLoaded++ - } + filesLoaded++ if (filesLoaded === maxSequence + 1) { thisObject.eventHandler.raiseEvent('Files Updated', undefined) From 3632ace5b833841904e405c78a63b1f668f1baea Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 25 Sep 2019 09:34:46 +0200 Subject: [PATCH 39/92] Better logging --- src/FileSequence.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/FileSequence.js b/src/FileSequence.js index a42bb9e7d..8fffd7c03 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -183,8 +183,6 @@ function newFileSequence () { try { if (finalized === true || initialized === false) { return } - logger.write('[INFO] updateFiles -> Entering function.') - /* To keep this data structure up-to-date we need to: @@ -212,17 +210,17 @@ function newFileSequence () { } case GLOBAL.CUSTOM_OK_RESPONSE.result: { - if (ERROR_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> err.message = ' + err.message) } + if (ERROR_LOG === true) { logger.write('[WARN] updateFiles -> onSequenceFileReceived -> err.message = ' + err.message) } return } case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - if (ERROR_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> err.message = ' + err.message) } + if (ERROR_LOG === true) { logger.write('[WARN] updateFiles -> onSequenceFileReceived -> err.message = ' + err.message) } return } default: { - if (ERROR_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> Received Unexpected Response.') } + if (ERROR_LOG === true) { logger.write('[WARN] updateFiles -> onSequenceFileReceived -> Received Unexpected Response.') } return } } @@ -246,7 +244,7 @@ function newFileSequence () { return } case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { - if (ERROR_LOG === true) { logger.write('[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> err.message = ' + err.message) } + if (ERROR_LOG === true) { logger.write('[WARN] updateFiles -> onSequenceFileReceived -> onFileReceived -> err.message = ' + err.message) } return } } From 99d98b1089380f5e31876c2efb29325560e2ac29 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 7 Oct 2019 14:52:11 +0200 Subject: [PATCH 40/92] Session Concept Introduced --- src/DailyFiles.js | 6 +++--- src/FileCloud.js | 10 +++++++--- src/FileCursor.js | 9 ++++++--- src/FileSequence.js | 13 ++++++++----- src/MarketFiles.js | 9 ++++++--- src/SingleFile.js | 4 ++-- 6 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 8e58e1e52..e91ef8f04 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -45,7 +45,7 @@ function newDailyFiles () { } } - function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { + function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { try { callBackWhenFileReceived = callBackFunction @@ -60,7 +60,7 @@ function newDailyFiles () { /* First we will get the Data Range */ - fileCloud.getFile(pDevTeam, pBot, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) + fileCloud.getFile(pDevTeam, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) function onDataRangeReceived (err, pFile) { try { @@ -110,7 +110,7 @@ function newDailyFiles () { if (pSet.validPeriods.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDevTeam, pBot, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) + fileCursor.initialize(fileCloud, pDevTeam, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) function onInitialized (err) { try { switch (err.result) { diff --git a/src/FileCloud.js b/src/FileCloud.js index e7330ff7b..b853661ab 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -46,15 +46,15 @@ function newFileCloud () { } } - function getFile (pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } const MAX_RETRIES = 3 - getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(0, pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } @@ -112,6 +112,10 @@ function newFileCloud () { } } + if (pSession !== undefined) { + filePath = filePath.replace('@Session', pSession.name.replace(' ', '-')) + } + if (pExchange !== undefined) { filePath = filePath.replace('@Exchange', pExchange.name) } diff --git a/src/FileCursor.js b/src/FileCursor.js index e856afedd..9952a314b 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -27,6 +27,7 @@ function newFileCursor () { let fileCloud let devTeam let bot + let session let product let thisSet let periodName @@ -50,6 +51,7 @@ function newFileCursor () { fileCloud = undefined devTeam = undefined bot = undefined + session = undefined product = undefined thisSet = undefined periodName = undefined @@ -68,13 +70,14 @@ function newFileCursor () { } } - function initialize (pFileCloud, pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize (pFileCloud, pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { try { market = pMarket exchange = pExchange fileCloud = pFileCloud devTeam = pDevTeam bot = pBot + session = pSession product = pProduct thisSet = pSet periodName = pPeriodName @@ -109,7 +112,7 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -409,7 +412,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 8fffd7c03..40fa3eb46 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -22,6 +22,7 @@ function newFileSequence () { let market let devTeam let bot + let session let thisSet let product let finalized = false @@ -48,6 +49,7 @@ function newFileSequence () { market = undefined devTeam = undefined bot = undefined + session = undefined thisSet = undefined product = undefined @@ -57,7 +59,7 @@ function newFileSequence () { } } - function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) @@ -68,6 +70,7 @@ function newFileSequence () { market = pMarket devTeam = pDevTeam bot = pBot + session = pSession thisSet = pSet product = pProduct @@ -85,7 +88,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -130,7 +133,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -195,7 +198,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -231,7 +234,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 05aa887ba..717f0766d 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -26,6 +26,7 @@ function newMarketFiles () { let exchange let devTeam let bot + let session let thisSet let product let periodName @@ -53,6 +54,7 @@ function newMarketFiles () { exchange = undefined devTeam = undefined bot = undefined + session = undefined thisSet = undefined product = undefined periodName = undefined @@ -63,7 +65,7 @@ function newMarketFiles () { } } - function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) @@ -74,6 +76,7 @@ function newMarketFiles () { market = pMarket devTeam = pDevTeam bot = pBot + session = pSession thisSet = pSet product = pProduct @@ -87,7 +90,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -134,7 +137,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index af741c69a..b8d109aca 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -16,7 +16,7 @@ function newSingleFile () { return thisObject - function initialize (pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } @@ -34,7 +34,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDevTeam, pBot, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDevTeam, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try { From 266d5ca7c440b586158778b0bcb5061c4aad4919 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 8 Oct 2019 16:39:14 +0200 Subject: [PATCH 41/92] Small fix of Session --- src/FileCloud.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index b853661ab..e6730e8e3 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -113,7 +113,7 @@ function newFileCloud () { } if (pSession !== undefined) { - filePath = filePath.replace('@Session', pSession.name.replace(' ', '-')) + filePath = filePath.replace('@Session', pSession.id) } if (pExchange !== undefined) { From df61289965c58e879b917e1f626b8154e4585061 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 17 Oct 2019 10:44:17 +0200 Subject: [PATCH 42/92] session foldername --- src/FileCloud.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index e6730e8e3..42fc246ad 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -113,7 +113,19 @@ function newFileCloud () { } if (pSession !== undefined) { - filePath = filePath.replace('@Session', pSession.id) + let code + let sessionFolderName = pSession.id + if (pSession.code !== undefined) { + try { + code = JSON.parse(pSession.code) + if (code.folderName !== undefined) { + sessionFolderName = code.folderName + } + } catch (err) { + sessionFolderName = pSession.id + } + } + filePath = filePath.replace('@Session', sessionFolderName) } if (pExchange !== undefined) { From 192362a4af766fdbfbac4491bf3bbc1b863a7c73 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 21 Oct 2019 14:23:53 +0200 Subject: [PATCH 43/92] id added to foldername --- src/FileCloud.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index 42fc246ad..85843234b 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -119,7 +119,7 @@ function newFileCloud () { try { code = JSON.parse(pSession.code) if (code.folderName !== undefined) { - sessionFolderName = code.folderName + sessionFolderName = code.folderName + '-' + pSession.id } } catch (err) { sessionFolderName = pSession.id From 3930f58f7e9c046d410bb81eb5dc673fdfc1c1d4 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 21 Oct 2019 15:31:03 +0200 Subject: [PATCH 44/92] Data Range Updated by Events --- src/FileCursor.js | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index 9952a314b..f1452062f 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -38,12 +38,15 @@ function newFileCursor () { let finalized = false let eventSubscriptionIdDatasetUpdated + let eventSubscriptionIdDataRangeUpdated return thisObject function finalize () { try { systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) + systemEventHandler.stopListening('Data Range Updated', eventSubscriptionIdDataRangeUpdated) + thisObject.eventHandler = undefined market = undefined @@ -87,19 +90,51 @@ function newFileCursor () { endDateRange = pEndDateRange let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName - systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) + systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) + key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + pPeriodName + systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) + console.log(key) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) - function onResponse (message) { + function onResponseDataSet (message) { eventSubscriptionIdDatasetUpdated = message.eventSubscriptionId } + + function onResponseDataRange (message) { + eventSubscriptionIdDataRangeUpdated = message.eventSubscriptionId + } } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } + function updateDataRange (message) { + try { + if (finalized === true) { return } + + if (message.event === undefined || message.event.dateRange === undefined) { return } + + beginDateRange = new Date(message.event.dateRange.begin) + endDateRange = new Date(message.event.dateRange.end) + + let minDate = beginDateRange.valueOf() + let maxDate = endDateRange.valueOf() + ONE_DAY_IN_MILISECONDS + + for (let key of thisObject.files.keys()) { + let keyDate = new Date(key) + + if (keyDate.valueOf() < minDate || keyDate.valueOf() > maxDate) { + thisObject.files.delete(key) + thisObject.eventHandler.raiseEvent('Files Updated') + } + } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] updateDataRange -> err = ' + err.stack) } + } + } + function updateFiles (message) { try { if (finalized === true) { return } From 9a329a0818bb5409385a3ba21e928cb9628f0cc9 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 21 Oct 2019 17:15:14 +0200 Subject: [PATCH 45/92] Removing Log --- src/FileCursor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index f1452062f..16b97ecfc 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -94,7 +94,7 @@ function newFileCursor () { key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + pPeriodName systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) - console.log(key) + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) function onResponseDataSet (message) { From 66cdd3f3fd70c7464e96f7db90650669cd0c88d4 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 26 Dec 2019 10:45:42 +0100 Subject: [PATCH 46/92] Data Mine Refactoring --- src/DailyFiles.js | 8 ++++---- src/FileCloud.js | 20 ++++++++++---------- src/FileCursor.js | 16 ++++++++-------- src/FileSequence.js | 20 ++++++++++---------- src/MarketFiles.js | 16 ++++++++-------- src/SingleFile.js | 8 ++++---- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index e91ef8f04..744ded18f 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -45,14 +45,14 @@ function newDailyFiles () { } } - function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { try { callBackWhenFileReceived = callBackFunction let exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this product of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this product of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } fileCloud = newFileCloud() @@ -60,7 +60,7 @@ function newDailyFiles () { /* First we will get the Data Range */ - fileCloud.getFile(pDevTeam, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) function onDataRangeReceived (err, pFile) { try { @@ -110,7 +110,7 @@ function newDailyFiles () { if (pSet.validPeriods.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDevTeam, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) function onInitialized (err) { try { switch (err.result) { diff --git a/src/FileCloud.js b/src/FileCloud.js index 85843234b..c113521c4 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -46,18 +46,18 @@ function newFileCloud () { } } - function getFile (pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } const MAX_RETRIES = 3 - getFileRecursively(0, pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(0, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - function getFileRecursively (pRetryCounter, pDevTeam, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } let fileName let filePath @@ -85,7 +85,7 @@ function newFileCloud () { if (fileName === undefined) { logger.write('[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: ') - logger.write('[ERROR] getFile -> getFileRecursively -> pDevTeam = ' + JSON.stringify(pDevTeam)) + logger.write('[ERROR] getFile -> getFileRecursively -> pDataMine = ' + JSON.stringify(pDataMine)) logger.write('[ERROR] getFile -> getFileRecursively -> pBot = ' + JSON.stringify(pBot)) logger.write('[ERROR] getFile -> getFileRecursively -> pSet = ' + JSON.stringify(pSet)) logger.write('[ERROR] getFile -> getFileRecursively -> pExchange = ' + JSON.stringify(pExchange)) @@ -100,8 +100,8 @@ function newFileCloud () { fileName = fileName.replace('@AssetB', pMarket.assetB) } - if (pDevTeam !== undefined) { - filePath = filePath.replace('@DevTeam', pDevTeam.codeName) + if (pDataMine !== undefined) { + filePath = filePath.replace('@DataMine', pDataMine.codeName) } if (pBot !== undefined) { @@ -151,9 +151,9 @@ function newFileCloud () { let containerName - containerName = pDevTeam.codeName.toLowerCase() + containerName = pDataMine.codeName.toLowerCase() - blobService.getBlobToText(containerName, filePath + '/' + fileName, pDevTeam.host, onFileReceived) + blobService.getBlobToText(containerName, filePath + '/' + fileName, pDataMine.host, onFileReceived) function onFileReceived (err, text, response) { try { @@ -190,7 +190,7 @@ function newFileCloud () { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = ' + MAX_RETRIES) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = ' + pRetryCounter) } - getFileRecursively(pRetryCounter + 1, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(pRetryCounter + 1, pDataMine, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) return } else { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. ') } diff --git a/src/FileCursor.js b/src/FileCursor.js index 16b97ecfc..22bc7507e 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -25,7 +25,7 @@ function newFileCursor () { let market let exchange let fileCloud - let devTeam + let dataMine let bot let session let product @@ -52,7 +52,7 @@ function newFileCursor () { market = undefined exchange = undefined fileCloud = undefined - devTeam = undefined + dataMine = undefined bot = undefined session = undefined product = undefined @@ -73,12 +73,12 @@ function newFileCursor () { } } - function initialize (pFileCloud, pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { try { market = pMarket exchange = pExchange fileCloud = pFileCloud - devTeam = pDevTeam + dataMine = pDataMine bot = pBot session = pSession product = pProduct @@ -89,10 +89,10 @@ function newFileCursor () { beginDateRange = pBeginDateRange endDateRange = pEndDateRange - let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) - key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + pPeriodName + key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + pPeriodName systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) @@ -147,7 +147,7 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -447,7 +447,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 40fa3eb46..3a17299cd 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -20,7 +20,7 @@ function newFileSequence () { let files = new Map() let maxSequence = -1 // This is replaced by the content of the sequence file, which contains an index that starts on zero. In the case that the sequence file is not found the default value is -1 so when you add 1 it gives you the amount of files in the sequence, zero. let market - let devTeam + let dataMine let bot let session let thisSet @@ -47,7 +47,7 @@ function newFileSequence () { files = undefined maxSequence = undefined market = undefined - devTeam = undefined + dataMine = undefined bot = undefined session = undefined thisSet = undefined @@ -59,16 +59,16 @@ function newFileSequence () { } } - function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } market = pMarket - devTeam = pDevTeam + dataMine = pDataMine bot = pBot session = pSession thisSet = pSet @@ -79,7 +79,7 @@ function newFileSequence () { callerId = newUniqueId() - let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { @@ -88,7 +88,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -133,7 +133,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -198,7 +198,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -234,7 +234,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 717f0766d..e36443c60 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -24,7 +24,7 @@ function newMarketFiles () { let market let exchange - let devTeam + let dataMine let bot let session let thisSet @@ -52,7 +52,7 @@ function newMarketFiles () { market = undefined exchange = undefined - devTeam = undefined + dataMine = undefined bot = undefined session = undefined thisSet = undefined @@ -65,16 +65,16 @@ function newMarketFiles () { } } - function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } market = pMarket - devTeam = pDevTeam + dataMine = pDataMine bot = pBot session = pSession thisSet = pSet @@ -90,7 +90,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -103,7 +103,7 @@ function newMarketFiles () { } if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { - let key = devTeam.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) @@ -137,7 +137,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(devTeam, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index b8d109aca..4673a3be2 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -16,17 +16,17 @@ function newSingleFile () { return thisObject - function initialize (pDevTeam, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDevTeam.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } let exchange = ecosystem.getExchange(pProduct, pExchange) if (pExchange !== undefined && exchange === undefined) { // We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product. - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = ' + pDevTeam.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange } fileCloud = newFileCloud() @@ -34,7 +34,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDevTeam, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try { From 754435d8cc96ba3a256072c192420af45c65f976 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 27 Dec 2019 13:27:21 +0100 Subject: [PATCH 47/92] Time Frame Refectoring --- src/DailyFiles.js | 16 ++++++++-------- src/FileCursor.js | 30 +++++++++++++++--------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 744ded18f..8ee0a0097 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -9,7 +9,7 @@ function newDailyFiles () { eventHandler: undefined, getFileCursor: getFileCursor, setDatetime: setDatetime, - setTimePeriod: setTimePeriod, + setTimeFrame: setTimeFrame, getExpectedFiles: getExpectedFiles, getFilesLoaded: getFilesLoaded, initialize: initialize, @@ -45,7 +45,7 @@ function newDailyFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimeFrame, callBackFunction) { try { callBackWhenFileReceived = callBackFunction @@ -110,7 +110,7 @@ function newDailyFiles () { if (pSet.validPeriods.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized) + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, onInitialized) function onInitialized (err) { try { switch (err.result) { @@ -215,19 +215,19 @@ function newDailyFiles () { } } - function setTimePeriod (pTimePeriod, pDatetime) { + function setTimeFrame (pTimeFrame, pDatetime) { try { filesLoaded = 0 expectedFiles = 0 - fileCursors.forEach(setTimePeriodToEach) + fileCursors.forEach(setTimeFrameToEach) - function setTimePeriodToEach (fileCursor, key, map) { - fileCursor.setTimePeriod(pTimePeriod, pDatetime) + function setTimeFrameToEach (fileCursor, key, map) { + fileCursor.setTimeFrame(pTimeFrame, pDatetime) expectedFiles = expectedFiles + fileCursor.getExpectedFiles() fileCursor.reload(onFileReceived) } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> err = ' + err.stack) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimeFrame -> err = ' + err.stack) } } } diff --git a/src/FileCursor.js b/src/FileCursor.js index 22bc7507e..860ab221d 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -12,7 +12,7 @@ function newFileCursor () { eventHandler: undefined, // received from parent reload: reload, setDatetime: setDatetime, - setTimePeriod: setTimePeriod, + setTimeFrame: setTimeFrame, files: new Map(), getExpectedFiles: getExpectedFiles, initialize: initialize, @@ -31,7 +31,7 @@ function newFileCursor () { let product let thisSet let periodName - let timePeriod + let timeFrame let beginDateRange let endDateRange @@ -58,7 +58,7 @@ function newFileCursor () { product = undefined thisSet = undefined periodName = undefined - timePeriod = undefined + timeFrame = undefined beginDateRange = undefined endDateRange = undefined @@ -66,14 +66,14 @@ function newFileCursor () { cursorDate = undefined fileCloud = undefined periodName = undefined - timePeriod = undefined + timeFrame = undefined finalized = true } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] finalize -> err = ' + err.stack) } } } - function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimeFrame, pCursorDate, pCurrentTimeFrame, pBeginDateRange, pEndDateRange, callBackFunction) { try { market = pMarket exchange = pExchange @@ -85,7 +85,7 @@ function newFileCursor () { thisSet = pSet periodName = pPeriodName cursorDate = removeTime(pCursorDate) - timePeriod = pTimePeriod + timeFrame = pTimeFrame beginDateRange = pBeginDateRange endDateRange = pEndDateRange @@ -167,7 +167,7 @@ function newFileCursor () { } } - function setTimePeriod (pTimePeriod, pDatetime) { + function setTimeFrame (pTimeFrame, pDatetime) { try { if (finalized === true) { return } @@ -177,7 +177,7 @@ function newFileCursor () { We say there is a saving mode where the cursor is running at a minimum size. When the end user aproaches the time period the cursor is set, then it should exit the saving mode and go to its actual size. - To do this we are going to measure the distance from the Time Period received to the one the cursors was initialized with. + To do this we are going to measure the distance from the Time Frame received to the one the cursors was initialized with. If these periods are consecutive, it means that the cursor should exit saving mode and load its full size. */ @@ -189,11 +189,11 @@ function newFileCursor () { for (let i = 0; i < dailyFilePeriods.length; i++) { let period = dailyFilePeriods[i] - if (period[0] === pTimePeriod) { + if (period[0] === pTimeFrame) { positionA = i } - if (period[0] === timePeriod) { + if (period[0] === timeFrame) { positionB = i } } @@ -206,7 +206,7 @@ function newFileCursor () { function enterSavingMode () { try { - switch (timePeriod) { + switch (timeFrame) { case _45_MINUTES_IN_MILISECONDS: { @@ -277,13 +277,13 @@ function newFileCursor () { default: } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> enterSavingMode -> err = ' + err.stack) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimeFrame -> enterSavingMode -> err = ' + err.stack) } } } function exitSavingMode () { try { - switch (timePeriod) { + switch (timeFrame) { case _45_MINUTES_IN_MILISECONDS: { @@ -354,11 +354,11 @@ function newFileCursor () { default: } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> exitSavingMode -> err = ' + err.stack) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimeFrame -> exitSavingMode -> err = ' + err.stack) } } } } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] setTimePeriod -> err = ' + err.stack) } + if (ERROR_LOG === true) { logger.write('[ERROR] setTimeFrame -> err = ' + err.stack) } } } From 6f5e5b3f09cd5d277daf1981746404f459c1826c Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 5 Jan 2020 15:15:16 +0100 Subject: [PATCH 48/92] baseAsset and quotedAsset refactoring --- src/FileCloud.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index c113521c4..b7652eac2 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -96,8 +96,8 @@ function newFileCloud () { } if (pMarket !== undefined) { - fileName = fileName.replace('@AssetA', pMarket.assetA) - fileName = fileName.replace('@AssetB', pMarket.assetB) + fileName = fileName.replace('@BaseAsset', pMarket.baseAsset) + fileName = fileName.replace('@QuotedAsset', pMarket.quotedAsset) } if (pDataMine !== undefined) { From a2d23584c84cd769fd369510076c11cc278c17e3 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 15 Jan 2020 08:29:50 +0100 Subject: [PATCH 49/92] Integrating UI data --- src/DailyFiles.js | 12 ++++----- src/FileCloud.js | 55 ++++++++++++-------------------------- src/FileCursor.js | 16 ++++++------ src/FileSequence.js | 20 +++++++------- src/FileStorage.js | 64 +++++++-------------------------------------- src/MarketFiles.js | 20 +++++++------- src/SingleFile.js | 8 +++--- 7 files changed, 65 insertions(+), 130 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 8ee0a0097..9a5820253 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -45,14 +45,14 @@ function newDailyFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pDatetime, pTimeFrame, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, callBackFunction) { try { callBackWhenFileReceived = callBackFunction let exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this product of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this product of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } fileCloud = newFileCloud() @@ -60,7 +60,7 @@ function newDailyFiles () { /* First we will get the Data Range */ - fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) function onDataRangeReceived (err, pFile) { try { @@ -107,10 +107,10 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pSet.validPeriods.includes(periodName) === true) { + if (pDataset.validPeriods.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, onInitialized) + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, onInitialized) function onInitialized (err) { try { switch (err.result) { @@ -143,7 +143,7 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pSet.validPeriods.includes(periodName) === true) { + if (pDataset.validPeriods.includes(periodName) === true) { let fileCursor = fileCursors.get(periodTime) fileCursor.reload(onFileReceived) } diff --git a/src/FileCloud.js b/src/FileCloud.js index b7652eac2..97b3885f5 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -22,53 +22,32 @@ function newFileCloud () { return thisObject function initialize (pBot) { - try { - if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> pBot = ' + pBot.codeName) } - - switch (STORAGE_PROVIDER) { - case 'Azure': { - blobService = AzureStorage.Blob.createBlobServiceWithSas(pBot.storage.fileUri, pBot.storage.sas) - break - } - case 'AAWeb': { - blobService = newFileStorage() - break - } - default: { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> Storage Provider not supported -> process.env.STORAGE_PROVIDER = ' + STORAGE_PROVIDER) } - return - } - } - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } - // callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); TODO> Handle this callback. - } + blobService = newFileStorage() } - function getFile (pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } const MAX_RETRIES = 3 - getFileRecursively(0, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(0, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pSet.filePath + '-' + pSet.fileName) } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pDataset.filePath + '-' + pDataset.fileName) } let fileName let filePath if (pDataRange === undefined) { - fileName = pSet.fileName - filePath = pSet.filePath + fileName = pDataset.fileName + filePath = pDataset.filePath } else { - if (pSet.dataRange !== undefined) { - fileName = pSet.dataRange.fileName - filePath = pSet.dataRange.filePath + if (pDataset.dataRange !== undefined) { + fileName = pDataset.dataRange.fileName + filePath = pDataset.dataRange.filePath } else { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, @@ -87,7 +66,7 @@ function newFileCloud () { logger.write('[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: ') logger.write('[ERROR] getFile -> getFileRecursively -> pDataMine = ' + JSON.stringify(pDataMine)) logger.write('[ERROR] getFile -> getFileRecursively -> pBot = ' + JSON.stringify(pBot)) - logger.write('[ERROR] getFile -> getFileRecursively -> pSet = ' + JSON.stringify(pSet)) + logger.write('[ERROR] getFile -> getFileRecursively -> pDataset = ' + JSON.stringify(pDataset)) logger.write('[ERROR] getFile -> getFileRecursively -> pExchange = ' + JSON.stringify(pExchange)) logger.write('[ERROR] getFile -> getFileRecursively -> pMarket = ' + JSON.stringify(pMarket)) logger.write('[ERROR] getFile -> getFileRecursively -> pPeriodName = ' + JSON.stringify(pPeriodName)) @@ -101,14 +80,14 @@ function newFileCloud () { } if (pDataMine !== undefined) { - filePath = filePath.replace('@DataMine', pDataMine.codeName) + filePath = filePath.replace('@DataMine', pDataMine.code.codeName) } if (pBot !== undefined) { if (pBot.cloneId !== undefined) { - filePath = filePath.replace('@Bot', pBot.codeName + '-' + pBot.cloneId) + filePath = filePath.replace('@Bot', pBot.code.codeName + '-' + pBot.cloneId) } else { - filePath = filePath.replace('@Bot', pBot.codeName) + filePath = filePath.replace('@Bot', pBot.code.codeName) } } @@ -129,7 +108,7 @@ function newFileCloud () { } if (pExchange !== undefined) { - filePath = filePath.replace('@Exchange', pExchange.name) + filePath = filePath.replace('@Exchange', pExchange.code.code.codeName) } filePath = filePath.replace('@Period', pPeriodName) @@ -151,7 +130,7 @@ function newFileCloud () { let containerName - containerName = pDataMine.codeName.toLowerCase() + containerName = pDataMine.code.codeName.toLowerCase() blobService.getBlobToText(containerName, filePath + '/' + fileName, pDataMine.host, onFileReceived) @@ -190,7 +169,7 @@ function newFileCloud () { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = ' + MAX_RETRIES) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = ' + pRetryCounter) } - getFileRecursively(pRetryCounter + 1, pDataMine, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(pRetryCounter + 1, pDataMine, pBot, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) return } else { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. ') } diff --git a/src/FileCursor.js b/src/FileCursor.js index 860ab221d..439ffe270 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -29,7 +29,7 @@ function newFileCursor () { let bot let session let product - let thisSet + let dataset let periodName let timeFrame let beginDateRange @@ -56,7 +56,7 @@ function newFileCursor () { bot = undefined session = undefined product = undefined - thisSet = undefined + dataset = undefined periodName = undefined timeFrame = undefined beginDateRange = undefined @@ -73,7 +73,7 @@ function newFileCursor () { } } - function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, pPeriodName, pTimeFrame, pCursorDate, pCurrentTimeFrame, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pTimeFrame, pCursorDate, pCurrentTimeFrame, pBeginDateRange, pEndDateRange, callBackFunction) { try { market = pMarket exchange = pExchange @@ -82,17 +82,17 @@ function newFileCursor () { bot = pBot session = pSession product = pProduct - thisSet = pSet + dataset = pDataset periodName = pPeriodName cursorDate = removeTime(pCursorDate) timeFrame = pTimeFrame beginDateRange = pBeginDateRange endDateRange = pEndDateRange - let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) - key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + pPeriodName + key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + pPeriodName systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) @@ -147,7 +147,7 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -447,7 +447,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 3a17299cd..cc9682be7 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -23,7 +23,7 @@ function newFileSequence () { let dataMine let bot let session - let thisSet + let dataset let product let finalized = false let initialized = false @@ -50,7 +50,7 @@ function newFileSequence () { dataMine = undefined bot = undefined session = undefined - thisSet = undefined + dataset = undefined product = undefined finalized = true @@ -59,19 +59,19 @@ function newFileSequence () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } market = pMarket dataMine = pDataMine bot = pBot session = pSession - thisSet = pSet + dataset = pDataset product = pProduct fileCloud = newFileCloud() @@ -79,7 +79,7 @@ function newFileSequence () { callerId = newUniqueId() - let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { @@ -88,7 +88,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -133,7 +133,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -198,7 +198,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -234,7 +234,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/FileStorage.js b/src/FileStorage.js index e19f33a90..8eb476629 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -1,5 +1,5 @@ -function newFileStorage() { - const MODULE_NAME = 'File Cloud' +function newFileStorage () { + const MODULE_NAME = 'File Storage' const INFO_LOG = false const logger = newWebDebugLog() logger.fileName = MODULE_NAME @@ -29,61 +29,17 @@ function newFileStorage() { return thisObject - async function getBlobToText(container, filePath, host, callBackFunction) { + async function getBlobToText (container, filePath, host, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } - if (host.url.indexOf('localhost') !== -1) { - callServer(undefined, 'Storage/' + container + '/' + filePath, (response, fileContent) => { - if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) - } else { - callBackFunction(response) - } - }) - } else { - let headers - let accessToken = window.localStorage.getItem('access_token') - if (accessToken !== null) { - headers = { - authorization: 'Bearer ' + accessToken - } - } - - let response = await axios({ - url: host.url + 'graphql', - method: 'post', - data: { - query: ` - query web_FileContent($file: web_FileInput){ - web_FileContent(file: $file) - } - `, - variables: { - file: { - container: container.toLowerCase(), - filePath, - storage: host.storage, - accessKey: host.accessKey - } - } - }, - headers: headers - }) - - if (response.data.errors) { - let error = { code: response.data.errors[0] } - callBackFunction(error) - return - } - - if (response.data.data.web_FileContent) { - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, response.data.data.web_FileContent) + callServer(undefined, 'Storage/' + container + '/' + filePath, (response, fileContent) => { + if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) } else { - callBackFunction({ code: 'The specified key does not exist.' }) + callBackFunction(response) } - } - + }) } catch (err) { if (verifyRetry(err.code) && currentRetry < MAX_RETRY) { currentRetry++ @@ -99,9 +55,9 @@ function newFileStorage() { } } - function verifyRetry(errorCode) { + function verifyRetry (errorCode) { for (let i = 0; i < recoverableErrors.length; i++) { - const error = recoverableErrors[i]; + const error = recoverableErrors[i] if (error === errorCode) { return true } diff --git a/src/MarketFiles.js b/src/MarketFiles.js index e36443c60..e9c6f961c 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -27,7 +27,7 @@ function newMarketFiles () { let dataMine let bot let session - let thisSet + let dataset let product let periodName @@ -55,7 +55,7 @@ function newMarketFiles () { dataMine = undefined bot = undefined session = undefined - thisSet = undefined + dataset = undefined product = undefined periodName = undefined @@ -65,19 +65,19 @@ function newMarketFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { exchange = ecosystem.getExchange(pProduct, pExchange) if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } market = pMarket dataMine = pDataMine bot = pBot session = pSession - thisSet = pSet + dataset = pDataset product = pProduct fileCloud = newFileCloud() @@ -89,8 +89,8 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + if (dataset.validPeriods.includes(periodName) === true) { + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -103,7 +103,7 @@ function newMarketFiles () { } if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { - let key = dataMine.codeName + '-' + bot.codeName + '-' + product.codeName + '-' + thisSet.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) @@ -136,8 +136,8 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (thisSet.validPeriods.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + if (dataset.validPeriods.includes(periodName) === true) { + fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index 4673a3be2..031a553c9 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -16,17 +16,17 @@ function newSingleFile () { return thisObject - function initialize (pDataMine, pBot, pSession, pProduct, pSet, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.codeName + '-' + pBot.codeName + '-' + pProduct.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pProduct.code.codeName) } let exchange = ecosystem.getExchange(pProduct, pExchange) if (pExchange !== undefined && exchange === undefined) { // We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product. - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.codeName = ' + pDataMine.codeName + ', pBot.codeName = ' + pBot.codeName + ', pProduct.codeName = ' + pProduct.codeName + ', pExchange = ' + pExchange + throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange } fileCloud = newFileCloud() @@ -34,7 +34,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDataMine, pBot, pSession, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try { From 406499438a9211a5586bda801565abaec816b00e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 15 Jan 2020 09:04:54 +0100 Subject: [PATCH 50/92] Bugs Fixed --- src/DailyFiles.js | 10 +++------- src/FileCloud.js | 22 +++++++++------------- src/FileSequence.js | 7 +------ src/MarketFiles.js | 11 +++-------- src/SingleFile.js | 8 +------- 5 files changed, 17 insertions(+), 41 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 9a5820253..37b7c7464 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -49,11 +49,7 @@ function newDailyFiles () { try { callBackWhenFileReceived = callBackFunction - let exchange = ecosystem.getExchange(pProduct, pExchange) - - if (exchange === undefined) { - throw 'Exchange not supoorted by this product of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange - } + let exchange = pExchange fileCloud = newFileCloud() fileCloud.initialize(pBot) @@ -107,7 +103,7 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pDataset.validPeriods.includes(periodName) === true) { + if (pDataset.code.validTimeFrames.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, onInitialized) @@ -143,7 +139,7 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pDataset.validPeriods.includes(periodName) === true) { + if (pDataset.code.validTimeFrames.includes(periodName) === true) { let fileCursor = fileCursors.get(periodTime) fileCursor.reload(onFileReceived) } diff --git a/src/FileCloud.js b/src/FileCloud.js index 97b3885f5..3cb31e8d5 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -36,18 +36,18 @@ function newFileCloud () { function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pDataset.filePath + '-' + pDataset.fileName) } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pDataset.code.filePath + '-' + pDataset.code.fileName) } let fileName let filePath if (pDataRange === undefined) { - fileName = pDataset.fileName - filePath = pDataset.filePath + fileName = pDataset.code.fileName + filePath = pDataset.code.filePath } else { - if (pDataset.dataRange !== undefined) { - fileName = pDataset.dataRange.fileName - filePath = pDataset.dataRange.filePath + if (pDataset.code.dataRange !== undefined) { + fileName = pDataset.code.dataRange.fileName + filePath = pDataset.code.dataRange.filePath } else { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, @@ -84,11 +84,7 @@ function newFileCloud () { } if (pBot !== undefined) { - if (pBot.cloneId !== undefined) { - filePath = filePath.replace('@Bot', pBot.code.codeName + '-' + pBot.cloneId) - } else { - filePath = filePath.replace('@Bot', pBot.code.codeName) - } + filePath = filePath.replace('@Bot', pBot.code.codeName) } if (pSession !== undefined) { @@ -108,7 +104,7 @@ function newFileCloud () { } if (pExchange !== undefined) { - filePath = filePath.replace('@Exchange', pExchange.code.code.codeName) + filePath = filePath.replace('@Exchange', pExchange.code.codeName) } filePath = filePath.replace('@Period', pPeriodName) @@ -130,7 +126,7 @@ function newFileCloud () { let containerName - containerName = pDataMine.code.codeName.toLowerCase() + containerName = pDataMine.codeName.toLowerCase() blobService.getBlobToText(containerName, filePath + '/' + fileName, pDataMine.host, onFileReceived) diff --git a/src/FileSequence.js b/src/FileSequence.js index cc9682be7..3646cfcc1 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -61,12 +61,7 @@ function newFileSequence () { function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { - exchange = ecosystem.getExchange(pProduct, pExchange) - - if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange - } - + exchange = pExchange market = pMarket dataMine = pDataMine bot = pBot diff --git a/src/MarketFiles.js b/src/MarketFiles.js index e9c6f961c..272f35476 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -67,12 +67,7 @@ function newMarketFiles () { function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { try { - exchange = ecosystem.getExchange(pProduct, pExchange) - - if (exchange === undefined) { - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange - } - + exchange = pExchange market = pMarket dataMine = pDataMine bot = pBot @@ -89,7 +84,7 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (dataset.validPeriods.includes(periodName) === true) { + if (dataset.code.validTimeFrames.includes(periodName) === true) { fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { @@ -136,7 +131,7 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (dataset.validPeriods.includes(periodName) === true) { + if (dataset.code.validTimeFrames.includes(periodName) === true) { fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { diff --git a/src/SingleFile.js b/src/SingleFile.js index 031a553c9..1b12b44ef 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -21,13 +21,7 @@ function newSingleFile () { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pProduct.code.codeName) } - let exchange = ecosystem.getExchange(pProduct, pExchange) - - if (pExchange !== undefined && exchange === undefined) { - // We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product. - - throw 'Exchange not supoorted by this pProduct of the ecosystem! - pDataMine.code.codeName = ' + pDataMine.code.codeName + ', pBot.code.codeName = ' + pBot.code.codeName + ', pProduct.code.codeName = ' + pProduct.code.codeName + ', pExchange = ' + pExchange - } + let exchange = pExchange fileCloud = newFileCloud() fileCloud.initialize(pBot) From b0fdfc8a4be626fac15e5f30726bad56a203b149 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 15 Jan 2020 09:11:43 +0100 Subject: [PATCH 51/92] Bug Fixed --- src/FileCloud.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index 3cb31e8d5..f7bd02161 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -126,9 +126,9 @@ function newFileCloud () { let containerName - containerName = pDataMine.codeName.toLowerCase() + containerName = pDataMine.code.codeName.toLowerCase() - blobService.getBlobToText(containerName, filePath + '/' + fileName, pDataMine.host, onFileReceived) + blobService.getBlobToText(containerName, filePath + '/' + fileName, undefined, onFileReceived) function onFileReceived (err, text, response) { try { From 797fb06544041ff010b794f15d0467316e63d88d Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 21 Jan 2020 14:44:58 +0100 Subject: [PATCH 52/92] adapting --- src/FileCloud.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index f7bd02161..b7569918b 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -91,13 +91,8 @@ function newFileCloud () { let code let sessionFolderName = pSession.id if (pSession.code !== undefined) { - try { - code = JSON.parse(pSession.code) - if (code.folderName !== undefined) { - sessionFolderName = code.folderName + '-' + pSession.id - } - } catch (err) { - sessionFolderName = pSession.id + if (pSession.code.folderName !== undefined) { + sessionFolderName = pSession.code.folderName + '-' + pSession.id } } filePath = filePath.replace('@Session', sessionFolderName) From 1909e8c54ffbacb3231815bee9de3f769a182ccc Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 28 Jan 2020 13:28:54 +0100 Subject: [PATCH 53/92] Removing Container Name from Paths --- FileStorage | 54 ---------------------------------------------- src/FileCloud.js | 8 +------ src/FileStorage.js | 6 +++--- 3 files changed, 4 insertions(+), 64 deletions(-) delete mode 100644 FileStorage diff --git a/FileStorage b/FileStorage deleted file mode 100644 index c6c12d006..000000000 --- a/FileStorage +++ /dev/null @@ -1,54 +0,0 @@ -function newFileStorage() { - - const MODULE_NAME = 'File Cloud' - const INFO_LOG = false - const ERROR_LOG = true - const logger = newWebDebugLog() - logger.fileName = MODULE_NAME - - thisObject = { - getBlobToText: getBlobToText - }; - - return thisObject; - - function getBlobToText(pContainerName, pPath, callBackFunction) { - - let xhttp = new XMLHttpRequest(); - xhttp.onreadystatechange = function () { - if (this.readyState === 4 && this.status === 200) { - - try { - let response = JSON.parse(xhttp.responseText); - callBackFunction(response.err, response.text, ""); - } catch (err) { - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] AppPreLoader -> getBlobToText -> Invalid JSON received. "); } - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] AppPreLoader -> getBlobToText -> response.text = " & response.text); } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE, "", ""); - } - } - }; - - try { - - let request = { - conatinerName: pContainerName, - path: pPath - }; - - request = JSON.stringify(request); - - let path = "FileService"; - - let blob = new Blob([request], { type: 'text/plain' }); - - xhttp.open("GET", path, true); - xhttp.send(blob); - - } catch (err) { - - if (ERROR_LOG === true) { console.log(spacePad(MODULE_NAME, 50) + " : " + "[ERROR] getBlobToText -> err.message = " & err.message); } - - } - } -} \ No newline at end of file diff --git a/src/FileCloud.js b/src/FileCloud.js index b7569918b..79caf7511 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -119,11 +119,7 @@ function newFileCloud () { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> filePath = ' + filePath) } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> fileName = ' + fileName) } - let containerName - - containerName = pDataMine.code.codeName.toLowerCase() - - blobService.getBlobToText(containerName, filePath + '/' + fileName, undefined, onFileReceived) + blobService.getBlobToText(filePath + '/' + fileName, undefined, onFileReceived) function onFileReceived (err, text, response) { try { @@ -146,7 +142,6 @@ function newFileCloud () { return } - if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> containerName = ' + containerName) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Unexpected Error Ocurred.') } @@ -181,7 +176,6 @@ function newFileCloud () { return } catch (err) { if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> err = ' + err.stack) } - if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> containerName = ' + containerName) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } diff --git a/src/FileStorage.js b/src/FileStorage.js index 8eb476629..9a9abd787 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -29,11 +29,11 @@ function newFileStorage () { return thisObject - async function getBlobToText (container, filePath, host, callBackFunction) { + async function getBlobToText (filePath, host, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } - callServer(undefined, 'Storage/' + container + '/' + filePath, (response, fileContent) => { + callServer(undefined, 'Storage/' + filePath, (response, fileContent) => { if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) } else { @@ -44,7 +44,7 @@ function newFileStorage () { if (verifyRetry(err.code) && currentRetry < MAX_RETRY) { currentRetry++ if (INFO_LOG === true) { console.log('[INFO] getTextFile -> Retrying connection to the server because received error: ' + err.code + '. Retry #: ' + currentRetry) } - getBlobToText(container, filePath, host, callBackFunction) + getBlobToText(filePath, host, callBackFunction) } else if (err.message === 'Request aborted') { let err = { code: 'The specified key does not exist.' } callBackFunction(err) From 3d8f92ff3b53e2a2b2d117ad7fbe00991f5ffdb2 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 26 Feb 2020 14:40:04 +0100 Subject: [PATCH 54/92] Validations added --- src/DailyFiles.js | 8 ++++++++ src/MarketFiles.js | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 37b7c7464..3ea192514 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -54,6 +54,14 @@ function newDailyFiles () { fileCloud = newFileCloud() fileCloud.initialize(pBot) + /* Some Validations */ + if (pDataset.code.validTimeFrames === undefined) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + /* First we will get the Data Range */ fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 272f35476..dedf776c6 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -78,6 +78,14 @@ function newMarketFiles () { fileCloud = newFileCloud() fileCloud.initialize(pBot) + /* Some Validations */ + if (dataset.code.validTimeFrames === undefined) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + /* Now we will get the market files */ for (let i = 0; i < marketFilesPeriods.length; i++) { From 9194f88bd20274466a48975c01a5e81de104e4c0 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 6 Mar 2020 12:28:58 +0100 Subject: [PATCH 55/92] Dataset Updated Bugs Fixed --- src/FileCursor.js | 6 ++++-- src/FileSequence.js | 2 +- src/MarketFiles.js | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index 439ffe270..fa10dd904 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -89,12 +89,14 @@ function newFileCursor () { beginDateRange = pBeginDateRange endDateRange = pEndDateRange - let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.code.codeName + '-' + market.baseAsset + '/' + market.quotedAsset systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) - key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + pPeriodName + key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + exchange.code.codeName + '-' + market.baseAsset + '/' + market.quotedAsset systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) + console.log('Daily Files Data Range Updated Listening to Events at: ' + key) + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) function onResponseDataSet (message) { diff --git a/src/FileSequence.js b/src/FileSequence.js index 3646cfcc1..52c546c27 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -74,7 +74,7 @@ function newFileSequence () { callerId = newUniqueId() - let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.code.codeName + '-' + market.baseAsset + '/' + market.quotedAsset systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index dedf776c6..27a132d3f 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -106,7 +106,7 @@ function newMarketFiles () { } if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { - let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.code.codeName + '-' + market.baseAsset + '/' + market.quotedAsset systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) From dc46f4e50d8ca07e24f3183b93f8daad0641e591 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 6 Mar 2020 12:31:55 +0100 Subject: [PATCH 56/92] Dataset Updated Bugs Fixed --- src/FileCursor.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index fa10dd904..d2d3d902e 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -95,8 +95,6 @@ function newFileCursor () { key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + exchange.code.codeName + '-' + market.baseAsset + '/' + market.quotedAsset systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) - console.log('Daily Files Data Range Updated Listening to Events at: ' + key) - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) function onResponseDataSet (message) { From 92a4f5a50492d3c600e0f084caa2990a22026fc6 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 11 Mar 2020 14:20:44 +0100 Subject: [PATCH 57/92] Directory Structure Changed --- src/FileCloud.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index 79caf7511..c951880ac 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -75,8 +75,8 @@ function newFileCloud () { } if (pMarket !== undefined) { - fileName = fileName.replace('@BaseAsset', pMarket.baseAsset) - fileName = fileName.replace('@QuotedAsset', pMarket.quotedAsset) + filePath = filePath.replace('@BaseAsset', pMarket.baseAsset) + filePath = filePath.replace('@QuotedAsset', pMarket.quotedAsset) } if (pDataMine !== undefined) { From 8e2cbbe6300a5af6db4cb5229ed2ece8020ad7d7 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 11 Mar 2020 17:56:34 +0100 Subject: [PATCH 58/92] Update FileCloud.js --- src/FileCloud.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index c951880ac..1b159317d 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -99,7 +99,7 @@ function newFileCloud () { } if (pExchange !== undefined) { - filePath = filePath.replace('@Exchange', pExchange.code.codeName) + filePath = filePath.replace('@Exchange', pExchange.name) } filePath = filePath.replace('@Period', pPeriodName) From 6fbb74d03c03a9248ccb2000b5695bac93916977 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 12 Mar 2020 11:09:41 +0100 Subject: [PATCH 59/92] bug fixed --- src/FileCursor.js | 4 ++-- src/FileSequence.js | 2 +- src/MarketFiles.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index d2d3d902e..1608fc7c3 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -89,10 +89,10 @@ function newFileCursor () { beginDateRange = pBeginDateRange endDateRange = pEndDateRange - let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.code.codeName + '-' + market.baseAsset + '/' + market.quotedAsset + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) - key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + exchange.code.codeName + '-' + market.baseAsset + '/' + market.quotedAsset + key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) diff --git a/src/FileSequence.js b/src/FileSequence.js index 52c546c27..759eab6d3 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -74,7 +74,7 @@ function newFileSequence () { callerId = newUniqueId() - let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.code.codeName + '-' + market.baseAsset + '/' + market.quotedAsset + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 27a132d3f..d028c5417 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -106,7 +106,7 @@ function newMarketFiles () { } if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { - let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.code.codeName + '-' + market.baseAsset + '/' + market.quotedAsset + let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) From 3549a34991b53a8b474cff2e9f8368e56c3bceb9 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 19 Mar 2020 11:27:08 +0100 Subject: [PATCH 60/92] Update MarketFiles.js --- src/MarketFiles.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index d028c5417..e2fa6e665 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -36,13 +36,15 @@ function newMarketFiles () { let intervalHandle let finalized = false - let eventSubscriptionIdDatasetUpdated + let eventSubscriptionIdDatasetUpdated = [] return thisObject function finalize () { try { - systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) + for (let i = 0; i < eventSubscriptionIdDatasetUpdated.length; i++) { + systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated[i]) + } thisObject.eventHandler.finalize() thisObject.eventHandler = undefined @@ -108,11 +110,11 @@ function newMarketFiles () { if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) - + console.log('Dataset Updated:' + key + '-' + periodName) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) function onResponse (message) { - eventSubscriptionIdDatasetUpdated = message.eventSubscriptionId + eventSubscriptionIdDatasetUpdated.push(message.eventSubscriptionId) } } } catch (err) { @@ -132,7 +134,7 @@ function newMarketFiles () { try { if (finalized === true) { return } let updatedFiles = 0 - + console.log('updateFiles ' + periodName) /* Now we will get the market files */ for (let i = 0; i < marketFilesPeriods.length; i++) { From 2e56fa079bc572a7c5b45e90e70ec0f79150b09e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 19 Mar 2020 11:47:00 +0100 Subject: [PATCH 61/92] Update MarketFiles.js --- src/MarketFiles.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index e2fa6e665..c370e205b 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -110,7 +110,7 @@ function newMarketFiles () { if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) - console.log('Dataset Updated:' + key + '-' + periodName) + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) function onResponse (message) { From d6adb10664023eb3640123fb1a4e2f25e25eabab Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 21 Mar 2020 10:36:16 +0100 Subject: [PATCH 62/92] Update MarketFiles.js --- src/MarketFiles.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index c370e205b..7dcda95d0 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -134,7 +134,7 @@ function newMarketFiles () { try { if (finalized === true) { return } let updatedFiles = 0 - console.log('updateFiles ' + periodName) + /* Now we will get the market files */ for (let i = 0; i < marketFilesPeriods.length; i++) { From e9c3965d9079804dd6d2defca43d9680fdd738f9 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 28 Mar 2020 17:40:44 +0100 Subject: [PATCH 63/92] Data Mines Includes --- src/FileStorage.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/FileStorage.js b/src/FileStorage.js index 9a9abd787..bdd833546 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -29,11 +29,15 @@ function newFileStorage () { return thisObject - async function getBlobToText (filePath, host, callBackFunction) { + async function getBlobToText (filePath, host, callBackFunction, pathComplete) { try { if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } - callServer(undefined, 'Storage/' + filePath, (response, fileContent) => { + let folder = '' + if (pathComplete === false || pathComplete === undefined) { + folder = 'Storage/' + } + callServer(undefined, folder + filePath, (response, fileContent) => { if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) } else { From 48f22e3eef6ed6fed266316f41d9afbefb20095e Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 4 Apr 2020 19:55:22 +0200 Subject: [PATCH 64/92] @product --- src/DailyFiles.js | 2 +- src/FileCloud.js | 8 +++++--- src/FileCursor.js | 4 ++-- src/FileSequence.js | 8 ++++---- src/MarketFiles.js | 4 ++-- src/SingleFile.js | 2 +- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 3ea192514..faf199ce8 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -64,7 +64,7 @@ function newDailyFiles () { /* First we will get the Data Range */ - fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) function onDataRangeReceived (err, pFile) { try { diff --git a/src/FileCloud.js b/src/FileCloud.js index 1b159317d..4ee129964 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -25,15 +25,15 @@ function newFileCloud () { blobService = newFileStorage() } - function getFile (pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } const MAX_RETRIES = 3 - getFileRecursively(0, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pDataset.code.filePath + '-' + pDataset.code.fileName) } @@ -103,6 +103,8 @@ function newFileCloud () { } filePath = filePath.replace('@Period', pPeriodName) + filePath = filePath.replace('@Dataset', pDataset.code.codeName) + filePath = filePath.replace('@Product', pProduct.code.codeName) if (pDatetime !== undefined) { filePath = filePath.replace('@Year', pDatetime.getUTCFullYear()) diff --git a/src/FileCursor.js b/src/FileCursor.js index 1608fc7c3..d35cad879 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -147,7 +147,7 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -447,7 +447,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 759eab6d3..d06eb5bc1 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -83,7 +83,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -128,7 +128,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -193,7 +193,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -229,7 +229,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 7dcda95d0..1954bc5e9 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -95,7 +95,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (dataset.code.validTimeFrames.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -142,7 +142,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (dataset.code.validTimeFrames.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index 1b12b44ef..ba9971e6f 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -28,7 +28,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDataMine, pBot, pSession, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try { From 51393ad327b119471aed25b95dba536f194d2be8 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 6 Apr 2020 08:21:51 +0200 Subject: [PATCH 65/92] refactoring --- src/FileCursor.js | 8 ++++---- src/FileSequence.js | 4 ++-- src/MarketFiles.js | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index d35cad879..d1bede545 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -44,8 +44,8 @@ function newFileCursor () { function finalize () { try { - systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) - systemEventHandler.stopListening('Data Range Updated', eventSubscriptionIdDataRangeUpdated) + eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) + eventsServerClient.stopListening('Data Range Updated', eventSubscriptionIdDataRangeUpdated) thisObject.eventHandler = undefined @@ -90,10 +90,10 @@ function newFileCursor () { endDateRange = pEndDateRange let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) + eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - systemEventHandler.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) + eventsServerClient.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) diff --git a/src/FileSequence.js b/src/FileSequence.js index d06eb5bc1..feedf0b32 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -37,7 +37,7 @@ function newFileSequence () { function finalize () { try { - systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) + eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) thisObject.eventHandler.finalize() thisObject.eventHandler = undefined @@ -75,7 +75,7 @@ function newFileSequence () { callerId = newUniqueId() let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) + eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { eventSubscriptionIdDatasetUpdated = message.eventSubscriptionId diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 1954bc5e9..734314cfd 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -43,7 +43,7 @@ function newMarketFiles () { function finalize () { try { for (let i = 0; i < eventSubscriptionIdDatasetUpdated.length; i++) { - systemEventHandler.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated[i]) + eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated[i]) } thisObject.eventHandler.finalize() @@ -109,7 +109,7 @@ function newMarketFiles () { if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - systemEventHandler.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) + eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) From 50264e967596b4613faa41d5bcb4a824cf413072 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 6 Apr 2020 17:48:32 +0200 Subject: [PATCH 66/92] Bug fixed with caller Id --- src/FileCursor.js | 6 ++++-- src/MarketFiles.js | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index d1bede545..88c1463e1 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -90,10 +90,12 @@ function newFileCursor () { endDateRange = pEndDateRange let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponseDataSet, updateFiles) + let callerId = key + '-' + periodName + newUniqueId() + eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponseDataSet, updateFiles) key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - eventsServerClient.listenToEvent(key, 'Data Range Updated', undefined, key, onResponseDataRange, updateDataRange) + callerId = key + '-' + periodName + newUniqueId() + eventsServerClient.listenToEvent(key, 'Data Range Updated', undefined, callerId, onResponseDataRange, updateDataRange) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 734314cfd..ec1a45ee2 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -109,7 +109,8 @@ function newMarketFiles () { if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset - eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, key + '-' + periodName, onResponse, updateFiles) + let callerId = key + '-' + periodName + newUniqueId() + eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) From 5ed0fcaa23e20a9124994ef8207b12c852449f88 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 21 Apr 2020 14:34:26 +0200 Subject: [PATCH 67/92] host and pot implementation --- src/DailyFiles.js | 4 ++-- src/FileCloud.js | 8 ++++---- src/FileSequence.js | 4 ++-- src/FileStorage.js | 17 +++++++++++------ src/MarketFiles.js | 4 ++-- src/SingleFile.js | 4 ++-- 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index faf199ce8..356d87f85 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -45,14 +45,14 @@ function newDailyFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, callBackFunction) { try { callBackWhenFileReceived = callBackFunction let exchange = pExchange fileCloud = newFileCloud() - fileCloud.initialize(pBot) + fileCloud.initialize(pBot, pHost, pPort) /* Some Validations */ if (pDataset.code.validTimeFrames === undefined) { diff --git a/src/FileCloud.js b/src/FileCloud.js index 4ee129964..051705797 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -17,12 +17,12 @@ function newFileCloud () { initialize: initialize } - let blobService + let fileStorage return thisObject - function initialize (pBot) { - blobService = newFileStorage() + function initialize (pBot, pHost, pPort) { + fileStorage = newFileStorage(pHost, pPort) } function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { @@ -121,7 +121,7 @@ function newFileCloud () { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> filePath = ' + filePath) } if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> fileName = ' + fileName) } - blobService.getBlobToText(filePath + '/' + fileName, undefined, onFileReceived) + fileStorage.getFileFromHost(filePath + '/' + fileName, onFileReceived) function onFileReceived (err, text, response) { try { diff --git a/src/FileSequence.js b/src/FileSequence.js index feedf0b32..e9f0f9861 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -59,7 +59,7 @@ function newFileSequence () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pHost, pPort, callBackFunction) { try { exchange = pExchange market = pMarket @@ -70,7 +70,7 @@ function newFileSequence () { product = pProduct fileCloud = newFileCloud() - fileCloud.initialize(bot) + fileCloud.initialize(bot, pHost, pPort) callerId = newUniqueId() diff --git a/src/FileStorage.js b/src/FileStorage.js index bdd833546..6bd1d1883 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -1,4 +1,4 @@ -function newFileStorage () { +function newFileStorage (host, port) { const MODULE_NAME = 'File Storage' const INFO_LOG = false const logger = newWebDebugLog() @@ -24,20 +24,25 @@ function newFileStorage () { ] let thisObject = { - getBlobToText: getBlobToText + getFileFromHost: getFileFromHost } return thisObject - async function getBlobToText (filePath, host, callBackFunction, pathComplete) { + async function getFileFromHost (filePath, callBackFunction, pathComplete) { try { - if (INFO_LOG === true) { logger.write('[INFO] getBlobToText -> Entering function.') } + if (INFO_LOG === true) { logger.write('[INFO] getFileFromHost -> Entering function.') } let folder = '' if (pathComplete === false || pathComplete === undefined) { folder = 'Storage/' } - callServer(undefined, folder + filePath, (response, fileContent) => { + let url = folder + filePath + if (host !== undefined) { + url = 'http://' + host + ':' + port + '/' + folder + filePath + } + + callServer(undefined, url, (response, fileContent) => { if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) } else { @@ -48,7 +53,7 @@ function newFileStorage () { if (verifyRetry(err.code) && currentRetry < MAX_RETRY) { currentRetry++ if (INFO_LOG === true) { console.log('[INFO] getTextFile -> Retrying connection to the server because received error: ' + err.code + '. Retry #: ' + currentRetry) } - getBlobToText(filePath, host, callBackFunction) + getFileFromHost(filePath, callBackFunction) } else if (err.message === 'Request aborted') { let err = { code: 'The specified key does not exist.' } callBackFunction(err) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index ec1a45ee2..5beb8268e 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -67,7 +67,7 @@ function newMarketFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pHost, pPort, callBackFunction) { try { exchange = pExchange market = pMarket @@ -78,7 +78,7 @@ function newMarketFiles () { product = pProduct fileCloud = newFileCloud() - fileCloud.initialize(pBot) + fileCloud.initialize(pBot, pHost, pPort) /* Some Validations */ if (dataset.code.validTimeFrames === undefined) { diff --git a/src/SingleFile.js b/src/SingleFile.js index ba9971e6f..50c4858e7 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -16,7 +16,7 @@ function newSingleFile () { return thisObject - function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pHost, pPort, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pProduct.code.codeName) } @@ -24,7 +24,7 @@ function newSingleFile () { let exchange = pExchange fileCloud = newFileCloud() - fileCloud.initialize(pBot) + fileCloud.initialize(pBot, pHost, pPort) /* Now we will get the file */ From d307f1c028b0f61b61c63b2dc05e714b26b13791 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 22 Apr 2020 13:57:49 +0200 Subject: [PATCH 68/92] Remote Backends Implemented --- src/DailyFiles.js | 4 ++-- src/FileCursor.js | 21 ++++++++++++++++++++- src/FileSequence.js | 16 +++++++++++++++- src/MarketFiles.js | 16 +++++++++++++++- src/SingleFile.js | 13 ++++++++++++- 5 files changed, 64 insertions(+), 6 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 356d87f85..55cb27de9 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -45,7 +45,7 @@ function newDailyFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, callBackFunction) { + function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, pEventsServerClient, callBackFunction) { try { callBackWhenFileReceived = callBackFunction @@ -114,7 +114,7 @@ function newDailyFiles () { if (pDataset.code.validTimeFrames.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, onInitialized) + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) function onInitialized (err) { try { switch (err.result) { diff --git a/src/FileCursor.js b/src/FileCursor.js index 88c1463e1..29ddb900e 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -34,6 +34,7 @@ function newFileCursor () { let timeFrame let beginDateRange let endDateRange + let eventsServerClient let finalized = false @@ -61,6 +62,7 @@ function newFileCursor () { timeFrame = undefined beginDateRange = undefined endDateRange = undefined + eventsServerClient = undefined thisObject.files = undefined cursorDate = undefined @@ -73,7 +75,23 @@ function newFileCursor () { } } - function initialize (pFileCloud, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pTimeFrame, pCursorDate, pCurrentTimeFrame, pBeginDateRange, pEndDateRange, callBackFunction) { + function initialize ( + pFileCloud, + pDataMine, + pBot, + pSession, + pProduct, + pDataset, + pExchange, + pMarket, + pPeriodName, + pTimeFrame, + pCursorDate, + pCurrentTimeFrame, + pBeginDateRange, + pEndDateRange, + pEventsServerClient, + callBackFunction) { try { market = pMarket exchange = pExchange @@ -88,6 +106,7 @@ function newFileCursor () { timeFrame = pTimeFrame beginDateRange = pBeginDateRange endDateRange = pEndDateRange + eventsServerClient = pEventsServerClient let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset let callerId = key + '-' + periodName + newUniqueId() diff --git a/src/FileSequence.js b/src/FileSequence.js index e9f0f9861..f14de19f4 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -32,6 +32,7 @@ function newFileSequence () { let eventSubscriptionIdDatasetUpdated let callerId + let eventsServerClient return thisObject @@ -52,6 +53,7 @@ function newFileSequence () { session = undefined dataset = undefined product = undefined + eventsServerClient = undefined finalized = true } catch (err) { @@ -59,7 +61,18 @@ function newFileSequence () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pHost, pPort, callBackFunction) { + function initialize ( + pDataMine, + pBot, + pSession, + pProduct, + pDataset, + pExchange, + pMarket, + pHost, + pPort, + pEventsServerClient, + callBackFunction) { try { exchange = pExchange market = pMarket @@ -68,6 +81,7 @@ function newFileSequence () { session = pSession dataset = pDataset product = pProduct + eventsServerClient = pEventsServerClient fileCloud = newFileCloud() fileCloud.initialize(bot, pHost, pPort) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 5beb8268e..666389d08 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -37,6 +37,7 @@ function newMarketFiles () { let finalized = false let eventSubscriptionIdDatasetUpdated = [] + let eventsServerClient return thisObject @@ -60,6 +61,7 @@ function newMarketFiles () { dataset = undefined product = undefined periodName = undefined + eventsServerClient = undefined finalized = true } catch (err) { @@ -67,7 +69,18 @@ function newMarketFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pHost, pPort, callBackFunction) { + function initialize ( + pDataMine, + pBot, + pSession, + pProduct, + pDataset, + pExchange, + pMarket, + pHost, + pPort, + pEventsServerClient, + callBackFunction) { try { exchange = pExchange market = pMarket @@ -76,6 +89,7 @@ function newMarketFiles () { session = pSession dataset = pDataset product = pProduct + eventsServerClient = pEventsServerClient fileCloud = newFileCloud() fileCloud.initialize(pBot, pHost, pPort) diff --git a/src/SingleFile.js b/src/SingleFile.js index 50c4858e7..17d2542de 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -16,7 +16,18 @@ function newSingleFile () { return thisObject - function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pHost, pPort, callBackFunction) { + function initialize ( + pDataMine, + pBot, + pSession, + pProduct, + pDataset, + pExchange, + pMarket, + pHost, + pPort, + pEventsServerClient, + callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pProduct.code.codeName) } From 9ae7a45d9d487d1465e6ac2826b7cd28ab9e9ae9 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 25 Apr 2020 09:09:06 +0200 Subject: [PATCH 69/92] Update FileStorage.js --- src/FileStorage.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/FileStorage.js b/src/FileStorage.js index 6bd1d1883..ee85468b0 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -27,6 +27,9 @@ function newFileStorage (host, port) { getFileFromHost: getFileFromHost } + if (host === '' || host === undefined) { host = 'localhost' } + if (port === '' || port === undefined) { port = '34247' } + return thisObject async function getFileFromHost (filePath, callBackFunction, pathComplete) { From c63ecf644a1998abc86080fbd59c441b9ad26e67 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 23 May 2020 14:09:07 +0200 Subject: [PATCH 70/92] Split code and config --- src/DailyFiles.js | 6 +++--- src/FileCloud.js | 28 ++++++++++++++-------------- src/FileCursor.js | 4 ++-- src/FileSequence.js | 2 +- src/MarketFiles.js | 8 ++++---- src/SingleFile.js | 2 +- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 55cb27de9..d84a67d97 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -55,7 +55,7 @@ function newDailyFiles () { fileCloud.initialize(pBot, pHost, pPort) /* Some Validations */ - if (pDataset.code.validTimeFrames === undefined) { + if (pDataset.config.validTimeFrames === undefined) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) } if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) @@ -111,7 +111,7 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pDataset.code.validTimeFrames.includes(periodName) === true) { + if (pDataset.config.validTimeFrames.includes(periodName) === true) { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) @@ -147,7 +147,7 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pDataset.code.validTimeFrames.includes(periodName) === true) { + if (pDataset.config.validTimeFrames.includes(periodName) === true) { let fileCursor = fileCursors.get(periodTime) fileCursor.reload(onFileReceived) } diff --git a/src/FileCloud.js b/src/FileCloud.js index 051705797..69dfac0d8 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -36,18 +36,18 @@ function newFileCloud () { function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pDataset.code.filePath + '-' + pDataset.code.fileName) } + if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.config.codeName + '-' + pBot.config.codeName + '-' + pDataset.config.filePath + '-' + pDataset.config.fileName) } let fileName let filePath if (pDataRange === undefined) { - fileName = pDataset.code.fileName - filePath = pDataset.code.filePath + fileName = pDataset.config.fileName + filePath = pDataset.config.filePath } else { - if (pDataset.code.dataRange !== undefined) { - fileName = pDataset.code.dataRange.fileName - filePath = pDataset.code.dataRange.filePath + if (pDataset.config.dataRange !== undefined) { + fileName = pDataset.config.dataRange.fileName + filePath = pDataset.config.dataRange.filePath } else { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, @@ -80,19 +80,19 @@ function newFileCloud () { } if (pDataMine !== undefined) { - filePath = filePath.replace('@DataMine', pDataMine.code.codeName) + filePath = filePath.replace('@DataMine', pDataMine.config.codeName) } if (pBot !== undefined) { - filePath = filePath.replace('@Bot', pBot.code.codeName) + filePath = filePath.replace('@Bot', pBot.config.codeName) } if (pSession !== undefined) { - let code + let config let sessionFolderName = pSession.id - if (pSession.code !== undefined) { - if (pSession.code.folderName !== undefined) { - sessionFolderName = pSession.code.folderName + '-' + pSession.id + if (pSession.config !== undefined) { + if (pSession.config.folderName !== undefined) { + sessionFolderName = pSession.config.folderName + '-' + pSession.id } } filePath = filePath.replace('@Session', sessionFolderName) @@ -103,8 +103,8 @@ function newFileCloud () { } filePath = filePath.replace('@Period', pPeriodName) - filePath = filePath.replace('@Dataset', pDataset.code.codeName) - filePath = filePath.replace('@Product', pProduct.code.codeName) + filePath = filePath.replace('@Dataset', pDataset.config.codeName) + filePath = filePath.replace('@Product', pProduct.config.codeName) if (pDatetime !== undefined) { filePath = filePath.replace('@Year', pDatetime.getUTCFullYear()) diff --git a/src/FileCursor.js b/src/FileCursor.js index 29ddb900e..85f3bcc8c 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -108,11 +108,11 @@ function newFileCursor () { endDateRange = pEndDateRange eventsServerClient = pEventsServerClient - let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset + let key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset let callerId = key + '-' + periodName + newUniqueId() eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponseDataSet, updateFiles) - key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset + key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset callerId = key + '-' + periodName + newUniqueId() eventsServerClient.listenToEvent(key, 'Data Range Updated', undefined, callerId, onResponseDataRange, updateDataRange) diff --git a/src/FileSequence.js b/src/FileSequence.js index f14de19f4..187576503 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -88,7 +88,7 @@ function newFileSequence () { callerId = newUniqueId() - let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset + let key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 666389d08..a6b0187bb 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -95,7 +95,7 @@ function newMarketFiles () { fileCloud.initialize(pBot, pHost, pPort) /* Some Validations */ - if (dataset.code.validTimeFrames === undefined) { + if (dataset.config.validTimeFrames === undefined) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) } if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) @@ -108,7 +108,7 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (dataset.code.validTimeFrames.includes(periodName) === true) { + if (dataset.config.validTimeFrames.includes(periodName) === true) { fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { @@ -122,7 +122,7 @@ function newMarketFiles () { } if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { - let key = dataMine.code.codeName + '-' + bot.code.codeName + '-' + product.code.codeName + '-' + dataset.code.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset + let key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset let callerId = key + '-' + periodName + newUniqueId() eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) @@ -156,7 +156,7 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (dataset.code.validTimeFrames.includes(periodName) === true) { + if (dataset.config.validTimeFrames.includes(periodName) === true) { fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { diff --git a/src/SingleFile.js b/src/SingleFile.js index 17d2542de..9a6fdbae8 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -30,7 +30,7 @@ function newSingleFile () { callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.code.codeName + '-' + pBot.code.codeName + '-' + pProduct.code.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.config.codeName + '-' + pBot.config.codeName + '-' + pProduct.config.codeName) } let exchange = pExchange From 83b793b67ad592928bb32d4644d6378cb2e6d91d Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 7 Jul 2020 19:16:11 +0200 Subject: [PATCH 71/92] Ports Changed --- src/FileStorage.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/FileStorage.js b/src/FileStorage.js index ee85468b0..937b293c7 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -1,4 +1,4 @@ -function newFileStorage (host, port) { +function newFileStorage(host, port) { const MODULE_NAME = 'File Storage' const INFO_LOG = false const logger = newWebDebugLog() @@ -28,11 +28,11 @@ function newFileStorage (host, port) { } if (host === '' || host === undefined) { host = 'localhost' } - if (port === '' || port === undefined) { port = '34247' } + if (port === '' || port === undefined) { port = '34248' } return thisObject - async function getFileFromHost (filePath, callBackFunction, pathComplete) { + async function getFileFromHost(filePath, callBackFunction, pathComplete) { try { if (INFO_LOG === true) { logger.write('[INFO] getFileFromHost -> Entering function.') } @@ -67,7 +67,7 @@ function newFileStorage (host, port) { } } - function verifyRetry (errorCode) { + function verifyRetry(errorCode) { for (let i = 0; i < recoverableErrors.length; i++) { const error = recoverableErrors[i] if (error === errorCode) { From 41d978212cf8f15826cd84b5562c7d618315c96a Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 24 Jul 2020 11:02:14 +0200 Subject: [PATCH 72/92] Time Frame Filter at Multi Period Market --- src/MarketFiles.js | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index a6b0187bb..b605e579b 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -145,10 +145,11 @@ function newMarketFiles () { } } - function updateFiles () { + function updateFiles (message) { try { if (finalized === true) { return } let updatedFiles = 0 + if (message.event === undefined) { return } /* Now we will get the market files */ @@ -156,21 +157,23 @@ function newMarketFiles () { let periodTime = marketFilesPeriods[i][0] let periodName = marketFilesPeriods[i][1] - if (dataset.config.validTimeFrames.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + if (dataset.config.validTimeFrames.includes(periodName) !== true) { continue } + if (message.event.timeFrames !== undefined) { + if (message.event.timeFrames.includes(periodName) !== true) { continue } + } + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) - function onFileReceived (err, file) { - try { - if (finalized === true) { return } - files.set(periodTime, file) - updatedFiles++ + function onFileReceived (err, file) { + try { + if (finalized === true) { return } + files.set(periodTime, file) + updatedFiles++ - if (updatedFiles === marketFilesPeriods.length) { - thisObject.eventHandler.raiseEvent('Files Updated') - } - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onFileReceived -> err = ' + err.stack) } + if (updatedFiles === marketFilesPeriods.length) { + thisObject.eventHandler.raiseEvent('Files Updated') } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] updateFiles -> onFileReceived -> err = ' + err.stack) } } } } From 6681c23a18e8573f41567b1efba5cbfa54e3c0e7 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 10:41:13 +0200 Subject: [PATCH 73/92] Preparing to read the Time Frames file --- src/DailyFiles.js | 108 ++++++++++++++++++++------------------------ src/FileCloud.js | 72 +++++++++++++---------------- src/FileCursor.js | 4 +- src/FileSequence.js | 8 ++-- src/MarketFiles.js | 4 +- src/SingleFile.js | 2 +- 6 files changed, 91 insertions(+), 107 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index d84a67d97..52305e64a 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -50,40 +50,37 @@ function newDailyFiles () { callBackWhenFileReceived = callBackFunction let exchange = pExchange + let beginDateRange + let endDateRange fileCloud = newFileCloud() fileCloud.initialize(pBot, pHost, pPort) - /* Some Validations */ + /* Config Validations */ if (pDataset.config.validTimeFrames === undefined) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Market Files for bot ' + pBot.name) } + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = Can not initialize Daily Files for bot ' + pBot.name) } if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = You need to define validTimeFrames at the Dataset config. ') } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) return } - /* First we will get the Data Range */ + getDataRangeFile() - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived) - - function onDataRangeReceived (err, pFile) { - try { - let beginDateRange - let endDateRange + function getDataRangeFile () { + /* First we will get the Data Range */ + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived) + function onDataRangeReceived (err, pFile) { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { beginDateRange = pFile.begin endDateRange = pFile.end - break } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) return } - case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { if (err.message === 'File does not exist.') { beginDateRange = new Date() @@ -97,7 +94,6 @@ function newDailyFiles () { endDateRange = new Date() break } - /* If none of the previous conditions are met, the we return the err to the caller. */ callBackFunction(err) return } @@ -106,60 +102,56 @@ function newDailyFiles () { return } } - /* Now we will get the daily files */ - for (i = 0; i < dailyFilePeriods.length; i++) { - let periodTime = dailyFilePeriods[i][0] - let periodName = dailyFilePeriods[i][1] + createFileCursors() + } + } - if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = newFileCursor() - fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) - function onInitialized (err) { - try { - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - break - } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) - return - } - default: { - callBackWhenFileReceived(err) - return - } + function createFileCursors () { + /* Now we will get the daily files */ + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] + + if (pDataset.config.validTimeFrames.includes(periodName) === true) { + let fileCursor = newFileCursor() + fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) + function onInitialized (err) { + try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + default: { + callBackWhenFileReceived(err) + return } - fileCursors.set(periodTime, fileCursor) - expectedFiles = expectedFiles + fileCursor.getExpectedFiles() - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + fileCursors.set(periodTime, fileCursor) + expectedFiles = expectedFiles + fileCursor.getExpectedFiles() + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } - loadThemAll() + } + loadThemAll() + } - function loadThemAll () { - try { - for (i = 0; i < dailyFilePeriods.length; i++) { - let periodTime = dailyFilePeriods[i][0] - let periodName = dailyFilePeriods[i][1] + function loadThemAll () { + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] - if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = fileCursors.get(periodTime) - fileCursor.reload(onFileReceived) - } - } - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> loadThemAll -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) - } + if (pDataset.config.validTimeFrames.includes(periodName) === true) { + let fileCursor = fileCursors.get(periodTime) + fileCursor.reload(onFileReceived) } - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { diff --git a/src/FileCloud.js b/src/FileCloud.js index 69dfac0d8..06bd8ac9d 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -1,15 +1,12 @@ function newFileCloud () { const MODULE_NAME = 'File Cloud' - const INFO_LOG = false const ERROR_LOG = true const logger = newWebDebugLog() logger.fileName = MODULE_NAME /* - This is the module in the system that actually connects to the cloud storage and grabs from there the needed files. - */ let thisObject = { @@ -18,47 +15,55 @@ function newFileCloud () { } let fileStorage - return thisObject function initialize (pBot, pHost, pPort) { fileStorage = newFileStorage(pHost, pPort) } - function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> Entering function.') } - const MAX_RETRIES = 3 + getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) - getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) - - function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> key = ' + pDataMine.config.codeName + '-' + pBot.config.codeName + '-' + pDataset.config.filePath + '-' + pDataset.config.fileName) } - let fileName let filePath - if (pDataRange === undefined) { + if (pDataRange === undefined && pTimeFrames === undefined) { fileName = pDataset.config.fileName filePath = pDataset.config.filePath } else { - if (pDataset.config.dataRange !== undefined) { - fileName = pDataset.config.dataRange.fileName - filePath = pDataset.config.dataRange.filePath - } else { - let customErr = { - result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, - message: 'Missing Configuration.' + if (pDataRange !== undefined) { + if (pDataset.config.dataRange !== undefined) { + fileName = pDataset.config.dataRange.fileName + filePath = pDataset.config.dataRange.filePath + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Missing Configuration.' + } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found at Dataset ' + pDataset.name) } + callBackFunction(customErr) + return + } + } + if (pTimeFrames !== undefined) { + if (pTimeFrames.config.timeFrames !== undefined) { + fileName = pTimeFrames.config.timeFrames.fileName + filePath = pTimeFrames.config.timeFrames.filePath + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Missing Configuration.' + } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } + if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Time Frames configuration could not be found at Dataset ' + pDataset.name) } + callBackFunction(customErr) + return } - - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } - if (ERROR_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found. ') } - - callBackFunction(customErr) - return } } @@ -118,17 +123,10 @@ function newFileCloud () { fileName = fileName.replace('@Sequence', pSequence) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> filePath = ' + filePath) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> fileName = ' + fileName) } - fileStorage.getFileFromHost(filePath + '/' + fileName, onFileReceived) function onFileReceived (err, text, response) { try { - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = ' + filePath) } - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> fileName = ' + fileName) } - let data if (err && err.result !== GLOBAL.DEFAULT_OK_RESPONSE.result) { @@ -137,9 +135,6 @@ function newFileCloud () { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, message: 'File does not exist.' } - - if (INFO_LOG === true) { logger.write('[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = ' + customErr.message) } - callBackFunction(customErr) return } @@ -171,9 +166,6 @@ function newFileCloud () { } else { try { data = JSON.parse(text) - - if (INFO_LOG === true) { logger.write('[INFO] getFile -> getFileRecursively -> onFileReceived -> Data received is valid JSON.') } - callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data) return } catch (err) { diff --git a/src/FileCursor.js b/src/FileCursor.js index 85f3bcc8c..134442d34 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -168,7 +168,7 @@ function newFileCursor () { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -468,7 +468,7 @@ function newFileCursor () { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 187576503..67ac3b493 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -97,7 +97,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -142,7 +142,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -207,7 +207,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, onSequenceFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -243,7 +243,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index b605e579b..31c8945cb 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -109,7 +109,7 @@ function newMarketFiles () { let periodName = marketFilesPeriods[i][1] if (dataset.config.validTimeFrames.includes(periodName) === true) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -161,7 +161,7 @@ function newMarketFiles () { if (message.event.timeFrames !== undefined) { if (message.event.timeFrames.includes(periodName) !== true) { continue } } - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index 9a6fdbae8..3103d1207 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -39,7 +39,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try { From 3b6ed944fa697342cd12bc3cbeb30c51a12f1628 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 11:31:51 +0200 Subject: [PATCH 74/92] Time Frames Files in Daily --- src/DailyFiles.js | 105 +++++++++++++++++++++++++++++++++------------- src/FileCloud.js | 6 +-- 2 files changed, 79 insertions(+), 32 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 52305e64a..6540e2fa1 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -52,6 +52,7 @@ function newDailyFiles () { let exchange = pExchange let beginDateRange let endDateRange + let timeFrames fileCloud = newFileCloud() fileCloud.initialize(pBot, pHost, pPort) @@ -102,6 +103,42 @@ function newDailyFiles () { return } } + getTimeFramesFile() + } + } + + function getTimeFramesFile () { + /* First we will get the Data Range */ + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) + + function onTimeFramesReceived (err, pFile) { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + timeFrames = pFile + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (err.message === 'File does not exist.') { + /* It is ok, we will deal with this later */ + break + } + if (err.message === 'Missing Configuration.') { + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration to locate the timeFrames is missing at the dataSet.') } + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> Time Frames Filters will be ignored -> Product = ' + pProduct.config.codeName) } + break + } + callBackFunction(err) + return + } + default: { + callBackFunction(err) + return + } + } createFileCursors() } } @@ -112,31 +149,34 @@ function newDailyFiles () { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] - if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = newFileCursor() - fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) - function onInitialized (err) { - try { - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - break - } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) - return - } - default: { - callBackWhenFileReceived(err) - return - } + if (pDataset.config.validTimeFrames.includes(periodName) === false) { continue } + if (timeFrames !== undefined) { + if (timeFrames.includes(periodName) === false) { continue } + } + + let fileCursor = newFileCursor() + fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) + function onInitialized (err) { + try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + default: { + callBackWhenFileReceived(err) + return } - fileCursors.set(periodTime, fileCursor) - expectedFiles = expectedFiles + fileCursor.getExpectedFiles() - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } + fileCursors.set(periodTime, fileCursor) + expectedFiles = expectedFiles + fileCursor.getExpectedFiles() + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } @@ -144,14 +184,21 @@ function newDailyFiles () { } function loadThemAll () { - for (i = 0; i < dailyFilePeriods.length; i++) { - let periodTime = dailyFilePeriods[i][0] - let periodName = dailyFilePeriods[i][1] + try { + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] - if (pDataset.config.validTimeFrames.includes(periodName) === true) { - let fileCursor = fileCursors.get(periodTime) - fileCursor.reload(onFileReceived) + if (pDataset.config.validTimeFrames.includes(periodName) === true) { + let fileCursor = fileCursors.get(periodTime) + if (fileCursor !== undefined) { + fileCursor.reload(onFileReceived) + } + } } + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> loadThemAll -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } } catch (err) { diff --git a/src/FileCloud.js b/src/FileCloud.js index 06bd8ac9d..6cfb1772b 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -51,9 +51,9 @@ function newFileCloud () { } } if (pTimeFrames !== undefined) { - if (pTimeFrames.config.timeFrames !== undefined) { - fileName = pTimeFrames.config.timeFrames.fileName - filePath = pTimeFrames.config.timeFrames.filePath + if (pDataset.config.timeFrames !== undefined) { + fileName = pDataset.config.timeFrames.fileName + filePath = pDataset.config.timeFrames.filePath } else { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, From d4443b03505eb6593d86c417553627ea75876eae Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 12:06:52 +0200 Subject: [PATCH 75/92] Market Files reading Time Frames Files --- src/DailyFiles.js | 69 +++++++++++++++++++++++++++------------------- src/MarketFiles.js | 60 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 95 insertions(+), 34 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 6540e2fa1..63aee23be 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -144,43 +144,54 @@ function newDailyFiles () { } function createFileCursors () { + try { /* Now we will get the daily files */ - for (i = 0; i < dailyFilePeriods.length; i++) { - let periodTime = dailyFilePeriods[i][0] - let periodName = dailyFilePeriods[i][1] + for (i = 0; i < dailyFilePeriods.length; i++) { + let periodTime = dailyFilePeriods[i][0] + let periodName = dailyFilePeriods[i][1] - if (pDataset.config.validTimeFrames.includes(periodName) === false) { continue } - if (timeFrames !== undefined) { - if (timeFrames.includes(periodName) === false) { continue } - } + if (pDataset.config.validTimeFrames.includes(periodName) === false) { + filesLoaded++ + continue + } + if (timeFrames !== undefined) { + if (timeFrames.includes(periodName) === false) { + filesLoaded++ + continue + } + } - let fileCursor = newFileCursor() - fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) - function onInitialized (err) { - try { - switch (err.result) { - case GLOBAL.DEFAULT_OK_RESPONSE.result: { - break + let fileCursor = newFileCursor() + fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s + fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) + function onInitialized (err) { + try { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + default: { + callBackWhenFileReceived(err) + return + } } - case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { - callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE) - return - } - default: { - callBackWhenFileReceived(err) - return - } + fileCursors.set(periodTime, fileCursor) + expectedFiles = expectedFiles + fileCursor.getExpectedFiles() + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } - fileCursors.set(periodTime, fileCursor) - expectedFiles = expectedFiles + fileCursor.getExpectedFiles() - } catch (err) { - if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> onFileReceived -> onInitialized -> err = ' + err.stack) } - callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } } + loadThemAll() + } catch (err) { + if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> createFileCursors -> err = ' + err.stack) } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) } - loadThemAll() } function loadThemAll () { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 31c8945cb..64bd607af 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -82,6 +82,8 @@ function newMarketFiles () { pEventsServerClient, callBackFunction) { try { + let timeFrames + exchange = pExchange market = pMarket dataMine = pDataMine @@ -102,13 +104,61 @@ function newMarketFiles () { return } - /* Now we will get the market files */ + getTimeFramesFile() - for (let i = 0; i < marketFilesPeriods.length; i++) { - let periodTime = marketFilesPeriods[i][0] - let periodName = marketFilesPeriods[i][1] + function getTimeFramesFile () { + /* First we will get the Data Range */ + fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) + + function onTimeFramesReceived (err, pFile) { + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + timeFrames = pFile + break + } + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) + return + } + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + if (err.message === 'File does not exist.') { + /* It is ok, we will deal with this later */ + break + } + if (err.message === 'Missing Configuration.') { + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration to locate the timeFrames is missing at the dataSet.') } + if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> Time Frames Filters will be ignored -> Product = ' + pProduct.config.codeName) } + break + } + callBackFunction(err) + return + } + default: { + callBackFunction(err) + return + } + } + getMarketFiles() + } + } + + function getMarketFiles () { + /* Now we will get the market files */ + for (let i = 0; i < marketFilesPeriods.length; i++) { + let periodTime = marketFilesPeriods[i][0] + let periodName = marketFilesPeriods[i][1] + + if (dataset.config.validTimeFrames.includes(periodName) === false) { + filesNotLoaded++ + continue + } + if (timeFrames !== undefined) { + if (timeFrames.includes(periodName) === false) { + filesNotLoaded++ + continue + } + } - if (dataset.config.validTimeFrames.includes(periodName) === true) { fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { From b38d32b3ca4350b119e3436f790727e960d6696b Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 12:47:54 +0200 Subject: [PATCH 76/92] Configured to not Support Time Frames --- src/DailyFiles.js | 4 ++++ src/FileCloud.js | 13 +++++++++++-- src/MarketFiles.js | 4 ++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 63aee23be..47c6b1ce6 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -126,6 +126,10 @@ function newDailyFiles () { /* It is ok, we will deal with this later */ break } + if (err.message === 'Configured to not Support This.') { + /* It is ok, we will deal with this later */ + break + } if (err.message === 'Missing Configuration.') { if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration to locate the timeFrames is missing at the dataSet.') } if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> Time Frames Filters will be ignored -> Product = ' + pProduct.config.codeName) } diff --git a/src/FileCloud.js b/src/FileCloud.js index 6cfb1772b..157ed7ff7 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -52,8 +52,17 @@ function newFileCloud () { } if (pTimeFrames !== undefined) { if (pDataset.config.timeFrames !== undefined) { - fileName = pDataset.config.timeFrames.fileName - filePath = pDataset.config.timeFrames.filePath + if (pDataset.config.timeFrames.fileName !== '') { + fileName = pDataset.config.timeFrames.fileName + filePath = pDataset.config.timeFrames.filePath + } else { + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: 'Configured to not Support This.' + } + callBackFunction(customErr) + return + } } else { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 64bd607af..15aff831b 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -125,6 +125,10 @@ function newMarketFiles () { /* It is ok, we will deal with this later */ break } + if (err.message === 'Configured to not Support This.') { + /* It is ok, we will deal with this later */ + break + } if (err.message === 'Missing Configuration.') { if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> The needed configuration to locate the timeFrames is missing at the dataSet.') } if (ERROR_LOG === true) { logger.write('[WARN] initialize -> onFileReceived -> Time Frames Filters will be ignored -> Product = ' + pProduct.config.codeName) } From a908ae5e4ab25e7f0c456a39a8c1d328c07a1f88 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 17:06:19 +0200 Subject: [PATCH 77/92] Update DailyFiles.js --- src/DailyFiles.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 47c6b1ce6..cf2a9638f 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -155,12 +155,10 @@ function newDailyFiles () { let periodName = dailyFilePeriods[i][1] if (pDataset.config.validTimeFrames.includes(periodName) === false) { - filesLoaded++ continue } if (timeFrames !== undefined) { if (timeFrames.includes(periodName) === false) { - filesLoaded++ continue } } From b60c4576cb6a4d39f12b67e6df77a2f626780988 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 17:31:42 +0200 Subject: [PATCH 78/92] Update MarketFiles.js --- src/MarketFiles.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 15aff831b..1a59120cd 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -17,6 +17,7 @@ function newMarketFiles () { let filesLoaded = 0 let filesNotLoaded = 0 + let filesExpected = 0 let fileCloud @@ -163,6 +164,7 @@ function newMarketFiles () { } } + filesExpected++ fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, file) { @@ -223,7 +225,7 @@ function newMarketFiles () { files.set(periodTime, file) updatedFiles++ - if (updatedFiles === marketFilesPeriods.length) { + if (updatedFiles === filesExpected) { thisObject.eventHandler.raiseEvent('Files Updated') } } catch (err) { From daa66ddb0ec7e7d8231d6233e2ad470f54c48844 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 27 Jul 2020 17:59:49 +0200 Subject: [PATCH 79/92] Update DailyFiles.js --- src/DailyFiles.js | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index cf2a9638f..decef072c 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -1,5 +1,5 @@ -  -function newDailyFiles () { + +function newDailyFiles() { const MODULE_NAME = 'Daily Files' const ERROR_LOG = true const logger = newWebDebugLog() @@ -26,7 +26,7 @@ function newDailyFiles () { return thisObject - function finalize () { + function finalize() { try { thisObject.eventHandler.finalize() thisObject.eventHandler = undefined @@ -37,7 +37,7 @@ function newDailyFiles () { fileCursors.forEach(finalizeEach) - function finalizeEach (item, key, mapObj) { + function finalizeEach(item, key, mapObj) { item.finalize() } } catch (err) { @@ -45,7 +45,7 @@ function newDailyFiles () { } } - function initialize (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, pEventsServerClient, callBackFunction) { + function initialize(pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, pEventsServerClient, callBackFunction) { try { callBackWhenFileReceived = callBackFunction @@ -67,11 +67,11 @@ function newDailyFiles () { getDataRangeFile() - function getDataRangeFile () { + function getDataRangeFile() { /* First we will get the Data Range */ fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived) - function onDataRangeReceived (err, pFile) { + function onDataRangeReceived(err, pFile) { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { beginDateRange = pFile.begin @@ -107,11 +107,11 @@ function newDailyFiles () { } } - function getTimeFramesFile () { + function getTimeFramesFile() { /* First we will get the Data Range */ fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) - function onTimeFramesReceived (err, pFile) { + function onTimeFramesReceived(err, pFile) { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { timeFrames = pFile @@ -147,9 +147,9 @@ function newDailyFiles () { } } - function createFileCursors () { + function createFileCursors() { try { - /* Now we will get the daily files */ + /* Now we will get the daily files */ for (i = 0; i < dailyFilePeriods.length; i++) { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] @@ -166,7 +166,7 @@ function newDailyFiles () { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) - function onInitialized (err) { + function onInitialized(err) { try { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { @@ -196,7 +196,7 @@ function newDailyFiles () { } } - function loadThemAll () { + function loadThemAll() { try { for (i = 0; i < dailyFilePeriods.length; i++) { let periodTime = dailyFilePeriods[i][0] @@ -220,7 +220,7 @@ function newDailyFiles () { } } - function onFileReceived (err) { + function onFileReceived(err) { try { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { @@ -247,7 +247,7 @@ function newDailyFiles () { } } - function getFileCursor (pPeriod) { + function getFileCursor(pPeriod) { try { return fileCursors.get(pPeriod) } catch (err) { @@ -255,13 +255,13 @@ function newDailyFiles () { } } - function setDatetime (pDatetime) { + function setDatetime(pDatetime) { try { filesLoaded = 0 expectedFiles = 0 fileCursors.forEach(setDatetimeToEach) - function setDatetimeToEach (fileCursor, key, map) { + function setDatetimeToEach(fileCursor, key, map) { fileCursor.setDatetime(pDatetime) expectedFiles = expectedFiles + fileCursor.getExpectedFiles() fileCursor.reload(onFileReceived) @@ -271,13 +271,13 @@ function newDailyFiles () { } } - function setTimeFrame (pTimeFrame, pDatetime) { + function setTimeFrame(pTimeFrame, pDatetime) { try { filesLoaded = 0 expectedFiles = 0 fileCursors.forEach(setTimeFrameToEach) - function setTimeFrameToEach (fileCursor, key, map) { + function setTimeFrameToEach(fileCursor, key, map) { fileCursor.setTimeFrame(pTimeFrame, pDatetime) expectedFiles = expectedFiles + fileCursor.getExpectedFiles() fileCursor.reload(onFileReceived) @@ -287,11 +287,11 @@ function newDailyFiles () { } } - function getExpectedFiles () { + function getExpectedFiles() { return expectedFiles } - function getFilesLoaded () { + function getFilesLoaded() { return filesLoaded } } From 280344605d8f102bae063100c1a07ff8a7ab97bf Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 28 Jul 2020 12:46:56 +0200 Subject: [PATCH 80/92] improvements --- src/DailyFiles.js | 1 + src/MarketFiles.js | 34 ++++++++++++++++++---------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index decef072c..f28017eb2 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -209,6 +209,7 @@ function newDailyFiles() { } } } + callBackWhenFileReceived(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> loadThemAll -> err = ' + err.stack) } callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 1a59120cd..e399df108 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -1,5 +1,5 @@ -  -function newMarketFiles () { + +function newMarketFiles() { const MODULE_NAME = 'Market Files' const ERROR_LOG = true const logger = newWebDebugLog() @@ -42,7 +42,7 @@ function newMarketFiles () { return thisObject - function finalize () { + function finalize() { try { for (let i = 0; i < eventSubscriptionIdDatasetUpdated.length; i++) { eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated[i]) @@ -70,7 +70,7 @@ function newMarketFiles () { } } - function initialize ( + function initialize( pDataMine, pBot, pSession, @@ -107,11 +107,11 @@ function newMarketFiles () { getTimeFramesFile() - function getTimeFramesFile () { + function getTimeFramesFile() { /* First we will get the Data Range */ fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) - function onTimeFramesReceived (err, pFile) { + function onTimeFramesReceived(err, pFile) { switch (err.result) { case GLOBAL.DEFAULT_OK_RESPONSE.result: { timeFrames = pFile @@ -147,7 +147,7 @@ function newMarketFiles () { } } - function getMarketFiles () { + function getMarketFiles() { /* Now we will get the market files */ for (let i = 0; i < marketFilesPeriods.length; i++) { let periodTime = marketFilesPeriods[i][0] @@ -167,7 +167,7 @@ function newMarketFiles () { filesExpected++ fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (finalized === true) { return } if (err.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { @@ -184,7 +184,7 @@ function newMarketFiles () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) - function onResponse (message) { + function onResponse(message) { eventSubscriptionIdDatasetUpdated.push(message.eventSubscriptionId) } } @@ -194,6 +194,8 @@ function newMarketFiles () { } } } + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject) } } catch (err) { if (ERROR_LOG === true) { logger.write('[ERROR] initialize -> err = ' + err.stack) } @@ -201,13 +203,13 @@ function newMarketFiles () { } } - function updateFiles (message) { + function updateFiles(message) { try { if (finalized === true) { return } let updatedFiles = 0 if (message.event === undefined) { return } - /* Now we will get the market files */ + /* Now we will get the market files */ for (let i = 0; i < marketFilesPeriods.length; i++) { let periodTime = marketFilesPeriods[i][0] @@ -219,7 +221,7 @@ function newMarketFiles () { } fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (finalized === true) { return } files.set(periodTime, file) @@ -238,19 +240,19 @@ function newMarketFiles () { } } - function getFile (pPeriod) { + function getFile(pPeriod) { return files.get(pPeriod) } - function getExpectedFiles () { + function getExpectedFiles() { return marketFilesPeriods.length } - function getFilesLoaded () { + function getFilesLoaded() { return filesLoaded } - function getFilesNotLoaded () { + function getFilesNotLoaded() { return filesNotLoaded } } From df7bea0836b760d1ae806554e1cae4909ab389d1 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 28 Jul 2020 17:27:48 +0200 Subject: [PATCH 81/92] Bug Dataset Updated Fixed --- src/FileCursor.js | 38 +++++++++++++++++++------------------- src/MarketFiles.js | 7 ++++--- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index 134442d34..9b11c6b6c 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -1,5 +1,5 @@ -function newFileCursor () { +function newFileCursor() { const MODULE_NAME = 'File Cursor' const INFO_LOG = false const ERROR_LOG = true @@ -43,7 +43,7 @@ function newFileCursor () { return thisObject - function finalize () { + function finalize() { try { eventsServerClient.stopListening('Dataset Updated', eventSubscriptionIdDatasetUpdated) eventsServerClient.stopListening('Data Range Updated', eventSubscriptionIdDataRangeUpdated) @@ -75,7 +75,7 @@ function newFileCursor () { } } - function initialize ( + function initialize( pFileCloud, pDataMine, pBot, @@ -118,11 +118,11 @@ function newFileCursor () { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE) - function onResponseDataSet (message) { + function onResponseDataSet(message) { eventSubscriptionIdDatasetUpdated = message.eventSubscriptionId } - function onResponseDataRange (message) { + function onResponseDataRange(message) { eventSubscriptionIdDataRangeUpdated = message.eventSubscriptionId } } catch (err) { @@ -131,7 +131,7 @@ function newFileCursor () { } } - function updateDataRange (message) { + function updateDataRange(message) { try { if (finalized === true) { return } @@ -156,7 +156,7 @@ function newFileCursor () { } } - function updateFiles (message) { + function updateFiles(message) { try { if (finalized === true) { return } @@ -170,7 +170,7 @@ function newFileCursor () { fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (finalized === true) { return } @@ -188,7 +188,7 @@ function newFileCursor () { } } - function setTimeFrame (pTimeFrame, pDatetime) { + function setTimeFrame(pTimeFrame, pDatetime) { try { if (finalized === true) { return } @@ -225,7 +225,7 @@ function newFileCursor () { enterSavingMode() } - function enterSavingMode () { + function enterSavingMode() { try { switch (timeFrame) { @@ -302,7 +302,7 @@ function newFileCursor () { } } - function exitSavingMode () { + function exitSavingMode() { try { switch (timeFrame) { @@ -383,7 +383,7 @@ function newFileCursor () { } } - function setDatetime (pDatetime) { + function setDatetime(pDatetime) { try { if (finalized === true) { return } @@ -398,7 +398,7 @@ function newFileCursor () { } } - function reload (callBackFunction) { + function reload(callBackFunction) { try { if (finalized === true) { return } @@ -411,7 +411,7 @@ function newFileCursor () { } } - function getFiles (callBackFunction) { + function getFiles(callBackFunction) { try { if (finalized === true) { return } @@ -422,7 +422,7 @@ function newFileCursor () { getNextFile() - function getNextFile () { + function getNextFile() { try { let targetDate = new Date(cursorDate) targetDate.setUTCDate(targetDate.getUTCDate() + j) @@ -479,7 +479,7 @@ function newFileCursor () { } } - function onFileReceived (err, file) { + function onFileReceived(err, file) { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> onFileReceived -> Entering function.') } @@ -524,7 +524,7 @@ function newFileCursor () { } } - function controlLoop () { + function controlLoop() { try { if (INFO_LOG === true) { logger.write('[INFO] getFiles -> controlLoop -> Entering function.') } @@ -548,7 +548,7 @@ function newFileCursor () { } } - function collectGarbage (callBackFunction) { + function collectGarbage(callBackFunction) { try { if (finalized === true) { return } @@ -570,7 +570,7 @@ function newFileCursor () { } } - function getExpectedFiles () { + function getExpectedFiles() { if (finalized === true) { return } if (INFO_LOG === true) { logger.write('[INFO] getExpectedFiles -> Entering function.') } diff --git a/src/MarketFiles.js b/src/MarketFiles.js index e399df108..cbb8028c8 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -207,7 +207,6 @@ function newMarketFiles() { try { if (finalized === true) { return } let updatedFiles = 0 - if (message.event === undefined) { return } /* Now we will get the market files */ @@ -216,8 +215,10 @@ function newMarketFiles() { let periodName = marketFilesPeriods[i][1] if (dataset.config.validTimeFrames.includes(periodName) !== true) { continue } - if (message.event.timeFrames !== undefined) { - if (message.event.timeFrames.includes(periodName) !== true) { continue } + if (message.event !== undefined) { + if (message.event.timeFrames !== undefined) { + if (message.event.timeFrames.includes(periodName) !== true) { continue } + } } fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) From 5deecf8055200535dde45d1b570f45af88becc81 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 14 Aug 2020 13:16:27 +0200 Subject: [PATCH 82/92] Possible bug fixed --- src/FileCursor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index 9b11c6b6c..12d5ab981 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -442,7 +442,7 @@ function newFileCursor() { if (beginDateRange !== undefined) { let compareBeginDate = removeTime(beginDateRange) - if (compareTargetDate.valueOf() < compareBeginDate) { + if (compareTargetDate.valueOf() < compareBeginDate.valueOf()) { controlLoop() return } @@ -451,7 +451,7 @@ function newFileCursor() { if (endDateRange !== undefined) { let compareEndDate = removeTime(endDateRange) - if (compareTargetDate.valueOf() > compareEndDate) { + if (compareTargetDate.valueOf() > compareEndDate.valueOf()) { controlLoop() return } From 13403ad0eed598760cdbc3e8e3f5e5af2c3b6850 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sat, 15 Aug 2020 11:14:10 +0200 Subject: [PATCH 83/92] Update FileCursor.js --- src/FileCursor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FileCursor.js b/src/FileCursor.js index 12d5ab981..4d46879ad 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -451,7 +451,7 @@ function newFileCursor() { if (endDateRange !== undefined) { let compareEndDate = removeTime(endDateRange) - if (compareTargetDate.valueOf() > compareEndDate.valueOf()) { + if (compareTargetDate.valueOf() >= compareEndDate.valueOf()) { controlLoop() return } From de918745d6376f94350f2a884b8e767530c241e0 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 16 Aug 2020 16:21:01 +0200 Subject: [PATCH 84/92] Update FileCloud.js --- src/FileCloud.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FileCloud.js b/src/FileCloud.js index 157ed7ff7..cfc819f98 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -103,10 +103,10 @@ function newFileCloud () { if (pSession !== undefined) { let config - let sessionFolderName = pSession.id + let sessionFolderName = pSession.type.replace(' ', '-').replace(' ', '-') + '-' + pSession.id if (pSession.config !== undefined) { if (pSession.config.folderName !== undefined) { - sessionFolderName = pSession.config.folderName + '-' + pSession.id + sessionFolderName = pSession.type.replace(' ', '-').replace(' ', '-') + '-' + pSession.config.folderName } } filePath = filePath.replace('@Session', sessionFolderName) From 750a3f91548807eb00535a2a4591d543e72519b2 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 23 Aug 2020 09:39:55 +0200 Subject: [PATCH 85/92] Update FileStorage.js --- src/FileStorage.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/FileStorage.js b/src/FileStorage.js index 937b293c7..113e9f2aa 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -27,9 +27,6 @@ function newFileStorage(host, port) { getFileFromHost: getFileFromHost } - if (host === '' || host === undefined) { host = 'localhost' } - if (port === '' || port === undefined) { port = '34248' } - return thisObject async function getFileFromHost(filePath, callBackFunction, pathComplete) { @@ -41,8 +38,10 @@ function newFileStorage(host, port) { folder = 'Storage/' } let url = folder + filePath - if (host !== undefined) { + if (host !== undefined && port !== undefined) { url = 'http://' + host + ':' + port + '/' + folder + filePath + } else { + url = folder + filePath } callServer(undefined, url, (response, fileContent) => { From 0d743c8d41ee95b5a38ca31a4252a037ec128502 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 7 Oct 2020 23:41:53 +0200 Subject: [PATCH 86/92] Deep Scan Medium Severity --- src/DailyFiles.js | 4 ++-- src/FileCloud.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index f28017eb2..222ccc995 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -150,7 +150,7 @@ function newDailyFiles() { function createFileCursors() { try { /* Now we will get the daily files */ - for (i = 0; i < dailyFilePeriods.length; i++) { + for (let i = 0; i < dailyFilePeriods.length; i++) { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] @@ -198,7 +198,7 @@ function newDailyFiles() { function loadThemAll() { try { - for (i = 0; i < dailyFilePeriods.length; i++) { + for (let i = 0; i < dailyFilePeriods.length; i++) { let periodTime = dailyFilePeriods[i][0] let periodName = dailyFilePeriods[i][1] diff --git a/src/FileCloud.js b/src/FileCloud.js index cfc819f98..a75095be6 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -139,7 +139,7 @@ function newFileCloud () { let data if (err && err.result !== GLOBAL.DEFAULT_OK_RESPONSE.result) { - if (err.code === 'BlobNotFound' | err.code === 'FileNotFound' | err.code === 'ParentNotFound' | err.code === 'The specified key does not exist.') { + if (err.code === 'BlobNotFound' || err.code === 'FileNotFound' || err.code === 'ParentNotFound' || err.code === 'The specified key does not exist.') { let customErr = { result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, message: 'File does not exist.' From 5b87d939c8f24074a9edaee9688e576fe0e18852 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 9 Oct 2020 09:41:44 +0200 Subject: [PATCH 87/92] Update MarketFiles.js --- src/MarketFiles.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/MarketFiles.js b/src/MarketFiles.js index cbb8028c8..1b04b9359 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -34,7 +34,6 @@ function newMarketFiles() { thisObject.eventHandler = newEventHandler() - let intervalHandle let finalized = false let eventSubscriptionIdDatasetUpdated = [] From 431557dc9256a41635250904cb74e10223e4a123 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Thu, 15 Oct 2020 16:22:46 +0200 Subject: [PATCH 88/92] Schemas Multi Project --- src/FileStorage.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FileStorage.js b/src/FileStorage.js index 113e9f2aa..77992863c 100644 --- a/src/FileStorage.js +++ b/src/FileStorage.js @@ -44,7 +44,7 @@ function newFileStorage(host, port) { url = folder + filePath } - callServer(undefined, url, (response, fileContent) => { + callWebServer(undefined, url, (response, fileContent) => { if (response.result === GLOBAL.DEFAULT_OK_RESPONSE.result) { callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, fileContent) } else { From 9d4c532a14d42247b7a170bee4ae1aa4039b3c2b Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Tue, 3 Nov 2020 09:31:41 +0100 Subject: [PATCH 89/92] refactoring dataMine --- src/DailyFiles.js | 8 ++++---- src/FileCloud.js | 14 +++++++------- src/FileCursor.js | 16 ++++++++-------- src/FileSequence.js | 18 +++++++++--------- src/MarketFiles.js | 16 ++++++++-------- src/SingleFile.js | 6 +++--- 6 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/DailyFiles.js b/src/DailyFiles.js index 222ccc995..63e10eaaa 100644 --- a/src/DailyFiles.js +++ b/src/DailyFiles.js @@ -45,7 +45,7 @@ function newDailyFiles() { } } - function initialize(pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, pEventsServerClient, callBackFunction) { + function initialize(pMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pDatetime, pTimeFrame, pHost, pPort, pEventsServerClient, callBackFunction) { try { callBackWhenFileReceived = callBackFunction @@ -69,7 +69,7 @@ function newDailyFiles() { function getDataRangeFile() { /* First we will get the Data Range */ - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived) + fileCloud.getFile(pMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, true, undefined, onDataRangeReceived) function onDataRangeReceived(err, pFile) { switch (err.result) { @@ -109,7 +109,7 @@ function newDailyFiles() { function getTimeFramesFile() { /* First we will get the Data Range */ - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) + fileCloud.getFile(pMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) function onTimeFramesReceived(err, pFile) { switch (err.result) { @@ -165,7 +165,7 @@ function newDailyFiles() { let fileCursor = newFileCursor() fileCursor.eventHandler = thisObject.eventHandler // We share our event handler with each file cursor, so that they can raise events there when files are changed.s - fileCursor.initialize(fileCloud, pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) + fileCursor.initialize(fileCloud, pMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, periodName, periodTime, pDatetime, pTimeFrame, beginDateRange, endDateRange, pEventsServerClient, onInitialized) function onInitialized(err) { try { switch (err.result) { diff --git a/src/FileCloud.js b/src/FileCloud.js index a75095be6..f75327002 100644 --- a/src/FileCloud.js +++ b/src/FileCloud.js @@ -21,12 +21,12 @@ function newFileCloud () { fileStorage = newFileStorage(pHost, pPort) } - function getFile (pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { + function getFile (pMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { try { const MAX_RETRIES = 3 - getFileRecursively(0, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) + getFileRecursively(0, pMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) - function getFileRecursively (pRetryCounter, pDataMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { + function getFileRecursively (pRetryCounter, pMine, pBot, pSession, pProduct, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, pTimeFrames, callBackFunction) { try { let fileName let filePath @@ -78,7 +78,7 @@ function newFileCloud () { if (fileName === undefined) { logger.write('[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: ') - logger.write('[ERROR] getFile -> getFileRecursively -> pDataMine = ' + JSON.stringify(pDataMine)) + logger.write('[ERROR] getFile -> getFileRecursively -> pMine = ' + JSON.stringify(pMine)) logger.write('[ERROR] getFile -> getFileRecursively -> pBot = ' + JSON.stringify(pBot)) logger.write('[ERROR] getFile -> getFileRecursively -> pDataset = ' + JSON.stringify(pDataset)) logger.write('[ERROR] getFile -> getFileRecursively -> pExchange = ' + JSON.stringify(pExchange)) @@ -93,8 +93,8 @@ function newFileCloud () { filePath = filePath.replace('@QuotedAsset', pMarket.quotedAsset) } - if (pDataMine !== undefined) { - filePath = filePath.replace('@DataMine', pDataMine.config.codeName) + if (pMine !== undefined) { + filePath = filePath.replace('@DataMine', pMine.config.codeName) } if (pBot !== undefined) { @@ -161,7 +161,7 @@ function newFileCloud () { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = ' + MAX_RETRIES) } if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = ' + pRetryCounter) } - getFileRecursively(pRetryCounter + 1, pDataMine, pBot, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + getFileRecursively(pRetryCounter + 1, pMine, pBot, pDataset, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) return } else { if (ERROR_LOG === true) { logger.write('[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. ') } diff --git a/src/FileCursor.js b/src/FileCursor.js index 4d46879ad..76693074f 100644 --- a/src/FileCursor.js +++ b/src/FileCursor.js @@ -25,7 +25,7 @@ function newFileCursor() { let market let exchange let fileCloud - let dataMine + let mine let bot let session let product @@ -53,7 +53,7 @@ function newFileCursor() { market = undefined exchange = undefined fileCloud = undefined - dataMine = undefined + mine = undefined bot = undefined session = undefined product = undefined @@ -77,7 +77,7 @@ function newFileCursor() { function initialize( pFileCloud, - pDataMine, + pMine, pBot, pSession, pProduct, @@ -96,7 +96,7 @@ function newFileCursor() { market = pMarket exchange = pExchange fileCloud = pFileCloud - dataMine = pDataMine + mine = pMine bot = pBot session = pSession product = pProduct @@ -108,11 +108,11 @@ function newFileCursor() { endDateRange = pEndDateRange eventsServerClient = pEventsServerClient - let key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset + let key = mine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset let callerId = key + '-' + periodName + newUniqueId() eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponseDataSet, updateFiles) - key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset + key = mine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset callerId = key + '-' + periodName + newUniqueId() eventsServerClient.listenToEvent(key, 'Data Range Updated', undefined, callerId, onResponseDataRange, updateDataRange) @@ -168,7 +168,7 @@ function newFileCursor() { dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2) - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) function onFileReceived(err, file) { try { @@ -468,7 +468,7 @@ function newFileCursor() { if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, periodName, targetDate, undefined, undefined, undefined, onFileReceived) } else { controlLoop() } diff --git a/src/FileSequence.js b/src/FileSequence.js index 67ac3b493..db3752acf 100644 --- a/src/FileSequence.js +++ b/src/FileSequence.js @@ -20,7 +20,7 @@ function newFileSequence () { let files = new Map() let maxSequence = -1 // This is replaced by the content of the sequence file, which contains an index that starts on zero. In the case that the sequence file is not found the default value is -1 so when you add 1 it gives you the amount of files in the sequence, zero. let market - let dataMine + let mine let bot let session let dataset @@ -48,7 +48,7 @@ function newFileSequence () { files = undefined maxSequence = undefined market = undefined - dataMine = undefined + mine = undefined bot = undefined session = undefined dataset = undefined @@ -62,7 +62,7 @@ function newFileSequence () { } function initialize ( - pDataMine, + pMine, pBot, pSession, pProduct, @@ -76,7 +76,7 @@ function newFileSequence () { try { exchange = pExchange market = pMarket - dataMine = pDataMine + mine = pMine bot = pBot session = pSession dataset = pDataset @@ -88,7 +88,7 @@ function newFileSequence () { callerId = newUniqueId() - let key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset + let key = mine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) function onResponse (message) { @@ -97,7 +97,7 @@ function newFileSequence () { /* First we will get the sequence max number */ - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) + fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) function onSequenceFileReceived (err, file) { try { @@ -142,7 +142,7 @@ function newFileSequence () { /* Now we will get the sequence of files */ for (let i = 0; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) + fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { @@ -207,7 +207,7 @@ function newFileSequence () { let currentMaxSequence = maxSequence - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) + fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, undefined, undefined, 'Sequence', undefined, undefined, onSequenceFileReceived) function onSequenceFileReceived (err, sequenceFile) { try { @@ -243,7 +243,7 @@ function newFileSequence () { /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ for (let i = currentMaxSequence; i <= maxSequence; i++) { - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) + fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, undefined, undefined, i, undefined, undefined, onFileReceived) function onFileReceived (err, file) { try { diff --git a/src/MarketFiles.js b/src/MarketFiles.js index 1b04b9359..be0c4f397 100644 --- a/src/MarketFiles.js +++ b/src/MarketFiles.js @@ -25,7 +25,7 @@ function newMarketFiles() { let market let exchange - let dataMine + let mine let bot let session let dataset @@ -55,7 +55,7 @@ function newMarketFiles() { market = undefined exchange = undefined - dataMine = undefined + mine = undefined bot = undefined session = undefined dataset = undefined @@ -70,7 +70,7 @@ function newMarketFiles() { } function initialize( - pDataMine, + pMine, pBot, pSession, pProduct, @@ -86,7 +86,7 @@ function newMarketFiles() { exchange = pExchange market = pMarket - dataMine = pDataMine + mine = pMine bot = pBot session = pSession dataset = pDataset @@ -108,7 +108,7 @@ function newMarketFiles() { function getTimeFramesFile() { /* First we will get the Data Range */ - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) + fileCloud.getFile(pMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, true, onTimeFramesReceived) function onTimeFramesReceived(err, pFile) { switch (err.result) { @@ -164,7 +164,7 @@ function newMarketFiles() { } filesExpected++ - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived(err, file) { try { @@ -177,7 +177,7 @@ function newMarketFiles() { } if (filesLoaded + filesNotLoaded === marketFilesPeriods.length) { - let key = dataMine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset + let key = mine.config.codeName + '-' + bot.config.codeName + '-' + product.config.codeName + '-' + dataset.config.codeName + '-' + exchange.name + '-' + market.baseAsset + '/' + market.quotedAsset let callerId = key + '-' + periodName + newUniqueId() eventsServerClient.listenToEvent(key, 'Dataset Updated', undefined, callerId, onResponse, updateFiles) @@ -219,7 +219,7 @@ function newMarketFiles() { if (message.event.timeFrames.includes(periodName) !== true) { continue } } } - fileCloud.getFile(dataMine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(mine, bot, session, product, dataset, exchange, market, periodName, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived(err, file) { try { diff --git a/src/SingleFile.js b/src/SingleFile.js index 3103d1207..5d5da4294 100644 --- a/src/SingleFile.js +++ b/src/SingleFile.js @@ -17,7 +17,7 @@ function newSingleFile () { return thisObject function initialize ( - pDataMine, + pMine, pBot, pSession, pProduct, @@ -30,7 +30,7 @@ function newSingleFile () { callBackFunction) { try { if (INFO_LOG === true) { logger.write('[INFO] initialize -> Entering function.') } - if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pDataMine.config.codeName + '-' + pBot.config.codeName + '-' + pProduct.config.codeName) } + if (INFO_LOG === true) { logger.write('[INFO] initialize -> key = ' + pMine.config.codeName + '-' + pBot.config.codeName + '-' + pProduct.config.codeName) } let exchange = pExchange @@ -39,7 +39,7 @@ function newSingleFile () { /* Now we will get the file */ - fileCloud.getFile(pDataMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, undefined, onFileReceived) + fileCloud.getFile(pMine, pBot, pSession, pProduct, pDataset, exchange, pMarket, undefined, undefined, undefined, undefined, undefined, onFileReceived) function onFileReceived (err, pFile) { try { From a9dcc5c337d4e5423919ac6077eff273674b66ff Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 15 Nov 2020 10:54:22 +0100 Subject: [PATCH 90/92] Update README.md --- README.md | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 71f275c98..01b008963 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,13 @@ # Files Component -The Files Component provides the functionality needed to load into the browser's memory data structures populated from plain text files grabbed from the cloud and optimized for different scenarios. Each of these scenarios are represented by a single js module as listed below. +This repo is one of the repositories built into Superalgos. For more information [read the docs](https://docs.superalgos.org/) or check the [Readme](https://github.com/Superalgos/Superalgos/) at the main repository of the project. -Once the information is initially loaded, it is automatically updated as times goes by or as the users changes its position on the timeline, both on the time dimension as in the time period (1m, 2m, 3m, 4m, 5m, 10m, ..., 1 hour, 2 hours, ..., 24 hs) dimension. +## Copyright and License Notice -## Single File +© Copyright 2020 Superalgos. Please see the [License](/LICENSE) for information on how you can use this software. -As the name suggests, this is used to load a single file. +## Disclaimer -## File Sequence +THE SOFTWARE AND ITS ASSOCIATED PRODUCTS AND SERVICES ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, SUITABILITY FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. -This is a multi-file data structure in which is stored information of a sequence of files. Let's imagine that the App using this component needs to read File_1, File_2, File_3 and File_4, this data structure is capable of doing that. - -## Market File - -The system-wide concept of a Market File is defined as a file containing data corresponding to the whole market history. If the data were candles for instance, it would be containing all candles since the beginning on the market until the head of the market (the most recent information). - -Since for each market there are multiple Market Files (one for 1 hour time period, one for 2 hours, 4 hours, 6, 8, 12 and 24 hs.) this data structure loads all of these files and have them readily available in memory for consumption. - -## Daily Files - -The system-wide concept of a Daily File is defined as a file containing data corresponding only to one single day. If the data were candles for instance, it would be containing all candles calculated from trades that happened in a single day only. - -Since for each market there are multiple Daily Files (one for 1 min time period, one for 2 min, 3 min, 4, 5, 10, 15, 20, 30 and 45 minutes) this data structure loads all of these files and have them readily available in memory for consumption. - -Moreover, it uses the concept of a File Cursor to load more files of the previous days and the following days in order to have them ready in memory if needed. - -## Installation - -The Files Component is part of the Canvas App. Please continue here to learn how to install it: - -https://github.com/Superalgos/CanvasApp +IN NO EVENT WILL SUPERALGOS BE LIABLE TO ANY PARTY FOR ANY DIRECT, INDIRECT, SPECIAL OR OTHER CONSEQUENTIAL DAMAGES FOR ANY USE OF THE SOFTWARE, OR ANY OTHER ASSOCIATED SERVICE OR WEB SITE, INCLUDING, WITHOUT LIMITATION, ANY LOST PROFITS, BUSINESS INTERRUPTION, LOSS OF FUNDS, PROGRAMS OR OTHER DATA OR OTHERWISE, EVEN IF WE ARE EXPRESSLY ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. From d0835a07b9e6066a3a70c5cc2fcbb2ea5cc8b494 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 15 Nov 2020 10:59:53 +0100 Subject: [PATCH 91/92] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 01b008963..ed3faa94c 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,9 @@ This repo is one of the repositories built into Superalgos. For more information THE SOFTWARE AND ITS ASSOCIATED PRODUCTS AND SERVICES ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, SUITABILITY FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. IN NO EVENT WILL SUPERALGOS BE LIABLE TO ANY PARTY FOR ANY DIRECT, INDIRECT, SPECIAL OR OTHER CONSEQUENTIAL DAMAGES FOR ANY USE OF THE SOFTWARE, OR ANY OTHER ASSOCIATED SERVICE OR WEB SITE, INCLUDING, WITHOUT LIMITATION, ANY LOST PROFITS, BUSINESS INTERRUPTION, LOSS OF FUNDS, PROGRAMS OR OTHER DATA OR OTHERWISE, EVEN IF WE ARE EXPRESSLY ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +# Maintainer + +[![Luis-Fernando-Molina](https://avatars1.githubusercontent.com/u/9479367?s=130&v=4)](https://github.com/Luis-Fernando-Molina) | +---| +[Luis-Fernando-Molina](https://github.com/Luis-Fernando-Molina) | From 03dfc9fff6264495a0a092d3195f200c3292d080 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 15 Nov 2020 11:16:42 +0100 Subject: [PATCH 92/92] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ed3faa94c..1a4136279 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Files Component +# Data Files This repo is one of the repositories built into Superalgos. For more information [read the docs](https://docs.superalgos.org/) or check the [Readme](https://github.com/Superalgos/Superalgos/) at the main repository of the project.