From e13507a18ae511b035249bd791782848c4810324 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Sun, 17 Feb 2019 10:47:33 +0100 Subject: [PATCH] Initial Commit --- DailyFiles.js | 392 +++++++++++++++++++++++++++ FileCloud.js | 252 ++++++++++++++++++ FileCursor.js | 688 ++++++++++++++++++++++++++++++++++++++++++++++++ FileSequence.js | 385 +++++++++++++++++++++++++++ MarketFiles.js | 253 ++++++++++++++++++ README.md | 27 ++ SingleFile.js | 105 ++++++++ 7 files changed, 2102 insertions(+) create mode 100644 DailyFiles.js create mode 100644 FileCloud.js create mode 100644 FileCursor.js create mode 100644 FileSequence.js create mode 100644 MarketFiles.js create mode 100644 README.md create mode 100644 SingleFile.js diff --git a/DailyFiles.js b/DailyFiles.js new file mode 100644 index 000000000..c7cbf8314 --- /dev/null +++ b/DailyFiles.js @@ -0,0 +1,392 @@ + +function newDailyFiles() { + + const MODULE_NAME = "Daily Files"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + let thisObject = { + getFileCursor: getFileCursor, + setDatetime: setDatetime, + setTimePeriod: setTimePeriod, + getExpectedFiles: getExpectedFiles, + getFilesLoaded: getFilesLoaded, + initialize: initialize, + finalize: finalize + } + + let filesLoaded = 0; + let expectedFiles = 0; + + let fileCloud; + + let fileCursors = new Map; + + let callBackWhenFileReceived; + + return thisObject; + + function finalize() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } + + filesLoaded = undefined; + expectedFiles = undefined; + fileCloud = undefined; + + fileCursors.forEach(finalizeEach); + + function finalizeEach(item, key, mapObj) { + + if (INFO_LOG === true) { logger.write("[INFO] finalize -> finalizeEach -> Entering function."); } + + item.finalize(); + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } + } + } + + function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } + + callBackWhenFileReceived = callBackFunction; + + let exchange = ecosystem.getExchange(pProduct, pExchange); + + if (exchange === undefined) { + + throw "Exchange not supoorted by this product of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; + + } + + fileCloud = newFileCloud(); + fileCloud.initialize(pBot); + + /* First we will get the Data Range */ + + fileCloud.getFile(pDevTeam, pBot, pSet, exchange, pMarket, undefined, undefined, undefined, true, onDataRangeReceived); + + function onDataRangeReceived(err, pFile) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Entering function."); } + + let beginDateRange; + let endDateRange; + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received OK Response."); } + + beginDateRange = new Date(pFile.begin); + endDateRange = new Date(pFile.end); + + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> err.message = " + err.message); } + + if (err.message === "File does not exist.") { + + if (INFO_LOG === true) { logger.write("[WARN] initialize -> onFileReceived -> No Date Range file found. maxDate will be set to current datetime. "); } + + endDateRange = new Date(); + break; + } + + if (err.message === "Missing Configuration.") { + + if (ERROR_LOG === true) { logger.write("[WARN] initialize -> onFileReceived -> The needed configuration for the dateRange at the dataSet of the product was not found."); } + if (ERROR_LOG === true) { logger.write("[WARN] initialize -> onFileReceived -> maxDate will be set to current datetime."); } + + endDateRange = new Date(); + break; + } + + /* If none of the previous conditions are met, the we return the err to the caller. */ + + callBackFunction(err); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + /* Now we will get the daily files */ + + for (i = 0; i < dailyFilePeriods.length; i++) { + + let periodTime = dailyFilePeriods[i][0]; + let periodName = dailyFilePeriods[i][1]; + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> periodTime = " + periodTime); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> periodName = " + periodName); } + + if (pSet.validPeriods.includes(periodName) === true) { + + let fileCursor = newFileCursor(); + fileCursor.initialize(fileCloud, pDevTeam, pBot, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, beginDateRange, endDateRange, onInitialized); + + function onInitialized(err) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Received FAIL Response."); } + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> Received Unexpected Response."); } + callBackWhenFileReceived(err); + return; + } + } + + fileCursors.set(periodTime, fileCursor); + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> expectedFiles = " + expectedFiles); } + + expectedFiles = expectedFiles + fileCursor.getExpectedFiles(); + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> onInitialized -> expectedFiles = " + expectedFiles); } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> onInitialized -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + } + } + + loadThemAll(); + + function loadThemAll() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> loadThemAll -> Entering function."); } + + for (i = 0; i < dailyFilePeriods.length; i++) { + + let periodTime = dailyFilePeriods[i][0]; + let periodName = dailyFilePeriods[i][1]; + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> loadThemAll -> periodTime = " + periodTime); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> loadThemAll -> periodName = " + periodName); } + + if (pSet.validPeriods.includes(periodName) === true) { + + let fileCursor = fileCursors.get(periodTime); + fileCursor.reload(onFileReceived); + + } + } + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> loadThemAll -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function onFileReceived(err) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received FAIL Response."); } + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> err.message = " + err.message); } + + callBackWhenFileReceived(err); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> Received Unexpected Response."); } + callBackWhenFileReceived(err); + return; + } + } + + filesLoaded++; + + if (INFO_LOG === true) { logger.write("[INFO] onFileReceived -> filesLoaded = " + filesLoaded); } + + callBackWhenFileReceived(GLOBAL.DEFAULT_OK_RESPONSE, thisObject); // Note that the call back is called for every file loaded at each cursor. + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] onFileReceived -> err = " + err); } + callBackWhenFileReceived(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function getFileCursor(pPeriod) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFileCursor -> Entering function."); } + + return fileCursors.get(pPeriod); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFileCursor -> err = " + err); } + + } + } + + function setDatetime(pDatetime) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> Entering function."); } + + filesLoaded = 0; + expectedFiles = 0; + + fileCursors.forEach(setDatetimeToEach) + + function setDatetimeToEach(fileCursor, key, map) { + + fileCursor.setDatetime(pDatetime); + + if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> expectedFiles = " + expectedFiles); } + + expectedFiles = expectedFiles + fileCursor.getExpectedFiles(); + + if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> expectedFiles = " + expectedFiles); } + + fileCursor.reload(onFileReceived); + + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setDatetime -> err = " + err); } + + } + } + + function setTimePeriod(pTimePeriod, pDatetime) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Entering function."); } + + filesLoaded = 0; + expectedFiles = 0; + + fileCursors.forEach(setTimePeriodToEach) + + function setTimePeriodToEach(fileCursor, key, map) { + + fileCursor.setTimePeriod(pTimePeriod, pDatetime); + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> expectedFiles = " + expectedFiles); } + + expectedFiles = expectedFiles + fileCursor.getExpectedFiles(); + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> expectedFiles = " + expectedFiles); } + + fileCursor.reload(onFileReceived); + + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> err = " + err); } + + } + } + + function getExpectedFiles() { + + if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> expectedFiles = " + expectedFiles); } + + return expectedFiles; + + } + + function getFilesLoaded() { + + if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> filesLoaded = " + filesLoaded); } + + return filesLoaded; + + } +} \ No newline at end of file diff --git a/FileCloud.js b/FileCloud.js new file mode 100644 index 000000000..4173eb528 --- /dev/null +++ b/FileCloud.js @@ -0,0 +1,252 @@ + +function newFileCloud() { + + const MODULE_NAME = "File Cloud"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + /* + + This is the module in the system that actually connects to the cloud storage and grabs from there the needed files. + + */ + + var thisObject = { + getFile: getFile, + initialize: initialize + }; + + let blobService; + + return thisObject; + + function initialize(pBot) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> pBot = " + pBot.codeName); } + + blobService = AzureStorage.Blob.createBlobServiceWithSas(pBot.storage.fileUri, pBot.storage.sas); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + //callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); TODO> Handle this callback. + } + } + + function getFile(pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } + + const MAX_RETRIES = 3; + + getFileRecursively(0, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + + function getFileRecursively(pRetryCounter, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pSet.filePath + "-" + pSet.fileName); } + + let fileName; + let filePath; + + if (pDataRange === undefined) { + + fileName = pSet.fileName; + filePath = pSet.filePath; + + } else { + + if (pSet.dataRange !== undefined) { + + fileName = pSet.dataRange.fileName; + filePath = pSet.dataRange.filePath; + + } else { + + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: "Missing Configuration." + }; + + if (ERROR_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = " + customErr.message); } + if (ERROR_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> Data Range configuration could not be found. "); } + + callBackFunction(customErr); + return; + } + } + + if (fileName === undefined) { + + logger.write("[ERROR] getFile -> getFileRecursively -> Inconsistant data. Check the following: "); + logger.write("[ERROR] getFile -> getFileRecursively -> pDevTeam = " + JSON.stringify(pDevTeam)); + logger.write("[ERROR] getFile -> getFileRecursively -> pBot = " + JSON.stringify(pBot)); + logger.write("[ERROR] getFile -> getFileRecursively -> pSet = " + JSON.stringify(pSet)); + logger.write("[ERROR] getFile -> getFileRecursively -> pExchange = " + JSON.stringify(pExchange)); + logger.write("[ERROR] getFile -> getFileRecursively -> pMarket = " + JSON.stringify(pMarket)); + logger.write("[ERROR] getFile -> getFileRecursively -> pPeriodName = " + JSON.stringify(pPeriodName)); + + throw ("Inconsistant data received."); + } + + if (pMarket !== undefined) { + + fileName = fileName.replace("@AssetA", pMarket.assetA); + fileName = fileName.replace("@AssetB", pMarket.assetB); + } + + if (pDevTeam !== undefined) { + + filePath = filePath.replace("@DevTeam", pDevTeam.codeName); + } + + if (pBot !== undefined) { + + filePath = filePath.replace("@Bot", pBot.codeName); + } + + if (pExchange !== undefined) { + + filePath = filePath.replace("@Exchange", pExchange.name); + } + + filePath = filePath.replace("@Period", pPeriodName); + + if (pDatetime !== undefined) { + + filePath = filePath.replace("@Year", pDatetime.getUTCFullYear()); + filePath = filePath.replace("@Month", pad(pDatetime.getUTCMonth() + 1, 2)); + filePath = filePath.replace("@Day", pad(pDatetime.getUTCDate(), 2)); + filePath = filePath.replace("@Hour", pad(pDatetime.getUTCHours(), 2)); + filePath = filePath.replace("@Minute", pad(pDatetime.getUTCMinutes(), 2)); + + } + + if (pSequence !== undefined) { + + fileName = fileName.replace("@Sequence", pSequence); + } + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> filePath = " + filePath); } + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> fileName = " + fileName); } + + let containerName; + + containerName = pDevTeam.codeName.toLowerCase(); + + blobService.getBlobToText(containerName, filePath + "/" + fileName, onFileReceived); + + function onFileReceived(err, text, response) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> filePath = " + filePath); } + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> fileName = " + fileName); } + + let data; + + if (err) { + + if (err.code === "BlobNotFound" | err.code === "FileNotFound" | err.code === "ParentNotFound") { + + let customErr = { + result: GLOBAL.CUSTOM_FAIL_RESPONSE.result, + message: "File does not exist." + }; + + if (INFO_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = " + customErr.message); } + + callBackFunction(customErr); + return; + } + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> containerName = " + containerName); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> filePath = " + filePath); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> fileName = " + fileName); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> Unexpected Error Ocurred."); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = " + err); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> text = " + text); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> response = " + response); } + + if (err.message === "XHR error") { + + if (pRetryCounter < MAX_RETRIES) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> Retrying to get this file. "); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = " + MAX_RETRIES); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = " + pRetryCounter); } + + getFileRecursively(pRetryCounter + 1, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pDatetime, pSequence, pDataRange, callBackFunction) + return; + + } else { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> Could not get this file from storage. "); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> MAX_RETRIES = " + MAX_RETRIES); } + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> pRetryCounter = " + pRetryCounter); } + + } + } + + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + + } else { + + try { + + data = JSON.parse(text); + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> getFileRecursively -> onFileReceived -> Data received is valid JSON."); } + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, data); + return; + + } catch (err) { + + if (INFO_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> err = " + err); } + + let customErr = { + result: GLOBAL.CUSTOM_OK_RESPONSE.result, + message: "Data not in JSON Format." + }; + + if (INFO_LOG === true) { logger.write("[WARN] getFile -> getFileRecursively -> onFileReceived -> customErr.message = " + customErr.message); } + + callBackFunction(customErr, text); + return; + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> getFileRecursively -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFile -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } +} \ No newline at end of file diff --git a/FileCursor.js b/FileCursor.js new file mode 100644 index 000000000..9592c2c1f --- /dev/null +++ b/FileCursor.js @@ -0,0 +1,688 @@ + +function newFileCursor() { + + const MODULE_NAME = "File Cursor"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + let cursorDate; + + let thisObject = { + reload: reload, + setDatetime: setDatetime, + setTimePeriod: setTimePeriod, + files: new Map, + getExpectedFiles: getExpectedFiles, + initialize: initialize, + finalize: finalize + }; + + let minCursorSize = 10; + let maxCursorSize = 30; + + let market; + let exchange; + let fileCloud; + let devTeam; + let bot; + let thisSet; + let periodName; + let timePeriod; + let beginDateRange; + let endDateRange; + + let intervalHandle; + + return thisObject; + + function finalize() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } + + clearInterval(intervalHandle); + + thisObject.files = undefined; + cursorDate = undefined; + fileCloud = undefined; + periodName = undefined; + timePeriod = undefined; + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } + } + } + + function initialize(pFileCloud, pDevTeam, pBot, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, pBeginDateRange, pEndDateRange, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pSet.codeName + "-" + pPeriodName); } + + market = pMarket; + exchange = pExchange; + fileCloud = pFileCloud; + devTeam = pDevTeam; + bot = pBot; + thisSet = pSet; + periodName = pPeriodName; + cursorDate = pCursorDate; + timePeriod = pTimePeriod; + beginDateRange = pBeginDateRange; + endDateRange = pEndDateRange; + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> market = " + market); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> periodName = " + periodName); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> cursorDate = " + cursorDate); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> beginDateRange = " + beginDateRange); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> endDateRange = " + endDateRange); } + + setTimePeriod(pCurrentTimePeriod, pCursorDate); + + intervalHandle = setInterval(updateFiles , timePeriod); + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function updateFiles() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Entering function."); } + + /* + + In order to know if we need to update any file we follow this logic: + + 1. If the current date is already a file in the cursor, we need to update it. + 2. If the current date is one day ahead of a file in the cursor, we need to add this one to the cursor. + + */ + + let targetDate; + let dateString; + let file; + + /* Situation 1 */ + + targetDate = new Date(); + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); + + file = thisObject.files.get(dateString); + + if (file !== undefined) { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> File found at the cursor, proceeding to update."); } + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived); + + return; + } + + /* Situation 2 */ + + targetDate = new Date( (new Date()).valueOf() - ONE_DAY_IN_MILISECONDS); + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); + + file = thisObject.files.get(dateString); // This is from yesterday + + targetDate = new Date(); + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); + + if (file !== undefined) { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Previous day file found at the cursor, proceeding to add current date."); } + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived); + + return; + } + + function onFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received FAIL Response."); } + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received Unexpected Response."); } + return; + } + } + + thisObject.files.set(dateString, file); + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> File updated."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> devTeam = " + devTeam.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> bot = " + bot.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> thisSet = " + thisSet.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> dateString = " + dateString); } + + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> err = " + err); } + } + } + + function setTimePeriod(pTimePeriod, pDatetime) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Entering function."); } + + /* + + We are implementing here an algorithm designed to save bandwidth, memory and processing power at the browser. + We say there is a saving mode where the cursor is running at a minimum size. When the end user aproaches the time period the cursor + is set, then it should exit the saving mode and go to its actual size. + + To do this we are going to measure the distance from the Time Period received to the one the cursors was initialized with. + If these periods are consecutive, it means that the cursor should exit saving mode and load its full size. + + */ + + cursorDate = pDatetime; // Adjust the cursor date to the one received. + + let positionA; + + for (let i = 0; i < dailyFilePeriods.length; i++) { + + let period = dailyFilePeriods[i]; + + if (period[0] === pTimePeriod) { + + positionA = i; + + } + + if (period[0] === timePeriod) { + + positionB = i; + + } + } + + if (Math.abs(positionB - positionA) <= 1) { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Will EXIT Saving Mode."); } + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> pTimePeriod = " + convertTimePeriodToName(pTimePeriod)); } + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> timePeriod = " + convertTimePeriodToName(timePeriod)); } + + exitSavingMode(); + + } else { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> Will ENTER Saving Mode."); } + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> pTimePeriod = " + convertTimePeriodToName(pTimePeriod)); } + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> timePeriod = " + convertTimePeriodToName(timePeriod)); } + + enterSavingMode(); + + } + + function enterSavingMode() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> enterSavingMode -> Entering function."); } + + switch (timePeriod) { + + case _45_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 30; + } + break; + case _40_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 20; + } + break; + case _30_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _20_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _15_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _10_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _5_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _4_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _3_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _2_MINUTES_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + case _1_MINUTE_IN_MILISECONDS: + { + minCursorSize = 3; + maxCursorSize = 15; + } + break; + default: + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> enterSavingMode -> err = " + err); } + } + } + + function exitSavingMode() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setTimePeriod -> exitSavingMode -> Entering function."); } + + switch (timePeriod) { + + case _45_MINUTES_IN_MILISECONDS: + { + minCursorSize = 15; + maxCursorSize = 30; + } + break; + case _40_MINUTES_IN_MILISECONDS: + { + minCursorSize = 13; + maxCursorSize = 20; + } + break; + case _30_MINUTES_IN_MILISECONDS: + { + minCursorSize = 11; + maxCursorSize = 15; + } + break; + case _20_MINUTES_IN_MILISECONDS: + { + minCursorSize = 9; + maxCursorSize = 15; + } + break; + case _15_MINUTES_IN_MILISECONDS: + { + minCursorSize = 7; + maxCursorSize = 15; + } + break; + case _10_MINUTES_IN_MILISECONDS: + { + minCursorSize = 7; + maxCursorSize = 15; + } + break; + case _5_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5; + maxCursorSize = 15; + } + break; + case _4_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5; + maxCursorSize = 15; + } + break; + case _3_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5; + maxCursorSize = 15; + } + break; + case _2_MINUTES_IN_MILISECONDS: + { + minCursorSize = 5; + maxCursorSize = 15; + } + break; + case _1_MINUTE_IN_MILISECONDS: + { + minCursorSize = 5; + maxCursorSize = 15; + } + break; + default: + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> exitSavingMode -> err = " + err); } + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setTimePeriod -> err = " + err); } + } + } + + function setDatetime(pDatetime) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] setDatetime -> Entering function."); } + + if (pDatetime === undefined) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setDatetime -> Received undefined datetime."); } + return; + } + + cursorDate = pDatetime; + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] setDatetime -> err = " + err); } + } + } + + function reload(callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] reload -> Entering function."); } + + getFiles(callBackFunction); + + collectGarbage(callBackFunction); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] reload -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function getFiles(callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> Entering function."); } + + let i = 0; + let j = 0; + + let dateString; + + getNextFile(); + + function getNextFile() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> Entering function."); } + + let targetDate = new Date(cursorDate); + targetDate.setUTCDate(targetDate.getUTCDate() + j); + + /* Small algorith to allow load first the current date, then alternate between the most forwad and the most backwards ones. */ + if (j === 0) { j++; } + else { + if (j < 0) { + j = -j; + j++; + } + else { + j = -j; + } + } + + let compareTargetDate = new Date(targetDate.toDateString()); + + if (beginDateRange !== undefined) { + + let compareBeginDate = new Date(beginDateRange.toDateString()); + + if (compareTargetDate.valueOf() < compareBeginDate.valueOf()) { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> Not loading the file for this date."); } + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> compareTargetDate = " + compareTargetDate); } + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate < compareBeginDate -> compareBeginDate = " + compareBeginDate); } + + controlLoop(); + return; + } + } + + if (endDateRange !== undefined) { + + let compareEndDate = new Date(endDateRange.toDateString()); + + if (compareTargetDate.valueOf() > compareEndDate.valueOf()) { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> Not loading the file for this date."); } + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> compareTargetDate = " + compareTargetDate); } + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> getNextFile -> compareTargetDate > compareEndDate -> compareEndDate = " + compareEndDate); } + + controlLoop(); + return; + } + } + + dateString = targetDate.getUTCFullYear() + '-' + pad(targetDate.getUTCMonth() + 1, 2) + '-' + pad(targetDate.getUTCDate(), 2); + + let currentDay = Math.trunc((new Date()).valueOf() / (24 * 60 * 60 * 1000)); + let targetDay = Math.trunc(targetDate.valueOf() / (24 * 60 * 60 * 1000)); + + if (targetDay > currentDay) { + + controlLoop(); + + } else { + + if (thisObject.files.get(dateString) === undefined) { // We dont reload files we already have. + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, targetDate, undefined, undefined, onFileReceived); + + } else { + + controlLoop(); + } + + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> getNextFile -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function onFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> err.message = " + err.message); } + + if (err.message === "File does not exist.") { + + controlLoop(); + return; + + } else { + + callBackFunction(err); + return; + } + + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> onFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + thisObject.files.set(dateString, file); + + controlLoop(); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function controlLoop() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] getFiles -> controlLoop -> Entering function."); } + + if (callBackFunction !== undefined) { + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE); + + } + + i++; + + if (i < minCursorSize) { + + getNextFile(); + + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> controlLoop -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] getFiles -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function collectGarbage(callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] collectGarbage -> Entering function."); } + + date = removeTime(cursorDate); + + let minDate = date.valueOf() - maxCursorSize * ONE_DAY_IN_MILISECONDS / 2; + let maxDate = date.valueOf() + maxCursorSize * ONE_DAY_IN_MILISECONDS / 2; + + for (let key of thisObject.files.keys()) { + + let keyDate = new Date(key); + + if (keyDate.valueOf() < minDate || keyDate.valueOf() > maxDate) { + + thisObject.files.delete(key); + + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] collectGarbage -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + + } + + function getExpectedFiles() { + + if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } + + return minCursorSize; + + } + +} \ No newline at end of file diff --git a/FileSequence.js b/FileSequence.js new file mode 100644 index 000000000..5028143d4 --- /dev/null +++ b/FileSequence.js @@ -0,0 +1,385 @@ + +function newFileSequence() { + + const MODULE_NAME = "File Sequence"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + let thisObject = { + eventHandler: undefined, + getFile: getFile, + getExpectedFiles: getExpectedFiles, + getFilesLoaded: getFilesLoaded, + initialize: initialize, + finalize: finalize + } + + let filesLoaded = 0; + let fileCloud; + let files = new Map; + let maxSequence = -1; // This is replaced by the content of the sequence file, which contains an index that starts on zero. In the case that the sequence file is not found the default value is -1 so when you add 1 it gives you the amount of files in the sequence, zero. + let market; + let devTeam; + let bot; + let thisSet; + let product; + + thisObject.eventHandler = newEventHandler(); + + let intervalHandle; + + return thisObject; + + function finalize() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] finalize -> devTeam = " + devTeam.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] finalize -> bot = " + bot.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] finalize -> thisSet = " + thisSet.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] finalize -> intervalHandle = " + intervalHandle); } + + clearInterval(intervalHandle); + + filesLoaded = undefined; + fileCloud = undefined; + files = undefined; + maxSequence = undefined; + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } + } + } + + function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } + + exchange = ecosystem.getExchange(pProduct, pExchange); + + if (exchange === undefined) { + + throw "Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; + + } + + intervalHandle = setInterval(updateFiles, _1_MINUTE_IN_MILISECONDS); + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> intervalHandle = " + intervalHandle); } + + market = pMarket; + devTeam = pDevTeam; + bot = pBot; + thisSet = pSet; + product = pProduct; + + fileCloud = newFileCloud(); + fileCloud.initialize(bot); + + /* First we will get the sequence max number */ + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, "Sequence", undefined, onSequenceFileReceived); + + function onSequenceFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> key = " + devTeam.codeName + "-" + bot.codeName + "-" + product.codeName); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received CUSTOM OK Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> err.message = " + err.message); } + + callBackFunction(err); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> err.message = " + err.message); } + + callBackFunction(err, thisObject); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + maxSequence = Number(file); + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> maxSequence = " + maxSequence); } + + /* Now we will get the sequence of files */ + + for (let i = 0; i <= maxSequence; i++) { + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived); + + function onFileReceived(err, file) { + + try { + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> err.message = " + err.message); } + + callBackFunction(err); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onSequenceFileReceived -> onFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + files.set(i, file); + + filesLoaded++; + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject); // Note that the callback is called for every file loaded. + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onSequenceFileReceived -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onSequenceFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function updateFiles() { + + try { + + let updateFiles = 0; + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Entering function."); } + + /* + + To keep this data structure up-to-date we need to: + + 1. Re-read the sequence file. + 2. Re-load the current last file. + 3. Re-load all new files after our last one. + + */ + + let currentMaxSequence = maxSequence; + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> currentMaxSequence = " + currentMaxSequence); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> intervalHandle = " + intervalHandle); } + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> devTeam = " + devTeam.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> bot = " + bot.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> thisSet = " + thisSet.codeName); } + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, "Sequence", undefined, onSequenceFileReceived); + + function onSequenceFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> key = " + devTeam.codeName + "-" + bot.codeName + "-" + product.codeName); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received FAIL Response."); } + return; + } + + case GLOBAL.CUSTOM_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received CUSTOM OK Response."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> err.message = " + err.message); } + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> err.message = " + err.message); } + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> Received Unexpected Response."); } + return; + } + } + + maxSequence = Number(file); + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> maxSequence = " + maxSequence); } + + /* Now we will get the sequence of files, but in this case only from the currentMaxSequence and above. */ + + for (let i = currentMaxSequence; i <= maxSequence; i++) { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> i = " + i); } + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, undefined, undefined, i, undefined, onFileReceived); + + function onFileReceived(err, file) { + + try { + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received FAIL Response."); } + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> err.message = " + err.message); } + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Received Unexpected Response."); } + return; + } + } + + files.set(i, file); + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> File Updated."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> i = " + i); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> file.length = " + file.length); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> filesLoaded = " + filesLoaded); } + + if (i !== currentMaxSequence) { + + filesLoaded++; + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> filesLoaded = " + filesLoaded); } + + } + + if (filesLoaded === maxSequence + 1) { + + thisObject.eventHandler.raiseEvent("Files Updated", undefined); + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onSequenceFileReceived -> onFileReceived -> Files Updated event Raised."); } + + } + + } catch (err) { + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onSequenceFileReceived -> onFileReceived -> err = " + err); } + } + } + } + + } catch (err) { + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onSequenceFileReceived -> err = " + err); } + } + } + + } catch (err) { + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> err = " + err); } + } + } + + function getFile(pSequence) { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] getFile -> pSequence = " + pSequence); } + + return files.get(pSequence); + + } + + function getExpectedFiles() { + + if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } + + return maxSequence + 1; + + } + + function getFilesLoaded() { + + if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> Entering function."); } + + return filesLoaded; + + } + +} \ No newline at end of file diff --git a/MarketFiles.js b/MarketFiles.js new file mode 100644 index 000000000..0bbf6d116 --- /dev/null +++ b/MarketFiles.js @@ -0,0 +1,253 @@ + +function newMarketFiles() { + + const MODULE_NAME = "Market Files"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + let thisObject = { + eventHandler: undefined, + getFile: getFile, + getExpectedFiles: getExpectedFiles, + getFilesLoaded: getFilesLoaded, + initialize: initialize, + finalize: finalize + } + + let filesLoaded = 0; + + let fileCloud; + + let files = new Map; + + let market; + let exchange; + let devTeam; + let bot; + let thisSet; + let periodName; + + thisObject.eventHandler = newEventHandler(); + + let intervalHandle; + + return thisObject; + + function finalize() { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] finalize -> Entering function."); } + + clearInterval(intervalHandle); + + filesLoaded = undefined; + files = undefined; + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] finalize -> err = " + err); } + } + } + + function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } + + exchange = ecosystem.getExchange(pProduct, pExchange); + + if (exchange === undefined) { + + throw "Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; + + } + + intervalHandle = setInterval(updateFiles, _10_MINUTES_IN_MILISECONDS); + + market = pMarket; + devTeam = pDevTeam; + bot = pBot; + thisSet = pSet; + + fileCloud = newFileCloud(); + fileCloud.initialize(pBot); + + /* Now we will get the market files */ + + for (let i = 0; i < marketFilesPeriods.length; i++) { + + let periodTime = marketFilesPeriods[i][0]; + let periodName = marketFilesPeriods[i][1]; + + if (thisSet.validPeriods.includes(periodName) === true) { + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived); + + function onFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> err.message = " + err.message); } + + callBackFunction(err); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + files.set(periodTime, file); + + filesLoaded++; + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE, thisObject); // Note that the callback is called for every file loaded. + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + } + } + + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function updateFiles() { + + try { + + let updateFiles = 0; + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> Entering function."); } + + /* Now we will get the market files */ + + for (let i = 0; i < marketFilesPeriods.length; i++) { + + let periodTime = marketFilesPeriods[i][0]; + let periodName = marketFilesPeriods[i][1]; + + if (thisSet.validPeriods.includes(periodName) === true) { + + fileCloud.getFile(devTeam, bot, thisSet, exchange, market, periodName, undefined, undefined, undefined, onFileReceived); + + function onFileReceived(err, file) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received FAIL Response."); } + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> err.message = " + err.message); } + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> Received Unexpected Response."); } + return; + } + } + + files.set(periodTime, file); + updateFiles++; + + if (updateFiles === marketFilesPeriods.length) { + + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> All files received. "); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> devTeam = " + devTeam.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> bot = " + bot.codeName); } + if (INFO_LOG === true) { logger.write("[INFO] updateFiles -> onFileReceived -> thisSet = " + thisSet.codeName); } + + thisObject.eventHandler.raiseEvent("Files Updated", undefined); + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> onFileReceived -> err = " + err); } + } + } + } + } + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] updateFiles -> err = " + err); } + } + } + + function getFile(pPeriod) { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } + + return files.get(pPeriod); + + } + + function getExpectedFiles() { + + if (INFO_LOG === true) { logger.write("[INFO] getExpectedFiles -> Entering function."); } + + return marketFilesPeriods.length; + + } + + function getFilesLoaded() { + + if (INFO_LOG === true) { logger.write("[INFO] getFilesLoaded -> Entering function."); } + + return filesLoaded; + + } + +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 000000000..b6abe364f --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +# Files Component + +The Files Component provides the functionality needed to load into the browser's memory data structures populated from plain text files grabbed from the cloud and optimized for different scenarios. Each of these scenarios are represented by a single js module as listed below. + +Once the information is initially loaded, it is automatically updated as times goes by or as the users changes its position on the timeline, both on the time dimension as in the time period (1m, 2m, 3m, 4m, 5m, 10m, ..., 1 hour, 2 hours, ..., 24 hs) dimension. + +## Single File + +As the name suggests, this is used to load a single file. + +## File Sequence + +This is a multi-file data structure in which is stored information of a sequence of files. Let's imagine that the App using this component needs to read File_1, File_2, File_3 and File_4, this data structure is capable of doing that. + +## Market File + +The system-wide concept of a Market File is defined as a file containing data corresponding to the whole market history. If the data were candles for instance, it would be containing all candles since the beginning on the market until the head of the market (the most recent information). + +Since for each market there are multiple Market Files (one for 1 hour time period, one for 2 hours, 4 hours, 6, 8, 12 and 24 hs.) this data structure loads all of these files and have them readily available in memory for consumption. + +## Daily Files + +The system-wide concept of a Daily File is defined as a file containing data corresponding only to one single day. If the data were candles for instance, it would be containing all candles calculated from trades that happened in a single day only. + +Since for each market there are multiple Daily Files (one for 1 min time period, one for 2 min, 3 min, 4, 5, 10, 15, 20, 30 and 45 minutes) this data structure loads all of these files and have them readily available in memory for consumption. + +Moreover, it uses the concept of a File Cursor to load more files of the previous days and the days following days in order to have them ready in memory if needed. diff --git a/SingleFile.js b/SingleFile.js new file mode 100644 index 000000000..65c8d8472 --- /dev/null +++ b/SingleFile.js @@ -0,0 +1,105 @@ + +function newSingleFile() { + + const MODULE_NAME = "Single File"; + const INFO_LOG = false; + const ERROR_LOG = true; + const logger = newWebDebugLog(); + logger.fileName = MODULE_NAME; + + let thisObject = { + getFile: getFile, + initialize: initialize + } + + let fileCloud; + let file; + + return thisObject; + + function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> key = " + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName); } + + let exchange = ecosystem.getExchange(pProduct, pExchange); + + if (pExchange !== undefined && exchange === undefined) { // We support no exchange as a parameter, but if provided, then it must be at the list of exchanges of that product. + + throw "Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange; + + } + + fileCloud = newFileCloud(); + fileCloud.initialize(pBot); + + /* Now we will get the file */ + + fileCloud.getFile(pDevTeam, pBot, pSet, exchange, pMarket, undefined, undefined, undefined, undefined, onFileReceived); + + function onFileReceived(err, pFile) { + + try { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Entering function."); } + + switch (err.result) { + case GLOBAL.DEFAULT_OK_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received OK Response."); } + break; + } + + case GLOBAL.DEFAULT_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received FAIL Response."); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + return; + } + + case GLOBAL.CUSTOM_FAIL_RESPONSE.result: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received CUSTOM FAIL Response."); } + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> err.message = " + err.message); } + + callBackFunction(err); + return; + } + + default: { + + if (INFO_LOG === true) { logger.write("[INFO] initialize -> onFileReceived -> Received Unexpected Response."); } + callBackFunction(err); + return; + } + } + + file = pFile; + + callBackFunction(GLOBAL.DEFAULT_OK_RESPONSE); + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> onFileReceived -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + } catch (err) { + + if (ERROR_LOG === true) { logger.write("[ERROR] initialize -> err = " + err); } + callBackFunction(GLOBAL.DEFAULT_FAIL_RESPONSE); + } + } + + function getFile() { + + if (INFO_LOG === true) { logger.write("[INFO] getFile -> Entering function."); } + + return file; + + } + +} \ No newline at end of file