Storage and downwards ready

AAWebPlatform.njsproj

@@ -113,6 +113,9 @@
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="SpreadChart.js" />
+    <Compile Include="Storage.js">
+      <SubType>Code</SubType>
+    </Compile>
     <Compile Include="temp.js" />
     <Compile Include="TempExosystem.js">
       <SubType>Code</SubType>

File.js

@@ -1,5 +1,5 @@
 
-function newFile() {
+function newFile() {  
 
     let fileObject = {
         getFile: getFile,
@@ -11,38 +11,22 @@ function newFile() {
 
     return fileObject;
 
-    function initialize(pTeamCodeName, pBotCodeName, pProductCodeName, pSetCodeName, pExchange, pMarket, callBackFunction) {
+    function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) {
 
-        let product = ecosystem.getProduct(ecosystem.getBot(ecosystem.getTeam(pTeamCodeName), pBotCodeName), pProductCodeName);
-
-        if (product === undefined) {
-
-            throw "Product not found at this bot of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName;
-
-        }
-
-        let exchange = ecosystem.getExchange(product, pExchange);
+        let exchange = ecosystem.getExchange(pProduct, pExchange);
 
         if (exchange === undefined) {
 
-            throw "Exchange not supoorted by this product of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName + ", pExchange = " + pExchange;
-
-        }
-
-        let productSet = ecosystem.getSet(product, pSetCodeName);
-
-        if (productSet === undefined) {
-
-            throw "Set not found at this product of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName + ", pSetCodeName = " + pSetCodeName;
+            throw "Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange;
 
         }
 
         fileCloud = newFileCloud();
-        fileCloud.initialize(pTeamCodeName, pBotCodeName);
+        fileCloud.initialize(pBot);
 
         /* Now we will get the file */
 
-        fileCloud.getFile(productSet, exchange, pMarket, undefined, undefined, onFileReceived);
+        fileCloud.getFile(pSet, exchange, pMarket, undefined, undefined, onFileReceived);
 
         function onFileReceived(pFile) {
 

FileCache.js

@@ -3,9 +3,13 @@ function newFileCache() {
 
     let fileCache = {
         getFile: getFile,
+        getExpectedFiles: getExpectedFiles,
+        getFilesLoaded: getFilesLoaded,
         initialize: initialize
     }
 
+    let filesLoaded = 0;
+
     let fileCloud;
 
     let files = new Map;
@@ -14,34 +18,18 @@ function newFileCache() {
     
     return fileCache;
 
-    function initialize(pTeamCodeName, pBotCodeName, pProductCodeName, pSetCodeName, pExchange, pMarket, callBackFunction) {
+    function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, callBackFunction) {
 
-        let product = ecosystem.getProduct(ecosystem.getBot(ecosystem.getTeam(pTeamCodeName), pBotCodeName), pProductCodeName);
-
-        if (product === undefined) {
-
-            throw "Product not found at this bot of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName;
-
-        }
-
-        let exchange = ecosystem.getExchange(product, pExchange);
+        let exchange = ecosystem.getExchange(pProduct, pExchange);
 
         if (exchange === undefined) {
 
-            throw "Exchange not supoorted by this product of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName + ", pExchange = " + pExchange;
-
-        }
-
-        let productSet = ecosystem.getSet(product, pSetCodeName);
-
-        if (productSet === undefined) {
-
-            throw "Set not found at this product of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName + ", pSetCodeName = " + pSetCodeName;
+            throw "Exchange not supoorted by this pProduct of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange;
 
         }
 
         fileCloud = newFileCloud();
-        fileCloud.initialize(pTeamCodeName, pBotCodeName);
+        fileCloud.initialize(pBot);
 
         /* Now we will get the market files */
 
@@ -50,15 +38,17 @@ function newFileCache() {
             let periodTime = marketFilesPeriods[i][0];
             let periodName = marketFilesPeriods[i][1];
 
-            if (productSet.validPeriods.includes(periodName) === true) {
+            if (pSet.validPeriods.includes(periodName) === true) {
 
-                fileCloud.getFile(productSet, exchange, pMarket, periodName, undefined, onFileReceived);
+                fileCloud.getFile(pSet, exchange, pMarket, periodName, undefined, onFileReceived);
 
                 function onFileReceived(file) {
 
                     files.set(periodTime, file);
 
-                    callBackFunction(); // Note that the call back is called for every file loaded.
+                    filesLoaded++;
+
+                    callBackFunction(); // Note that the callback is called for every file loaded.
 
                 }
             }
@@ -71,4 +61,16 @@ function newFileCache() {
 
     }
 
+    function getExpectedFiles() {
+
+        return marketFilesPeriods.length;
+
+    }
+
+    function getFilesLoaded() {
+
+        return filesLoaded;
+
+    }
+
 }

FileCloud.js

@@ -10,11 +10,9 @@ function newFileCloud() {
 
     return fileCloud;
 
-    function initialize(pTeamCodeName, pBotCodeName) {
+    function initialize(pBot) {
 
-        let bot = ecosystem.getBot(ecosystem.getTeam(pTeamCodeName), pBotCodeName);
-
-        fileService = AzureStorage.createFileServiceWithSas(bot.storage.fileUri, bot.storage.sas).withFilter(new AzureStorage.ExponentialRetryPolicyFilter());
+        fileService = AzureStorage.createFileServiceWithSas(pBot.storage.fileUri, pBot.storage.sas).withFilter(new AzureStorage.ExponentialRetryPolicyFilter());
 
     }
 
@@ -79,10 +77,4 @@ function newFileCloud() {
 
         }
     }
-
-
-}
-
-
-
-
+}

FileCursor.js

@@ -8,6 +8,7 @@ function newFileCursor() {
         setDatetime: setDatetime,
         setTimePeriod: setTimePeriod,
         files: undefined,
+        getExpectedFiles: getExpectedFiles,
         initialize: initialize
     }
 
@@ -19,18 +20,18 @@ function newFileCursor() {
     let market;
     let exchange;
     let fileCloud;
-    let productSet;
+    let thisSet;
     let periodName;
     let timePeriod;
 
     return fileCursor;
 
-    function initialize(pFileCloud, pProductSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, callBackFunction) {
+    function initialize(pFileCloud, pSet, pExchange, pMarket, pPeriodName, pTimePeriod, pCursorDate, pCurrentTimePeriod, callBackFunction) {
 
         market = pMarket;
         exchange = pExchange;
         fileCloud = pFileCloud;
-        productSet = pProductSet;
+        thisSet = pSet;
         periodName = pPeriodName;
         cursorDate = pCursorDate;
         timePeriod = pTimePeriod;
@@ -289,7 +290,7 @@ function newFileCursor() {
 
                 if (fileCursor.files.get(dateString) === undefined) { // We dont reload files we already have. 
 
-                    fileCloud.getFile(productSet, exchange, market, periodName, targetDate, onFileReceived);
+                    fileCloud.getFile(thisSet, exchange, market, periodName, targetDate, onFileReceived);
 
                 } else {
 
@@ -349,4 +350,10 @@ function newFileCursor() {
 
     }
 
+    function getExpectedFiles() {
+
+        return minCursorSize;
+
+    }
+
 }

FileCursorCache.js

@@ -5,9 +5,14 @@ function newFileCursorCache() {
         getFileCursor: getFileCursor,
         setDatetime: setDatetime,
         setTimePeriod: setTimePeriod,
+        getExpectedFiles: getExpectedFiles,
+        getFilesLoaded: getFilesLoaded,
         initialize: initialize
     }
 
+    let filesLoaded = 0;
+    let expectedFiles = 0;
+
     let fileCloud;
 
     let marketFiles = new Map;
@@ -15,34 +20,18 @@ function newFileCursorCache() {
 
     return fileCursorCache;
 
-    function initialize(pTeamCodeName, pBotCodeName, pProductCodeName, pSetCodeName, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) {
+    function initialize(pDevTeam, pBot, pProduct, pSet, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) {
 
-        let product = ecosystem.getProduct(ecosystem.getBot(ecosystem.getTeam(pTeamCodeName), pBotCodeName), pProductCodeName);
-
-        if (product === undefined) {
-
-            throw "Product not found at this bot of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName;
-
-        }
-
-        let exchange = ecosystem.getExchange(product, pExchange);
+        let exchange = ecosystem.getExchange(pProduct, pExchange);
 
         if (exchange === undefined) {
 
-            throw "Exchange not supoorted by this product of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName + ", pExchange = " + pExchange;
-
-        }
-
-        let productSet = ecosystem.getSet(product, pSetCodeName);
-
-        if (productSet === undefined) {
-
-            throw "Set not found at this product of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName + ", pSetCodeName = " + pSetCodeName;
+            throw "Exchange not supoorted by this product of the ecosystem! - pDevTeam.codeName = " + pDevTeam.codeName + ", pBot.codeName = " + pBot.codeName + ", pProduct.codeName = " + pProduct.codeName + ", pExchange = " + pExchange;
 
         }
 
         fileCloud = newFileCloud();
-        fileCloud.initialize(pTeamCodeName, pBotCodeName);
+        fileCloud.initialize(pBot);
 
         /* Now we will get the daily files */
 
@@ -51,15 +40,18 @@ function newFileCursorCache() {
             let periodTime = dailyFilePeriods[i][0];
             let periodName = dailyFilePeriods[i][1];
 
-            if (productSet.validPeriods.includes(periodName) === true) {
+            if (pSet.validPeriods.includes(periodName) === true) {
 
                 let fileCursor = newFileCursor();
-                fileCursor.initialize(fileCloud, productSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, onFileReceived);
+                fileCursor.initialize(fileCloud, pSet, exchange, pMarket, periodName, periodTime, pDatetime, pTimePeriod, onFileReceived);
 
                 fileCursors.set(periodTime, fileCursor);
 
+                expectedFiles = expectedFiles + fileCursor.getExpectedFiles();
+
                 function onFileReceived() {
 
+                    filesLoaded++;
                     callBackFunction(); // Note that the call back is called for every file loaded at each cursor.
 
                 }
@@ -94,4 +86,16 @@ function newFileCursorCache() {
 
         }
     }
+
+    function getExpectedFiles() {
+
+        return expectedFiles;
+
+    }
+
+    function getFilesLoaded() {
+
+        return filesLoaded;
+
+    }
 }

Storage.js

@@ -0,0 +1,133 @@
+
+function newStorage() {
+
+    /*
+
+    This object will initialize children objects that will end up loading the data of each set defined at each product of the bot received at initialization.
+    Once all the underlaying objects are fully initialized it will callback.
+
+    At the same time it will raise an event for each underlaying file being loaded, so that the UI can reflect the progress to the end user. 
+
+    */
+
+    thisObject = {
+
+        fileCache: undefined,
+        fileCursorCache: undefined,
+        file: undefined,
+
+        eventHandler: undefined,
+        initialize: initialize
+
+    }
+
+    thisObject.eventHandler = newEventHandler();
+            
+    return thisObject;
+
+}
+
+function initialize(pDevTeam, pBot, pProduct, pExchange, pMarket, pDatetime, pTimePeriod, callBackFunction) {
+
+    /* We name the event Handler to easy debugging. */
+
+    thisObject.eventHandler.name = "Storage" + pDevTeam.codeName + "-" + pBot.codeName + "-" + pProduct.codeName
+
+    let dataSetsToLoad = 0;
+    let dataSetsLoaded = 0;
+
+    for (i = 0; i < pProduct.sets.length; i++) {
+
+        let thisSet = pProduct.sets[i];
+
+        switch (thisSet.type) {
+            case 'Market Files': {
+
+                thisObject.fileCache = newFileCache();
+                thisObject.fileCache.initialize(pDevTeam, pBot, pProduct, thisSet, pExchange, pMarket, onCacheFileReady);
+                dataSetsToLoad++;
+
+            }
+                break;
+
+            case 'Daily Files': {
+
+                thisObject.fileCursorCache = newFileCursorCache();
+                thisObject.fileCursorCache.initialize(pDevTeam, pBot, pProduct, thisSet, pExchange, pMarket, pDatetime, pTimePeriod, onFileCursorReady);
+                dataSetsToLoad++;
+
+            }
+                break;
+
+            case 'Single File': {
+
+                thisObject.file = newFile();
+                thisObject.file.initialize(pDevTeam, pBot, pProduct, thisSet, pExchange, pMarket, onSingleFileReady);
+                dataSetsToLoad++;
+
+            }
+                break;
+
+        }
+    }
+
+    function onCacheFileReady() {
+
+        let event = {
+            totalValue: thisObject.fileCache.getExpectedFiles(),
+            currentValue: thisObject.fileCache.getFilesLoaded()
+        }
+
+        thisObject.eventHandler.raiseEvent('Storage File Ready', event);
+
+        if (event.currentValue === event.totalValue) {
+
+            dataSetsLoaded++;
+
+            checkInitializeComplete();
+        }
+    }
+
+    function onFileCursorReady() {
+
+        let event = {
+            totalValue: thisObject.fileCursorCache.getExpectedFiles(),
+            currentValue: thisObject.fileCursorCache.getFilesLoaded()
+        }
+
+        thisObject.eventHandler.raiseEvent('Storage File Ready', event);
+
+        if (event.currentValue === event.totalValue) {
+
+            dataSetsLoaded++;
+
+            checkInitializeComplete();
+        }
+    }
+
+    function onSingleFileReady() {
+
+        let event = {
+            totalValue: 1,
+            currentValue: 1
+        }
+
+        thisObject.eventHandler.raiseEvent('Storage File Ready', event);
+
+        if (event.currentValue === event.totalValue) {
+
+            dataSetsLoaded++;
+
+            checkInitializeComplete();
+        }
+    }
+
+    function checkInitializeComplete() {
+
+        if (dataSetsLoaded === dataSetsToLoad) {
+
+            callBackFunction();
+
+        }
+    }
+}

ecosystem.json

@@ -37,13 +37,13 @@
               "storageAccount": "aaolivia",
               "sets": [
                 {
-                  "codeName": "Market Files",
+                  "type": "Market Files",
                   "validPeriods": [ "24-hs", "12-hs", "08-hs", "06-hs", "04-hs", "03-hs", "02-hs", "01-hs" ],
                   "filePath": "@Exchange/Output/Candles/Multi-Period-Market/@Period",
                   "fileName": "@AssetA_@AssetB.json"
                 },
                 {
-                  "codeName": "Daily Files",
+                  "type": "Daily Files",
                   "validPeriods": [ "45-min", "40-min", "30-min", "20-min", "15-min", "10-min", "05-min", "04-min", "03-min", "02-min", "01-min" ],
                   "filePath": "@Exchange/Output/Candles/Multi-Period-Daily/@Period/@Year/@Month/@Day",
                   "fileName": "@AssetA_@AssetB.json"
@@ -68,13 +68,13 @@
               "storageAccount": "aaolivia",
               "sets": [
                 {
-                  "codeName": "Market Files",
+                  "type": "Market Files",
                   "validPeriods": [ "24-hs", "12-hs", "08-hs", "06-hs", "04-hs", "03-hs", "02-hs", "01-hs" ],
                   "filePath": "@Exchange/Output/Volumes/Multi-Period-Market/@Period",
                   "fileName": "@AssetA_@AssetB.json"
                 },
                 {
-                  "codeName": "Daily Files",
+                  "type": "Daily Files",
                   "validPeriods": [ "45-min", "40-min", "30-min", "20-min", "15-min", "10-min", "05-min", "04-min", "03-min", "02-min", "01-min" ],
                   "filePath": "@Exchange/Output/Volumes/Multi-Period-Daily/@Period/@Year/@Month/@Day",
                   "fileName": "@AssetA_@AssetB.json"
@@ -113,13 +113,13 @@
               "storageAccount": "aatom",
               "sets": [
                 {
-                  "codeName": "Market Files",
+                  "type": "Market Files",
                   "validPeriods": [ "24-hs", "12-hs", "08-hs", "06-hs", "04-hs", "03-hs", "02-hs", "01-hs" ],
                   "filePath": "@Exchange/Tom/dataSet.V1/Output/Candle-Stairs/Multi-Period-Market/@Period",
                   "fileName": "@AssetA_@AssetB.json"
                 },
                 {
-                  "codeName": "Daily Files",
+                  "type": "Daily Files",
                   "validPeriods": [ "45-min", "40-min", "30-min", "20-min", "15-min", "10-min", "05-min", "04-min", "03-min", "02-min", "01-min" ],
                   "filePath": "@Exchange/Tom/dataSet.V1/Output/Candle-Stairs/Multi-Period-Daily/@Period/@Year/@Month/@Day",
                   "fileName": "@AssetA_@AssetB.json"
@@ -143,13 +143,13 @@
               "storageAccount": "aatom",
               "sets": [
                 {
-                  "codeName": "Market Files",
+                  "type": "Market Files",
                   "validPeriods": [ "24-hs", "12-hs", "08-hs", "06-hs", "04-hs", "03-hs", "02-hs", "01-hs" ],
                   "filePath": "@Exchange/Tom/dataSet.V1/Output/Volume-Stairs/Multi-Period-Market/@Period",
                   "fileName": "@AssetA_@AssetB.json"
                 },
                 {
-                  "codeName": "Daily Files",
+                  "type": "Daily Files",
                   "validPeriods": [ "45-min", "40-min", "30-min", "20-min", "15-min", "10-min", "05-min", "04-min", "03-min", "02-min", "01-min" ],
                   "filePath": "@Exchange/Tom/dataSet.V1/Output/Volume-Stairs/Multi-Period-Daily/@Period/@Year/@Month/@Day",
                   "fileName": "@AssetA_@AssetB.json"
@@ -188,7 +188,7 @@
               "storageAccount": "aamariam",
               "sets": [
                 {
-                  "codeName": "Trading History File",
+                  "type": "Single File",
                   "validPeriods": [ "24-hs", "12-hs", "08-hs", "06-hs", "04-hs", "03-hs", "02-hs", "01-hs", "45-min", "40-min", "30-min", "20-min", "15-min", "10-min", "05-min", "04-min", "03-min", "02-min", "01-min" ],
                   "filePath": "@Exchange/Mariam/dataSet.V1/Output/Trading-Process",
                   "fileName": "Execution.History.json"