fez5587/Superalgos
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

File Cursos Cache Implemented

Browse Source
This commit is contained in:
Luis Fernando Molina committed 2018-02-20 12:10:52 +01:00
1 parent 89f8e0574f
commit 4241499f63
7 files changed
+246 -90
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

AAWebPlatform.njsproj
+3
View File
@@ -45,6 +45,9 @@
<Compile Include="FileCursor.js">
<SubType>Code</SubType>
</Compile>
<Compile Include="FileCursorCache.js">
<SubType>Code</SubType>
</Compile>
<Compile Include="Panels\BotsLayersPanel.js">
<SubType>Code</SubType>
</Compile>
Dashboard.js
+29
View File
@@ -5,6 +5,35 @@ var canvas;
var markets;
let ecosystem = newEcosystem();
let marketFilesPeriods =
'[' +
'[' + 24 * 60 * 60 * 1000 + ',' + '"24-hs"' + ']' + ',' +
'[' + 12 * 60 * 60 * 1000 + ',' + '"12-hs"' + ']' + ',' +
'[' + 8 * 60 * 60 * 1000 + ',' + '"08-hs"' + ']' + ',' +
'[' + 6 * 60 * 60 * 1000 + ',' + '"06-hs"' + ']' + ',' +
'[' + 4 * 60 * 60 * 1000 + ',' + '"04-hs"' + ']' + ',' +
'[' + 3 * 60 * 60 * 1000 + ',' + '"03-hs"' + ']' + ',' +
'[' + 2 * 60 * 60 * 1000 + ',' + '"02-hs"' + ']' + ',' +
'[' + 1 * 60 * 60 * 1000 + ',' + '"01-hs"' + ']' + ']';
marketFilesPeriods = JSON.parse(marketFilesPeriods);
let dailyFilePeriods =
'[' +
'[' + 45 * 60 * 1000 + ',' + '"45-min"' + ']' + ',' +
'[' + 40 * 60 * 1000 + ',' + '"40-min"' + ']' + ',' +
'[' + 30 * 60 * 1000 + ',' + '"30-min"' + ']' + ',' +
'[' + 20 * 60 * 1000 + ',' + '"20-min"' + ']' + ',' +
'[' + 15 * 60 * 1000 + ',' + '"15-min"' + ']' + ',' +
'[' + 10 * 60 * 1000 + ',' + '"10-min"' + ']' + ',' +
'[' + 05 * 60 * 1000 + ',' + '"05-min"' + ']' + ',' +
'[' + 04 * 60 * 1000 + ',' + '"04-min"' + ']' + ',' +
'[' + 03 * 60 * 1000 + ',' + '"03-min"' + ']' + ',' +
'[' + 02 * 60 * 1000 + ',' + '"02-min"' + ']' + ',' +
'[' + 01 * 60 * 1000 + ',' + '"01-min"' + ']' + ']';
dailyFilePeriods = JSON.parse(dailyFilePeriods);
var viewPort = newViewPort();
Ecosystem.js
+79 -5
View File
@@ -50,7 +50,7 @@ function newEcosystem() {
},
products: {
product: [{
codeName: "Candles",
codeName: "Market Candles",
displayName: "Market Candles per Period",
description: "A set of files per period that contains all candles of one whole market.",
storageAccount: "aaolivia",
@@ -67,7 +67,7 @@ function newEcosystem() {
},
layers: {
layer: [{
codeName: "Candlesticks",
codeName: "Market Candlesticks",
displayName: "Candlesticks",
exchangeList: ["Poloniex"],
validPeriods: ["24-hs", "12-hs", "06-hs", "03-hs", "01-hs"],
@@ -87,7 +87,7 @@ function newEcosystem() {
}]
}
}, {
codeName: "Volumes",
codeName: "Market Volumes",
displayName: "Market Volumes per Period",
description: "A set of files per period that contains all volumes of one whole market.",
storageAccount: "aaolivia",
@@ -104,7 +104,7 @@ function newEcosystem() {
},
layers: {
layer: [{
codeName: "Volumes",
codeName: "Market Volumes",
displayName: "Volumes",
exchangeList: ["Poloniex"],
validPeriods: ["24-hs", "12-hs", "06-hs", "03-hs", "01-hs"],
@@ -123,7 +123,81 @@ function newEcosystem() {
}
}]
}
}]
}, {
codeName: "Daily Candles",
displayName: "Daily Candles per Period",
description: "A set of files per period that contains all candles of one whole market.",
storageAccount: "aaolivia",
filePath: "@Exchange/Output/Candles/Multi-Period-Daily/@Period/@Year/@Month/@Day",
fileName: "@AssetA_@AssetB.json",
exchangeList: {
exchange: [{
name: "Poloniex",
datetimeRange: {
minDatetime: "2017-02-19 19:15:00.000 GMT+0000",
maxDatetime: "2018-02-13 10:10:00.000 GMT+0000"
}
}]
},
layers: {
layer: [{
codeName: "Daily Candlesticks",
displayName: "Candlesticks",
exchangeList: ["Poloniex"],
validPeriods: ["30-min", "15-min", "10-min", "05-min", "01-min"],
plotter: {
devTeam: "AAMasters",
repo: "AAOlivia-Plotter",
moduleName: "Candlesticks",
target: "Timeline",
dependencies: {
dependency: {
devTeam: "AAMasters",
repo: "Plotters-Utilities",
moduleName: "Candlesticks"
}
}
}
}]
}
}, {
codeName: "Daily Volumes",
displayName: "Daily Volumes per Period",
description: "A set of files per period that contains all volumes of one whole market.",
storageAccount: "aaolivia",
filePath: "@Exchange/Output/Candles/Multi-Period-Daily/@Period/@Year/@Month/@Day",
fileName: "@AssetA_@AssetB.json",
exchangeList: {
exchange: [{
name: "Poloniex",
datetimeRange: {
minDatetime: "2017-02-19 19:15:00.000 GMT+0000",
maxDatetime: "2018-02-13 10:10:00.000 GMT+0000"
}
}]
},
layers: {
layer: [{
codeName: "Daily Volumes",
displayName: "Volumes",
exchangeList: ["Poloniex"],
validPeriods: ["30-min", "15-min", "10-min", "05-min", "01-min"],
plotter: {
devTeam: "AAMasters",
repo: "AAOlivia-Plotter",
moduleName: "Volumes",
target: "Timeline",
dependencies: {
dependency: {
devTeam: "AAMasters",
repo: "Plotters-Utilities",
moduleName: "Volumes"
}
}
}
}]
}
}]
},
processes:
{
FileCache.js
+9 -53
View File
@@ -1,52 +1,20 @@
function newFileCache() {
let marketFilesPeriods =
'[' +
'[' + 24 * 60 * 60 * 1000 + ',' + '"24-hs"' + ']' + ',' +
'[' + 12 * 60 * 60 * 1000 + ',' + '"12-hs"' + ']' + ',' +
'[' + 8 * 60 * 60 * 1000 + ',' + '"08-hs"' + ']' + ',' +
'[' + 6 * 60 * 60 * 1000 + ',' + '"06-hs"' + ']' + ',' +
'[' + 4 * 60 * 60 * 1000 + ',' + '"04-hs"' + ']' + ',' +
'[' + 3 * 60 * 60 * 1000 + ',' + '"03-hs"' + ']' + ',' +
'[' + 2 * 60 * 60 * 1000 + ',' + '"02-hs"' + ']' + ',' +
'[' + 1 * 60 * 60 * 1000 + ',' + '"01-hs"' + ']' + ']';
marketFilesPeriods = JSON.parse(marketFilesPeriods);
let dailyFilePeriods =
'[' +
'[' + 45 * 60 * 1000 + ',' + '"45-min"' + ']' + ',' +
'[' + 40 * 60 * 1000 + ',' + '"40-min"' + ']' + ',' +
'[' + 30 * 60 * 1000 + ',' + '"30-min"' + ']' + ',' +
'[' + 20 * 60 * 1000 + ',' + '"20-min"' + ']' + ',' +
'[' + 15 * 60 * 1000 + ',' + '"15-min"' + ']' + ',' +
'[' + 10 * 60 * 1000 + ',' + '"10-min"' + ']' + ',' +
'[' + 05 * 60 * 1000 + ',' + '"05-min"' + ']' + ',' +
'[' + 04 * 60 * 1000 + ',' + '"04-min"' + ']' + ',' +
'[' + 03 * 60 * 1000 + ',' + '"03-min"' + ']' + ',' +
'[' + 02 * 60 * 1000 + ',' + '"02-min"' + ']' + ',' +
'[' + 01 * 60 * 1000 + ',' + '"01-min"' + ']' + ']';
dailyFilePeriods = JSON.parse(dailyFilePeriods);
let fileCache = {
getMarketFile: getMarketFile,
getFileCursor: getFileCursor,
setDatetime: setDatetime,
getFile: getFile,
initialize: initialize
}
let fileCloud;
let marketFiles = new Map;
let fileCursors = new Map;
let files = new Map;
let datetime;
return fileCache;
function initialize(pTeamCodeName, pBotCodeName, pProductCodeName, pLayerCodeName, pExchange, pMarket, pDatetime, callBackFunction) {
function initialize(pTeamCodeName, pBotCodeName, pProductCodeName, pLayerCodeName, pExchange, pMarket, callBackFunction) {
let product = ecosystem.getProduct(ecosystem.getBot(ecosystem.getTeam(pTeamCodeName), pBotCodeName), pProductCodeName);
@@ -75,6 +43,8 @@ function newFileCache() {
fileCloud = newFileCloud();
fileCloud.initialize(pTeamCodeName, pBotCodeName);
/* Now we will get the market files */
for (i = 0; i < marketFilesPeriods.length; i++) {
let periodTime = marketFilesPeriods[i][0];
@@ -82,11 +52,11 @@ function newFileCache() {
if (layer.validPeriods.includes(periodName) === true) {
fileCloud.getMarketFile(product, exchange, pMarket, periodName, onFileReceived);
fileCloud.getFile(product, exchange, pMarket, periodName, undefined, onFileReceived);
function onFileReceived(file) {
marketFiles.set(periodTime, file);
files.set(periodTime, file);
callBackFunction(); // Note that the call back is called for every file loaded.
@@ -94,24 +64,10 @@ function newFileCache() {
}
}
}
function getMarketFile(pPeriod) {
function getFile(pPeriod) {
return marketFiles.get(pPeriod);
}
function getFileCursor(pPeriod) {
return fileCursors.get(pPeriod);
}
function setDatetime(newDatetime) {
datetime = newDatetime;
return files.get(pPeriod);
}
FileCloud.js
+12 -2
View File
@@ -2,7 +2,7 @@
function newFileCloud() {
var fileCloud = {
getMarketFile: getMarketFile,
getFile: getFile,
initialize: initialize
};
@@ -18,7 +18,7 @@ function newFileCloud() {
}
function getMarketFile(pProduct, pExchange, pMarket, pPeriodName, callBackFunction) {
function getFile(pProduct, pExchange, pMarket, pPeriodName, pDatetime, callBackFunction) {
let fileName = pProduct.fileName;
let filePath = pProduct.filePath;
@@ -29,6 +29,14 @@ function newFileCloud() {
filePath = filePath.replace("@Exchange", pExchange.name);
filePath = filePath.replace("@Period", pPeriodName);
if (pDatetime !== undefined) {
filePath = filePath.replace("@Year", pDatetime.getUTCFullYear());
filePath = filePath.replace("@Month", pad(pDatetime.getUTCMonth() + 1, 2));
filePath = filePath.replace("@Month", pad(pDatetime.getUTCDate(), 2));
}
fileService.getFileToText('data', filePath, fileName, undefined, onFileReceived);
function onFileReceived(err, text, response) {
@@ -56,6 +64,8 @@ function newFileCloud() {
}
}
}
FileCursor.js
+25 -30
View File
@@ -1,33 +1,35 @@
function newFileCursor() {
let dailyFiles = new Map;
let cursorDate = new Date(INITIAL_DATE);
let files = new Map;
let cursorDate;
let fileCursor = {
setDatetime: setDatetime,
dailyFiles: undefined,
files: undefined,
initialize: initialize
}
fileCursor.dailyFiles = dailyFiles;
fileCursor.files = files;
let cursorSize = 3;
let market;
let exchange;
let cursorSize = 3;
let server = newFileServer();
let fileCloud;
let product;
let periodName;
return fileCursor;
function initialize(pExchange, pMarket, callBackFunction) {
function initialize(pFileCloud, pProduct, pExchange, pMarket, pPeriodName, pCursorDate, callBackFunction) {
market = pMarket;
exchange = pExchange;
server.initialize();
fileCloud = pFileCloud;
product = pProduct;
periodName = pPeriodName;
cursorDate = pCursorDate;
getFiles(callBackFunction);
@@ -56,8 +58,9 @@ function newFileCursor() {
function getNextFile() {
let targetDate = new Date(cursorDate);
targetDate.setDate(targetDate.getDate() + j);
targetDate.setUTCDate(targetDate.getUTCDate() + j);
/* Small algorith to allow load first the current date, then alternate between the most forwad and the most backwards ones. */
if (j === 0) { j++; }
else {
if (j < 0) {
@@ -80,9 +83,9 @@ function newFileCursor() {
} else {
if (fileCursor.dailyFiles.get(dateString) === undefined) {
if (fileCursor.files.get(dateString) === undefined) { // We dont reload files we already have.
server.getDailyFile(exchange, market, targetDate, newDailyFilesReady)
fileCloud.getFile(product, exchange, market, periodName, targetDate, onFileReceived);
} else {
@@ -93,9 +96,9 @@ function newFileCursor() {
}
function newDailyFilesReady(dailyFileReceived) {
function onFileReceived(file) {
fileCursor.dailyFiles.set(dateString, dailyFileReceived);
fileCursor.files.set(dateString, file);
controlLoop();
@@ -104,19 +107,11 @@ function newFileCursor() {
function controlLoop() {
if (j === 3) { // We call the callBack after we have the first few files.
if (callBackFunction !== undefined) {
if (callBackFunction !== undefined) {
callBackFunction();
callBackFunction();
if (market === INITIAL_DEFAULT_MARKET) { // This is when we dont need a splash screen anymore!
splashScreenNeeded = false;
}
}
splashScreenNeeded = false; // This is when we dont need a splash screen anymore!
}
@@ -138,13 +133,13 @@ function newFileCursor() {
let minDate = date.valueOf() - cursorSize / 2 * 24 * 60 * 60 * 1000;
let maxDate = date.valueOf() + cursorSize / 2 * 24 * 60 * 60 * 1000;
for (let key of fileCursor.dailyFiles.keys()) {
for (let key of fileCursor.files.keys()) {
let keyDate = new Date(key);
if (keyDate.valueOf() < minDate || keyDate.valueOf() > maxDate) {
fileCursor.dailyFiles.delete(key);
fileCursor.files.delete(key);
}
FileCursorCache.js
+89
View File
@@ -0,0 +1,89 @@
function newFileCursorCache() {
let fileCursorCache = {
getFileCursor: getFileCursor,
setDatetime: setDatetime,
initialize: initialize
}
let fileCloud;
let marketFiles = new Map;
let fileCursors = new Map;
let datetime;
return fileCursorCache;
function initialize(pTeamCodeName, pBotCodeName, pProductCodeName, pLayerCodeName, pExchange, pMarket, pDatetime, callBackFunction) {
let product = ecosystem.getProduct(ecosystem.getBot(ecosystem.getTeam(pTeamCodeName), pBotCodeName), pProductCodeName);
if (product === undefined) {
throw "Product not found at this bot of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName;
}
let exchange = ecosystem.getExchange(product, pExchange);
if (exchange === undefined) {
throw "Exchange not supoorted by this product of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName + ", pExchange = " + pExchange;
}
let layer = ecosystem.getLayer(product, pLayerCodeName);
if (layer === undefined) {
throw "Layer not found at this product of the ecosystem! - pTeamCodeName = " + pTeamCodeName + ", pBotCodeName = " + pBotCodeName + ", pProductCodeName = " + pProductCodeName + ", pLayerCodeName = " + pLayerCodeName;
}
fileCloud = newFileCloud();
fileCloud.initialize(pTeamCodeName, pBotCodeName);
/* Now we will get the daily files */
for (i = 0; i < dailyFilePeriods.length; i++) {
let periodTime = dailyFilePeriods[i][0];
let periodName = dailyFilePeriods[i][1];
if (layer.validPeriods.includes(periodName) === true) {
let fileCursor = newFileCursos();
fileCursor.initialize(fileCloud, product, exchange, pMarket, periodName, pDatetime, onFileReceived);
fileCursors.set(periodTime, fileCursor);
function onFileReceived() {
callBackFunction(); // Note that the call back is called for every file loaded at each cursor.
}
}
}
}
function getFileCursor(pPeriod) {
return fileCursors.get(pPeriod);
}
function setDatetime(newDatetime) {
datetime = newDatetime;
fileCursors.forEach(setDatetime)
function setDatetimeToEach(value, key, map) {
value.setDatetime(newDatetime);
}
}
}