From a061f8d9a0832e653dc02f1c016ec1e9b89429c1 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Fri, 11 May 2018 16:19:27 +0200 Subject: [PATCH] First Phase Migration to blobs containers --- .vs/AACloud/v15/.suo | Bin 116224 -> 97792 bytes Blob Storage.js | 50 +++++++++++++++++++++++++++------------ Debug Log.js | 54 +++++++++++++++++++++++++------------------ this.config.json | 6 ++--- 4 files changed, 69 insertions(+), 41 deletions(-) diff --git a/.vs/AACloud/v15/.suo b/.vs/AACloud/v15/.suo index 86bb29def59ed1ffd96c3c89b05cb2addc29c18a..f5379eeeed6b90bc0b42a0458141cbc5e6961216 100644 GIT binary patch delta 6286 zcmeI04^-4u7RTrQ7=~dSlaUDs1{fwRKtM3p{DU$GnXRUxXj+PaAZjScA4^mi`=jQX z$lT;-uBJ|A=9+)zSCN)ADw&yLCdbVFk?Yyhf89N{XQ|EE?;B=xmN@<0-OlOk<9xXH zz2E!Y_ujp4?t3$~l|ip6pgR!n zltIn|H+k!$A%}unc!@f6)F@~XAQB`2Ge`y|&drPAP#h2KsVaa?dEj$=3{Xm_1|EIZ7&-C`#Qe~GA00ixYMx}q+kI~a71u6X8) zuFyM2SC4mA13N%;CDjauA3Gh2R+j(yYTGulk^;#|NQxfgT?y?cs|8hTmQd!4nK_-pGGqj+guwt8{zNQxLUS} z@T+-2Ok#(7(%_yIn4 zG#fFGumG-ESEH#DYc=rK>yB!LZJnQe&1!V}TvEP%SR*;_%OTF$_!5u19O+%zDjV6$ z&56G4Dz@V!@CXp+Sv?TvneQyyfQU?f91Y?y+XOa)o!|+u1w0A1f^9%}{b|VU;7>qA z$yY?!UjV+kkk9hT^3Wms&?;j0B6tnF4EBOofD0S|2f-oG3|<9?fv~>6A@|P?yV_A9 z*|(Y%^V&9YIt~#ZlG2q+Pb5omPN?!E+4iV4q`Fi>ndG&snQ37m90+m2F`b z#j+I}Vw2?DZpMUmV~XCL^^i>m%iip(xKF)G-27Yy6)~j@S8jFgrA7_Cq0SnZc5~AS z4W$P-UXc8^!Oj|u1H*`Iu=Dj`k7Gl`u-*iu>(~cqQ7M-XNs@g>j#fljTs(YzxN653 zJ`|A7mv*$OMRxQ(2#m(e&MNJ%1xe0*BS;z7 zf|kth?JW-Mc+UtbRn`yo7RS2iE##E0d&fB6)Je$e>W;~!D%LYDvWH>MwIZNr4-zLG zpV%?e7^&k9E#wVXti``=W83>SZfl8PSIaD3(v}IoU_j3XXyqPf)4ANJc79|fxu=O7 z(G&2xSg?>{i~5`fRYi*X9Fe{B{Lk2tXUcnUkMrqV(UR(Qp{~ico(>Sly>$ocP9!|x zx-9XYQ$`+VOjfsj>2|a0cmzk*$Fud6F=!Yr*k!6ho?PFHZKp>2T(YSpsZk=A{q(Mo zRxv>@McJxBO?3g3C9&(Y6(0g|$|#oIACUd+c*95Wwt72EN;_6DOOEaRjE=rP731*ry>oV2T?gLsyjX|eQ!$heq5R*E z<2~Y{_e^YRD`HohmD7OjoQuq7Qc!>!e6eZ~8_xDYSbBDI z8#=xqX;o{u;ERIboZ`aj67#6C68`Ha$<)jZpU+6VeO5_zacRnJbBkt8FQ_W3NEudE zH6^d2ps2VqWmsuhVP#5QnMdKxANQB{&!s|HyOajdGve*2F5k{Bn=_|uE;oFdtepP1 z2Pv_i%_Hh+23n@C2Re_21dxUe$1lja<<$G%C#ln4nX^3l5xVi})6P|@r{Q5s#6I&B zt?L0uahcEgIqQ6P8R@0M>HJY`B;Wn9VL+Gb?8-}(QzldvSELq}Ri)0ZylUXbyIJ13 zoW_PN?O289|F`%bD;DyjTz<_WU5EcGF@cd+RuI?a3cBiP`o5mBaI+!FZ+7A|Nl-rg zGLGb3E2zK0==V(IQDc+gUUHD(S-*C15SrNCo@8&?^t~DJSI(dLNSOBr{LhnQ7X9b* zoNO(pc}>$6Pys35eA8Wu6JMKsB@s#t5N!e8#wu?DO=9(6qkKf7nUu|$wKL@#7vra! z^~;cg7L!e$@&F~tp9WHhd`6<-O|Dw9N%EbIw6!sS(vW`#_~qUd(b32g$0t4Ix}T`Y zwTm>wj%9;_CY=zML6sawG*GtcDS?Mwj*xA7vQcdljivVkrBoF!dB^;N>AL6_M!JJ&Uxf06PfTI*TO#Z`GCPat_Llr!_`0!#J7ern>d z62EoY+%$SR%~sQ4uh~NX`~mZeg!z$;GjR<$fNKa_OvWZl6@5T*S~Y3K)nt|Inu5F0 zrVD2I`K6Q{69Jd|iue&wl$`C9GbjyzD`gr?xALMB!{v)J={?z5K>dXA#%40hl8MZe zjI?8kyc)xW*JqFI9u;f&+ta&`?TuRVzW6Gx_FgV~hgwEfSA7`uWnN|8$zHt&txG^! zW6n*Hk?P!(`ron2hWW-Oqm7cumv|#(m!77{=5mS)={zI(O&u<*CohBsdTv%C$!?VQ zETg!doqLv*biQMW8(8?23mNVg$pyuIZ!YLxn3ht^>iW_CvHF?0{u=m6{7UR8ygYY+ kY}<_EpshJVc5NoBXS%`o`CR-E4&S1^ixqCBG||!eFV9~cApigX delta 6688 zcmd^@dvudU7Ql1Aq;2{ng|<{%F%7R!nzW$>DwOik1})2@trS+V3Vno<5|h#aHG85B{M7*?W^OP>@xSvwK!| zdVaa{{$}RR+__WFnHbONxOFAzlNAa@400|M3ZYN{a?76TEFs>La^H!TPhW>SnM-iP zdiJ#tN*1YxfIk@23{(THSs0s$Ohbksw<5z3>@L(FAbu@L2L$lXYhoY#VKI1-PfBTmGHEJl_f_aHYROA-2F2g=_f%Me;E4@#e4 zi`V|S_9N2Sit{?FWahZtZV}6)dZYFfNb59n(%@`_@<@{;`)cwA%)3;RNtj2P952@= zO;Wo_C`pqP-+)rqq@bv0E_J?p0A}SO*+@Pz6(J>!M107l8l8uEq){3sE&e#=TEX{U|?U0M8 zCEJ{!S5t%nJD8wnM-wyS_hF5{BClYZd~eH2b}(^g?2|ZzgQ=`NsU5uR>m)aZ&SD)X? zJ7gA6M?>8-<231;{CI&0xvk;$xle(lZS4i-$<|4_QN17uC1t?O8+j_^A<)v^i~gNo9Cx_TNd~?2!5x^ zHwt&sR@_PLz0i4Aj3C7<`!1%l#R;&IxpsLHu}$>Q_=^R9z2I}slxD7a8hqP7`Uy+A zMi{*uyQJyS5>U7g<220&kO$>_(q*JH^1InvJsx&@t0%JE^HtZ;yrW?c*6c(igv`yA zPv57CPIHlmj+;D978*a8ZQqv7x-%1`ijB{KZ>2LMk{ zlv+m5No)%yJ{Q=x4Xe~N#_`ySt!|u`R)UcUNGUQADMx&klE_XIEkXZk!M{`J-1IWA zmz5^z^dotwkSBV@RI z{KPw$fm!DsSbTm0&$q{J#5|h7^ZVn0R#8KBTD zaDft!HU}F#luQ>pUF0NntnO7)=id{AuqlaD6`4^`Q#x#5>CUz(>J)A0oYbKERHtZE zSnv>9G1r1+v`HE<*BMUHf-gpg%7$*t-Izl=DifJ6*DuF1d$=zBUL2aSb_;&Np=d^T zwD)8PLj3Wuki~BNB!`J7(%6wxS(>xh2G=-DW4U4+bDyYWDJL`Fe%6zgk8aZ&xtYNv-c~4?88QS^25i?4>iM0WlL+D4B4!)PDy2Z$qcQ;)*oNj!(&E z*=HM>=N*xG&WfGyionXwh`w{4t{lL)=bR{cdV?4V2MoVAs9i($IeZZOwbZA%R0b@oP^rOhzgVRPMW z;U<7Ff!zY601x(qe!QR`jDRhCj{tRi*8(UF)GFb1fNSYq6{j#qdJJHXsRGyZ11;aW z5R(00sQB7c7(<@&`9KDSJT~*Se3O5c2uWb&^AaJi^lBHSt#($MrPWyPa2lK2B7I!_ z!tPFjbpPi9uQotF-(`iQ^vObWjw6C|F;4FSGsm_66MtO;UuBSCiiVLGp@U^!A67-6 zL=W4P#nDodbQEaasFo6;P;PU$8f%>ktZl{yhkd@a3CHiScXV*E3i7zo0yW!$#|0ML zpUJN;1RHNQ!fakp5B|y05c@txr+0nXHeJ z2Ma;uq6onaMew3_K29GslRG0IE ztdlk8yQtU0Zde+CF%VLsN$VO1$AI^*0-Zk?wO=fC|0(v;YkLEqRzW7PGw){bj5#of z=?>Mfnpda&xMiCtSNR!Rw4ZFrd}lrXhsKGE4Pa#bKGCo@n)QKAb6^wvl8sf>FTp)U*)9!@N`8p#M1kir_g&!)NqoUds)Lng-4Vrtbu&C9cMFm(6K4R9RZBj>d?;5xq+JJQWxq za+`6y%|`QwRN%iBzaqnTd=l>|1b5gKTE85$Ds^8+V=3?t_3{OkevBNwaL@$>ynAyn z-uLdM|1?Ddy%pymeeT{3s30>0eQ&MNtoZKfk%?*c{&qB~~XD$rlK?8VLb#Xq|m>`#yba)_W1dqV` zkA%M!Mh8Lia#y@9Pg~`xw^Y2D-WHa8&WSef%KN%baz{?83e97g` zf}g&pdEbODWAxRduR=EOKpL+%;v)3`{xmnEI}mlw%D?B7h1cw7@03+#q(}FHry!Vv zar&>Q`5Jb0#Mhh~_BD&H@>r0E8EHBE3S06$Nl^J?`#GuW{65kI1BYipib{Jv2P2o; zS$w$?CeYFRk{m?ZO!T7hC@-^{YWXuJD8@Gq7srE>iPcGA|Da`U2JO1s@C)Xvx1NhD z!3S82!V(B+VG<e*ZmNX=d`mIt0_bkX0%KE$@PFO}8F8{F%?|$~AH9@>u=_l)0Uy0q zTyv+wjBDQ@Kb`H|JsOt(yE}qj0RGD_%RJ%Yi$=amHrVk2x55b8@B89a<6#Y5cslqO DABQ$1 diff --git a/Blob Storage.js b/Blob Storage.js index 32b22b57d..dba465f53 100644 --- a/Blob Storage.js +++ b/Blob Storage.js @@ -19,9 +19,6 @@ exports.newBlobStorage = function newBlobStorage(BOT) { logger.fileName = MODULE_NAME; logger.bot = bot; - const shareName = 'data'; - let dataOwner; - let thisObject = { initialize: initialize, createFolder: createFolder, @@ -30,20 +27,29 @@ exports.newBlobStorage = function newBlobStorage(BOT) { }; let blobService; + let containerName = 'data'; + let dataOwner; + let ambient = global.STORAGE_CONN_STRING_FOLDER; return thisObject; - function initialize(pDataOwner, callBackFunction) { try { if (pDataOwner === undefined) { dataOwner = bot.devTeam; - logger.fileName = MODULE_NAME + '.' + bot.devTeam + '.' + bot.codeName; + logger.fileName = MODULE_NAME + '.' + bot.devTeam + '.' + bot.codeName + '.' + containerName; } else { dataOwner = pDataOwner.devTeam; - logger.fileName = MODULE_NAME + '.' + pDataOwner.devTeam + '.' + pDataOwner.bot; + logger.fileName = MODULE_NAME + '.' + pDataOwner.devTeam + '.' + pDataOwner.bot + '.' + containerName; + + if (pDataOwner.ambient !== undefined) { + + ambient = pDataOwner.ambient; + containerName = pDataOwner.devTeam.toLowerCase(); + logger.fileName = MODULE_NAME + '.' + pDataOwner.devTeam + '.' + pDataOwner.bot + '.' + containerName; + } } if (FULL_LOG === true) { logger.write("[INFO] initialize -> Entering function."); } @@ -59,8 +65,22 @@ exports.newBlobStorage = function newBlobStorage(BOT) { } try { - blobService = storage.createBlobService(pConnObj.connectionString); - callBackFunction(global.DEFAULT_OK_RESPONSE); + + if (pConnObj.connectionString !== "") { + blobService = storage.createBlobService(pConnObj.connectionString); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + } + + if (pConnObj.accountName !== "") { + blobService = storage.createBlobService(pConnObj.accountName, pConnObj.sas); + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + } + + logger.write("[ERROR] initialize -> onConnectionStringReady -> Either a connectionString or an accountName must be provided."); + callBackFunction(global.DEFAULT_FAIL_RESPONSE); + return; } catch (err) { @@ -76,7 +96,7 @@ exports.newBlobStorage = function newBlobStorage(BOT) { try { let fs = require('fs'); - filePath = '../' + 'Connection-Strings' + '/' + global.STORAGE_CONN_STRING_FOLDER + '/' + dataOwner + '.azure.storage.connstring'; + filePath = '../' + 'Connection-Strings' + '/' + ambient + '/' + dataOwner + '.azure.storage.connstring'; let connObj = JSON.parse(fs.readFileSync(filePath, 'utf8')); callBack(global.DEFAULT_OK_RESPONSE, connObj); @@ -133,12 +153,12 @@ exports.newBlobStorage = function newBlobStorage(BOT) { if (FULL_LOG === true) { logger.write("[INFO] createTextFile -> About to create a text file."); - logger.write("[INFO] createTextFile -> shareName = " + shareName); + logger.write("[INFO] createTextFile -> containerName = " + containerName); logger.write("[INFO] createTextFile -> pFolderPath = " + pFolderPath); logger.write("[INFO] createTextFile -> pFileName = " + pFileName); } - blobService.createBlockBlobFromText(shareName, pFolderPath + "/" + pFileName, pFileContent, onFileCreated); + blobService.createBlockBlobFromText(containerName, pFolderPath + "/" + pFileName, pFileContent, onFileCreated); function onFileCreated(err, result, response) { @@ -169,7 +189,7 @@ exports.newBlobStorage = function newBlobStorage(BOT) { logger.write("[INFO] createTextFile -> onFileCreated -> secondTry -> Retrying to create the file."); - blobService.createBlockBlobFromText(shareName, pFolderPath + "/" + pFileName, pFileContent, onSecondTry); + blobService.createBlockBlobFromText(containerName, pFolderPath + "/" + pFileName, pFileContent, onSecondTry); function onSecondTry(err, result, response) { @@ -212,12 +232,12 @@ exports.newBlobStorage = function newBlobStorage(BOT) { if (FULL_LOG === true) { logger.write("[INFO] getTextFile -> About to get a text file."); - logger.write("[INFO] getTextFile -> shareName = " + shareName); + logger.write("[INFO] getTextFile -> containerName = " + containerName); logger.write("[INFO] getTextFile -> pFolderPath = " + pFolderPath); logger.write("[INFO] getTextFile -> pFileName = " + pFileName); } - blobService.getBlobToText(shareName, pFolderPath + "/" + pFileName, onFileReceived); + blobService.getBlobToText(containerName, pFolderPath + "/" + pFileName, onFileReceived); function onFileReceived(err, text, response) { @@ -247,7 +267,7 @@ exports.newBlobStorage = function newBlobStorage(BOT) { logger.write("[INFO] getTextFile -> onFileReceived -> secondTry -> Retrying to get the file."); - blobService.getBlobToText(shareName, pFolderPath + "/" + pFileName, onSecondTry); + blobService.getBlobToText(containerName, pFolderPath + "/" + pFileName, onSecondTry); function onSecondTry(err, text, response) { diff --git a/Debug Log.js b/Debug Log.js index aaeed4732..7b1679c79 100644 --- a/Debug Log.js +++ b/Debug Log.js @@ -240,43 +240,51 @@ function onfileCount(err, files) { - if (err) { return;} + if (err) { return; } - fileCount = files.length; + try { + fileCount = files.length; - fs.readdirSync(pFolderPath).forEach(fileName => { + fs.readdirSync(pFolderPath).forEach(fileName => { - fs.readFile(pFolderPath + "/" + fileName, onFileRead); + fs.readFile(pFolderPath + "/" + fileName, onFileRead); - function onFileRead(err, file) { + function onFileRead(err, file) { - try { + try { - filesChecked++; + filesChecked++; - if (file.indexOf("[ERROR]") > 0) { + if (file.indexOf("[ERROR]") > 0) { - errosFound = true; - - } - - if (filesChecked === fileCount) { - - if (errosFound === false) { - - rimraf.sync(pFolderPath); + errosFound = true; } + + if (filesChecked === fileCount) { + + if (errosFound === false) { + + rimraf.sync(pFolderPath); + + } + } + } + catch (err) { + // We ignore the error; + return; } } - catch (err) { - //console.log("Error trying to delete Loop Folder " + pFolderPath + ".Reading file " + fileName + ". err.message = " + err.message); - } - } - }) - } + }) + } + catch (err) { + // We ignore the error; + return; + } + } } catch (err) { + // We ignore the error; return; } } diff --git a/this.config.json b/this.config.json index a7fd1af67..a48480790 100644 --- a/this.config.json +++ b/this.config.json @@ -22,7 +22,7 @@ "process": "Multi-Period-Daily" }, { - "enabled": "true", + "enabled": "false", "botPath": "../Bots/AAMasters/AABruce-Indicator-Bot", "process": "One-Min-Daily-Candles-Volumes" }, @@ -37,9 +37,9 @@ "process": "Poloniex-Live-Trades" }, { - "enabled": "false", + "enabled": "true", "botPath": "../Bots/AAMasters/AAMiguel-Indicator-Bot", - "process": "Trades-Files-To-Blobs" + "process": "Trades-Blobs-To-Blobs" } ], "stopGracefully": "false",