From 2ff60c4b4c5e2522da743772f119e23655d585c1 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 25 Apr 2018 13:35:08 +0200 Subject: [PATCH] Ready to Debug BackTest functionality --- .vs/AACloud/v15/.suo | Bin 92672 -> 94720 bytes Assistant.js | 26 +++++++------ Context.js | 61 ++++++++++++++++++++++++++----- Run.js | 26 +++++++++---- Trading Bot Process Main Loop.js | 38 ++++++++++++++++++- this.config.json | 1 - 6 files changed, 120 insertions(+), 32 deletions(-) diff --git a/.vs/AACloud/v15/.suo b/.vs/AACloud/v15/.suo index aebf997930c7edc2ffc71fa60fc454a9a3faac4f..8411265a36494d475a6f311833b46f562701a416 100644 GIT binary patch delta 7642 zcmeI13s_Xu7RS#%%nXCPhNq|~BRy0C9ketta}ZEeR6sB*GD%6{8XyBb$YjhkDts`j zI_53K(bAqOV~<;zQ@+a5wD&bbv~ERdlBsD|<^I>1Gd0oU+Uxu3yLa>bX6?Q9I{WOi z*WUZAeXzf!w|}HxWa%zTS~5w}dC2O)nYOkz*7|`f*X4APL#*pXC6vF^DWOgTu97s3%*@NE`&>$JX>H5NmPTb%S6!O5iAPk%j&QP*3Zg5BB zV9?eHvKI&jJ-}jc3Frr|2a~|1;9?L6&H?w)d}GI&AmkE3Ea(G9fk4YEOMeNGqs7-VK36IF$b_fnQ!QJz_!)6ujDnHsIq z4(X7mhgg!17v2FsO-9+*Pn7PzexTPa-qzUFr%~Xi?hCwL;`;Vp>QXZj7VN1A(B4yX zq4Rp`TI9!r_MVytT`0%}F`zR@06*uuS*Yh;MM+YW#LnZc@sN3)N6os(n#-WD0`}Hw zz($*Xezi7{0*oU_?eFUb)XcKfpSm*xZVa?32xBJtm9|D3FT^VGcW_zbDFK?+2&Ru zNpplmfgQqXxLk+wo!~CO<@t~cz(TMH)PoIxz} zYH}sygIa!_kkE_!JM^q+)HK}klgR%AJPn@l(Laa$^IG`@$c?}aUIZ_JZD2Fl0=5Ef z>t)DSz;^H|cn$0TJHhK<7uYSZF5W=qP4E_Y8@vPF1-uCU3HE^Z!CvqIXaf5{GuRIf zfDge(;9uZl@Co=7w1CgR=ineX1ik==K`Z!DV4)mAhI@Pz@|c!CQ(xG_5uQey{E`h> z&|Mfx)#24*6den%5hJOpsKz(~YM!Jtqb0JwJSfU2OHu&frUD^@IKQrNnW!(fG@8=n zpKL2)On^F9g1Irz?(8o{xQ;~x8pxh}F2#q%k~?2#ybJaB_}Cb}LJyWpRGpka&gfXO zChKUkY%%!b#&bOC_!z$QLVX9QZm0V3eW+o4tYc3g)h$hSM8~`0W19k<*UFAqGd1Mv zDBh&DV!R!9srS88HtsZ6Yckf;n>y6b_qDkSHl+1NYv&Hs^RDiFDuV+y-mYmj7_GK@ zeQW~nfer7LNUpYGAv=4^j;ApS-PFG62zuJRFVUdD?jx;F9lw2g{okQh=;TuukkfMq z9uxgiXd6{64+%!f$oJ)q+1oe$a+useH^ugH#rJLM&=p8oZV*RytyV}N|u?$;=v04KpW;NRdBI1SE#HXva^WMJ2?Bz7(a=U@fq zC>^!rsj@P;&RHruKNgNXdfM{FAW=#patPT!v6Jb5&ZVn3rFV3Q{r?-`&;CZ}h7F;* zNjS4)vL5Oa<~Wjn{|lY>_S_z}f7FgWh+|#WPu;@d2P=D%Xguf^96|0sk<|EzER$R^ zOvlscyQxz0dVt`4hy$hXw7u%YB<01fVl*uq-yIva`K)L*ge&2>!c2uNakSBDreWK9 z77VD_uxHY9=Df_@#OUWvjOGCM+py$;5XAi*@cu-Mw=)1sCLZAqS0)O!$BtC7o;zvp&J9nf?9AH$OYp-9$?$?nk<0499#je)O1{bBl1^kVp zWZh#Y>ma-OjMMOptE2FYt6lMnn}lK+BVsY+3F10sO}?ntb1->5?cX{)d7M;?P_jyz zE)^ndERv?E8_#5E8p@>zK8vJO941B}a4nHIw6rOQ$BWA&_y!zj79sAmNd+*P$a_X( zmRWY0XRW}S0*~F+1vcK9lfVO^UWK15gso)=TRpAb`tQdt2wbq!xb+re*S3OvgE?ji zDXOp)WK|X03T$On6_d$+%{WuFq5nJQ>~4JfxzVNDbdwLg#&!W^W#zp~Y#C`O4%<{y zP;==WeUkRhv)+D1`j=s?*RqAOHdOSYFHQ#0lvl0fE*?s9jG4}jiFIAN{($(2-s;uf zx=zmC?#Gc)5O2GATJIf+mA_z5c07`)>w_!3T_;Wzt8hq5O+)b!z-H;1TmzY<<<~;41C3z4rh5|dX)XWwx}0;wUuVAe z$vEp**JCd8{mr6rn1Zzdd5K3?Q6ta)VjS=V42-8l3vY>WW#)aiyFX56u0X25!XO8G)Adl7kL^RcJ^nE3GWLk@g))6sw`=?y|n;7M_6ZiGi{RR7ahrMPq|S&nrg>ye%`;P4`GH#g;X->j>Z=gXbka z(Zo^D+Mpy35ElYoLma590SBZR@DYInY6YlsoeFCsGM1vW4DdAPrZ}xYo~QbQ;34oZ z@YJ-mJ*wrO&?GMd7v5&UPx%;O&i4=<2WAZHKP=V4TdDwRDz@^VB)C$WHVWGR{>46{ z6Rnrh7mM^13;u3S&9J2Q8GX@JY#Y{hULwK@*im4%@qj-EIaPRmY(lKX&L=^ZMp zVy5fGmLWnZ4Ha{uduTUeffttgK`DI!j~=C*-1@Y;udvFoF_ORw_abknV8-ttHcw|4 z71ElPFiJTV%ENCo`}jko(emOrdbA~qtG`#ujke-?Z{ls9Q@ZN77|n5|mDn(6&b&GK z!t~?;1N$c@Db_%-Q=M-O&%EPmect5vz187r+WL%44I|B%U3Ni3H+24E<|TKM=cV0` z=iv9I8Be|+j0q;+lF1S=E$+A7}%8v{e6j zW;B0<@}%aCRmhk@dixC}sDo#7;c1n`jhnd{A6;5XLB7rReSU9}+?79hNXoKc(WC2+_I<+Z z*cXwg`NYMFJ52PXWgo`quZLz9!l5E%Z@B2D=)y&Xz66_5iuzFkvFl%U6zkNd<)5D! zMK?9Rg!u6bxAItoND$X@U`WGJ>UFY*EP$Kyma zHHTWi8tj;GrRkg5U%$R;dFRsyhab0>-Y`Us?PSY5dJzZAik1k9IFUr|J(jQ{o6S7B zu(;HmPVa%qE|n4DyJjGY_XDd zZx}?U#!tiah*U(l7`MWGN>FJ3SV~BZR!mVMlP2^DQ5-`>xYC*`E>Ttv6)y+s)PE9O zgZTF?WyVNxIgQzTDK!s@`NO7_D8XoE{P(err@GCT>iPAz#PehaCMx9_!k}cQi+N&} zBBI1)%FYatsH`6=A{A?-i2l93<5=5QX~+>edL=U&M?hZw3()z6dj2wB;F%*6B2hB~ z?L3b)po~YU>wYkD1rYoW2#BdsHOTab7%DSh&-o>zban&( zgM0kObLbTH1Zsm*iZK&YhhV2Abh|NwOnK o^9G*E@^ldw(f;EiPZ-Rd>&Uq_zFz_UDfCNj-|kHPXL*+Y0ETYNZU6uP delta 6378 zcmeI0dr(x@9mntQvUgdQ*8&S7BCeQ35*~s&7(mRfEC`X;3L1%sAcA5E4_Q)*8rDam zc^TM~cv6$G%*JY}iAmVDWIAnKYigag(`cIJ;WP<4nKtQUIwoRL(uD2z?89sjt!f4|@PoqMnCUB&jQ(x1sP{}d9hlO(+)Nh&bzcDqqX1TtQ*Op_xz z#xB;lU;i_eL*nXWy>c&AvP*J>20h(H5VFDInJiNdWj#T*z7NDeTpIAX=;6D>yy6o7 zaT|yQH-IQ`6UYbAAPJ-ZEtm$dG${-iK@6A*f8qBVS0xCHyT)zrXjQ z?f3Ua)R%*a?&j=a)s();oGWG$&b5ODzznj%Tu=#wbLWFhkPBu25p5Abj1`ygwYTPW z--)`oCkx8OK)BwgW2=q zi{4JD;V+@`TX51V`1{Z=bktXfMiE#7iiHBBZb!Kkl=|927RWNlt3b6+zrt6Ff2=|M zPOvIKCgf{;?FL`jh;kiC>=9;368c`WH~Vy3QQqUL+sHXS5dBS9CmWYh%2H0iZ~t(PEf3aQ4oKnNofv&DxDSY#oNW4w>*<*ROX~Dz(L4&C1J8pO zzz`4-x9J7TJYp0(#W|(^{UCdoRlOgYE3DY3l;Lkx&av6SvSWknNRwd%N85k5L#Wpd zm=jBKcar*lHb(CV4fR_|eFtXR8^wD0)9io_v9SMMVg)Jr_Ml{TAOUULZ)oV?i6Di< z3Fou!@ieDl%Ls*RgJ?T=V8Akir>H3`UD)6eNczD5FaQP~1&@I^AkKrw!4G+ZnqqQ} zp!y@=0K)R(dJ6Re;K$$~I0T*sKLLk<&(u}kTi620B)8z$BinbHH6pRj0sc^OTAz6+ z*nS|{nf&U7IQ3ZbQUM!c})WZEx6U$YGG3hUZ7faaOj+8JvcbBg9hV2v+%47#lqM zn)AY6bwQ3-WythG<}{4#4r06CTNyqVaUlXCR7h`(Kjt#IPQn{j9C{D<*OJE~>6Q5Cys zG>Mg-)idKe8dm?N!D$=)KxQB9(}xaVOU2;N73m0XSExZ>*kj$)&}eOGZ!;`zw;C!p zwKq07{;1+nDl!fmQ^*#?Eh^f@+tO(t?e~m{s%&pcZHEFsi~vAtJY*HXl?IaS3BwYl4*X=BDu(zt0M>7B-VTy!vkXqKrcE6cpVoReeP8h6DsZPtv9T{--C zKFykPEoICkN^aAUF+E-)B;a{IONc@|WeSIw4_|SYL)%?+eGcQsW#ker``!QNr^pF@ z@;G#&cRYSk3BOz`8sfhO^Ce!(d{+%VZSt2Rk_tT@oDipzOL?1~3g{x6DaXj>BW%XM zlIYjWq`%GnsF54O%*_bO& z&-J%c8qd&p<&+Ru>}7#l1{SBGb*NfrPPehp`|OMvAwOi<&dkCD0n?0zP)XpXSMvaLOS(4&l0&!p~?v+^aRF4 zYX=n;xl3tKlOiO5iFJ?WAUQ{QV_t$>Jp5 zzl7AZk$0Bj5fFXY9jBsko{GEb$~_+TSBLR6k(Xsy)E&sF1bb8$5dicD3+`rnD> zuH}@bbVvnKF55FU`>U9ZIy?!C_iIVZ-mHp4TAK+kob)aTiI*ij(u#ar%_~+CQ|h7H zetE=P^cK0eo=4TnlOs4c37T@z_8Q^-l0=wfS-?86@Cbf%Dd`pQ z4e|@idX6*l9zT{-aOZDH>T$9OJS!=GC7`NPU>d8cD(=Lu)nb3*wQ@_BfW=MXP=%o z=qS!UR>Q_WF$8n(!#A^iY-H#68AZ^s*Q?_3$KOT8Hzg|Y0J7~|s^=A2yhaD`6-CB> E0Gou;qW}N^ diff --git a/Assistant.js b/Assistant.js index 09f77ba0f..90c2f40e7 100644 --- a/Assistant.js +++ b/Assistant.js @@ -115,9 +115,9 @@ /* If we are in Exchange Simulation Mode then there is not much to do here. */ - if (global.EXCHANGE_SIMULATION_MODE === true) { + if (bot.backTesting === true) { - logger.write("[WARN] initialize -> Exchange Simulation Mode detected. -> Skipping all validations."); + logger.write("[WARN] initialize -> Backtesting Mode detected. -> Skipping validateExchangeSyncronicity."); callBackFunction(global.DEFAULT_OK_RESPONSE); return; @@ -683,15 +683,16 @@ /* All validations passed, we proceed. */ - if (global.EXCHANGE_SIMULATION_MODE === false) { - - exchangeAPI.putPosition(global.MARKET, pType, pRate, pAmountA, pAmountB, onResponse); - - } else { + if (bot.backTesting === true) { let positionId = Math.trunc(Math.random(1) * 1000000); if (FULL_LOG === true) { logger.write("[INFO] putPosition -> Simulating Exchange Response -> orderId = " + positionId); } onResponse(global.DEFAULT_OK_RESPONSE, positionId); + + } else { + + exchangeAPI.putPosition(global.MARKET, pType, pRate, pAmountA, pAmountB, onResponse); + } function onResponse(err, pPositionId) { @@ -775,15 +776,16 @@ if (FULL_LOG === true) { logger.write("[INFO] movePosition -> pPosition = " + JSON.stringify(pPosition)); } if (FULL_LOG === true) { logger.write("[INFO] movePosition -> pNewRate = " + pNewRate); } - if (global.EXCHANGE_SIMULATION_MODE === false) { - - exchangeAPI.movePosition(pPosition, pNewRate, onResponse); - - } else { + if (bot.backTesting === true) { let positionId = Math.trunc(Math.random(1) * 1000000); if (FULL_LOG === true) { logger.write("[INFO] putPosition -> Simulating Exchange Response -> orderId = " + positionId); } onResponse(global.DEFAULT_OK_RESPONSE, positionId); + + } else { + + exchangeAPI.movePosition(pPosition, pNewRate, onResponse); + } function onResponse(err, pPositionId) { diff --git a/Context.js b/Context.js index fc42f878b..0c90ce503 100644 --- a/Context.js +++ b/Context.js @@ -121,6 +121,9 @@ switch (err.result) { case global.DEFAULT_OK_RESPONSE.result: { logger.write("[INFO] initialize -> onDone -> Execution finished well. :-)"); + + bot.hasTheBotJustStarted = false; + callBackFunction(global.DEFAULT_OK_RESPONSE); return; } @@ -157,13 +160,14 @@ statusReportModule = statusDependencies.statusReports.get(key); thisObject.statusReport = statusReportModule.file; - if (thisObject.statusReport.lastExecution !== undefined) { + if (bot.hasTheBotJustStarted === true) { - getExecutionHistory(callBack); // Happens when the Status Report exists. + createConext(callBack); } else { - createConext(callBack); // Happens when the Status Report does NOT exists. + getExecutionHistory(callBack); + } } catch (err) { @@ -178,7 +182,7 @@ if (FULL_LOG === true) { logger.write("[INFO] initialize -> getExecutionHistory -> Entering function."); } - let fileName = "Execution.History.json"; + let fileName = "Execution.History." + thisObject.statusReport.runType + "." + thisObject.statusReport.runIndex + ".json"; let filePath = bot.filePathRoot + "/Output/" + bot.process; if (FULL_LOG === true) { logger.write("[INFO] initialize -> getExecutionHistory -> fileName = " + fileName); } @@ -238,7 +242,7 @@ let date = new Date(thisObject.statusReport.lastExecution); - let fileName = "Execution.Context.json"; + let fileName = "Execution.Context." + thisObject.statusReport.runType + "." + thisObject.statusReport.runIndex + ".json"; let dateForPath = date.getUTCFullYear() + '/' + utilities.pad(date.getUTCMonth() + 1, 2) + '/' + utilities.pad(date.getUTCDate(), 2) + '/' + utilities.pad(date.getUTCHours(), 2) + '/' + utilities.pad(date.getUTCMinutes(), 2); let filePath = bot.filePathRoot + "/Output/" + bot.process + '/' + dateForPath; @@ -306,7 +310,47 @@ */ - thisObject.statusReport = {}; + let runType; + + if (bot.backTesting === true) { + runType = "Backtest"; + } else { + runType = "Live"; + } + + /* Here, we dont know if the Status Report was ever created or not. To test that we do this. */ + + if (thisObject.statusReport.runType === undefined) { // This means that the Status Report does not exist. + + thisObject.statusReport = { + runType: runType, + runIndex: 0, + liveRuns: [], + backtestRuns: [] + }; + + } else { // The Status Report does exist, we just need to move to the next runIndex. + + thisObject.statusReport.runType = runType; + let runContent = { + lastExecution: bot.processDatetime + }; + + switch (runType) { + + case "Backtest": { + thisObject.statusReport.backtestRuns.push(runContent); + thisObject.statusReport.runIndex = thisObject.statusReport.backtestRuns.lenght - 1; + break; + } + + case "Live": { + thisObject.statusReport.liveRuns.push(runContent); + thisObject.statusReport.runIndex = thisObject.statusReport.liveRuns.lenght - 1; + break; + } + } + } thisObject.executionHistory = []; @@ -398,7 +442,7 @@ if (FULL_LOG === true) { logger.write("[INFO] saveThemAll -> writeExecutionContext -> Entering function."); } - let fileName = "Execution.Context.json"; + let fileName = "Execution.Context." + thisObject.statusReport.runType + "." + thisObject.statusReport.runIndex +".json"; let dateForPath = bot.processDatetime.getUTCFullYear() + '/' + utilities.pad(bot.processDatetime.getUTCMonth() + 1, 2) + '/' + utilities.pad(bot.processDatetime.getUTCDate(), 2) + '/' + utilities.pad(bot.processDatetime.getUTCHours(), 2) + '/' + utilities.pad(bot.processDatetime.getUTCMinutes(), 2); let filePath = bot.filePathRoot + "/Output/" + bot.process + '/' + dateForPath; @@ -459,7 +503,7 @@ if (FULL_LOG === true) { logger.write("[INFO] saveThemAll -> writeExucutionHistory -> Entering function."); } - let fileName = "Execution.History.json"; + let fileName = "Execution.History." + thisObject.statusReport.runType + "." + thisObject.statusReport.runIndex + ".json"; let filePath = bot.filePathRoot + "/Output/" + bot.process; if (FULL_LOG === true) { logger.write("[INFO] saveThemAll -> writeExucutionHistory -> fileName = " + fileName); } @@ -535,7 +579,6 @@ if (FULL_LOG === true) { logger.write("[INFO] saveThemAll -> writeStatusReport -> Entering function."); } - statusReportModule.file.lastExecution = bot.processDatetime; statusReportModule.save(callBack); } diff --git a/Run.js b/Run.js index a3920f4f9..735db9fc4 100644 --- a/Run.js +++ b/Run.js @@ -17,10 +17,6 @@ catch (err) { return; } -/* You do not have funds at the exchange and still want run your bot? No problem activate the exchange simulation mode: */ - -global.EXCHANGE_SIMULATION_MODE = JSON.parse(global.PLATFORM_CONFIG.exchangeSimulationMode); - /* The following global variable tells the system if it is running on test mode or production. */ global.STORAGE_CONN_STRING_FOLDER = global.PLATFORM_CONFIG.storageConnStringFolder; // 'Testnet', 'Mixed' or 'Production', or whatever folder name the conn strings files are into. @@ -319,10 +315,24 @@ for (let p = 0; p < global.PLATFORM_CONFIG.executionList.length; p++) { runExtractionBot(botConfig, processConfig, month, year); break; } - case 'Indicator': { - runIndicatorBot(botConfig, processConfig, month, year); - break; - } + } + } + } + + if (processConfig.startMode.timePeriod !== undefined) { + + if (processConfig.startMode.timePeriod.run === "true") { + + botConfig.backTesting = true; + botConfig.hasTheBotJustStarted = true; + botConfig.timePeriod = processConfig.startMode.timePeriod; + + /* We override these waitTimes to the one specified at the timePeriod configuration. */ + + processConfig.normalWaitTime = processConfig.startMode.timePeriod.waitTime; + processConfig.retryWaitTime = processConfig.startMode.timePeriod.waitTime; + + switch (botConfig.type) { case 'Trading': { runTradingBot(botConfig, processConfig); break; diff --git a/Trading Bot Process Main Loop.js b/Trading Bot Process Main Loop.js index e7fabb498..124dfa0de 100644 --- a/Trading Bot Process Main Loop.js +++ b/Trading Bot Process Main Loop.js @@ -66,7 +66,7 @@ const BLOB_STORAGE = require(ROOT_DIR + 'Blob Storage'); const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log'); const POLONIEX_CLIENT_MODULE = require(ROOT_DIR + 'Poloniex API Client'); - const EXCHANGE_API = require(ROOT_DIR + 'ExchangeAPI'); + const EXCHANGE_API = require(ROOT_DIR + 'RealExchangeAPI'); const CONTEXT = require(ROOT_DIR + 'Context'); const DATASOURCE = require(ROOT_DIR + 'Datasource'); const ASSISTANT = require(ROOT_DIR + 'Assistant'); @@ -77,8 +77,41 @@ /* We define the datetime for the process that we are running now. This will be the official processing time for both the infraestructure and the bot. */ - bot.processDatetime = new Date(); // This will be considered the process date and time, so as to have it consistenly all over the execution. + if (bot.backTesting === true) { + if (FULL_LOG === true) { logger.write("[INFO] run -> loop -> Backtesting Mode detected."); } + + if (bot.hasTheBotJustStarted === true) { + + if (FULL_LOG === true) { logger.write("[INFO] run -> loop -> Bot just starting, setting initial datetime."); } + + bot.processDatetime = new Date(bot.timePeriod.beginDatetime); // Set the starting time as the configured beginDatetime. + + let endDatetime = new Date(bot.timePeriod.beginDatetime); + + if (bot.processDatetime.valueOf() > endDatetime.valueOf()) { + + if (FULL_LOG === true) { logger.write("[INFO] run -> loop -> End of Backtesting Period reached. Exiting Bot Process Loop."); } + + callBackFunction(global.DEFAULT_OK_RESPONSE); + return; + } + + } else { + + bot.processDatetime = new Date(bot.processDatetime.valueOf() + 60000); // We advance one minute in time. + } + + } else { // We are running Live! + + if (FULL_LOG === true) { logger.write("[INFO] run -> loop -> Live Mode detected."); } + + bot.processDatetime = new Date(); // This will be considered the process date and time, so as to have it consistenly all over the execution. + + } + + if (FULL_LOG === true) { logger.write("[INFO] run -> loop -> bot.processDatetime = " + bot.processDatetime); } + /* We will prepare first the infraestructure needed for the bot to run. There are 3 modules we need to sucessfullly initialize first. */ let context; @@ -210,6 +243,7 @@ if (FULL_LOG === true) { logger.write("[INFO] run -> loop -> initializeExchangeAPI -> Entering function."); } exchangeAPI = EXCHANGE_API.newExchangeAPI(bot, DEBUG_MODULE, POLONIEX_CLIENT_MODULE); + exchangeAPI.initialize(onInizialized); function onInizialized(err) { diff --git a/this.config.json b/this.config.json index 87d26b9af..ca677d9db 100644 --- a/this.config.json +++ b/this.config.json @@ -43,7 +43,6 @@ } ], "stopGracefully": "false", - "exchangeSimulationMode": "false", "storageConnStringFolder": "Production", "maxLogLoops": 5, "marketRateProvider": {