Several Functions Implemented

Trading-With-Stairs/Interval.js

@@ -32,11 +32,14 @@
         start: start
     };
 
+    let fs = require('fs');
+
     let markets;
 
     let charlyAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
     let oliviaAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
     let tomAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
+    let mariamAzureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
 
     let utilities = UTILITIES.newUtilities(bot);
 
@@ -61,6 +64,7 @@
             charlyAzureFileStorage.initialize("Charly");
             oliviaAzureFileStorage.initialize("Olivia");
             tomAzureFileStorage.initialize("Tom");
+            mariamAzureFileStorage.initialize("Mariam");
 
             markets = MARKETS_MODULE.newMarkets(bot);
             markets.initialize(callBackFunction);
@@ -125,6 +129,27 @@
                 assetB: "BTC",
             };
 
+            /*
+
+            Here we will keep the last status report, to be available during the whole process. The Status Report is a file the bot process
+            reads and saves again after each execution. Its main purpose is to know when the last execution was in order to locate the execution
+            context. When the bot runs for the first time it takes some vital parameters from there and it checks them through its lifecycle to see
+            if they changed. The Status Report file can eventually be manipulated by the bot operators / developers in order to change those parameters
+            or to point the last execution to a different date. Humans are not supose to manipulate the Execution Histroy or the execution Context files.
+
+            The Execution History is basically an index with dates of all the executions the bot did across its history. It allows the bot plotter
+            to know which datetimes have informacion about the bots execution in order to display it.
+
+            The Execution Context file records all the context information of the bot at the moment of execution and the final state of all of its
+            positions on the market.
+
+            */
+
+            let statusReport;           // 
+            let executionHistory;       // This is 
+            let executionContext;       // Here is the information of the last execution of this bot process.
+
+
             let exchangePositions = [];
             let openPositions = [];
 
@@ -138,7 +163,7 @@
 
                 try {
 
-                    return JSON.parse(fs.readFileSync('../../' + 'API-Keys' + EXCHANGE_NAME, 'utf8'));
+                    return JSON.parse(fs.readFileSync('../' + 'API-Keys' + '/' + EXCHANGE_NAME + '.json', 'utf8'));
 
                 }
                 catch (err) {
@@ -209,19 +234,47 @@
 
                         } else {
 
-                            exchangePositions = exchangeResponse;
-                            readBotPositions();
+                            /*
+
+                            This is what we receive from the exchange. We will convert this to our standard format for later use.
+
+                            [ { orderNumber: '151918418632',
+                                type: 'sell',
+                                rate: '20000.00000000',
+                                startingAmount: '0.00010000',
+                                amount: '0.00010000',
+                                total: '2.00000000',
+                                date: '2018-02-24 11:14:17',
+                                margin: 0 } ]
+
+                            */
+
+                            for (let i = 0; i < exchangeResponse.length; i++) {
+
+                                let openPosition = {
+                                    id: exchangeResponse[i].orderNumber,
+                                    type: exchangeResponse[i].type,
+                                    rate: exchangeResponse[i].rate,
+                                    amountA: exchangeResponse[i].total,
+                                    amountB: exchangeResponse[i].amount,
+                                    date: (new Date(exchangeResponse[i].date)).valueOf()
+                                }
+
+                                exchangePositions.push(openPosition);
+                            }
+
+                            getCloudFiles();
                         }
                     }
                     catch (err) {
                         const logText = "[ERROR] 'onExchangeCallReturned' - ERROR : " + err.message;
                         logger.write(logText);
-                        closeMarket();
+
                     }
                 }
             }
 
-            function readBotPositions() {
+            function getCloudFiles() {
 
                 /*
 
@@ -230,19 +283,458 @@
 
                 */
 
+                function getStatusReport() {
+
+                    /* If the process run and was interrupted, there should be a status report that allows us to resume execution. */
+
+                    let fileName = "Status.Report.json"
+                    let filePath = EXCHANGE_NAME + "/" + bot.name + "/" + bot.dataSetVersion + "/Processes/" + bot.process;
+
+                    mariamAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
+
+                    function onFileReceived(text) {
+
+                        let statusReport;
+
+                        try {
+
+                            statusReport = JSON.parse(text);
+
+                            if (statusReport.lastExecution === undefined) {
+
+                                createCloudFiles();
+
+                            } else {
+
+                                getExecutionHistory();
+
+                            }
+
+                        } catch (err) {
+
+                            /*
+
+                            It might happen that the file content is corrupt or it does not exist. The bot can not run without a Status Report,
+                            since it is risky to ignore its own history, so even for first time execution, a status report with the right format
+                            is needed.
+
+                            */
+
+                            const logText = "[ERROR] 'getStatusReport' - Bot cannot execute without a status report. ERROR : " + err.message;
+                            logger.write(logText);
+
+                        }
+                    }
+                }
+
+                function getExecutionHistory() {
+
+                    let fileName = "Execution.History.json"
+                    let filePath = EXCHANGE_NAME + "/" + bot.name + "/" + bot.dataSetVersion + "/Output/" + bot.process;
+
+                    mariamAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
+
+                    function onFileReceived(text) {
+
+                        let statusReport;
+
+                        try {
+
+                            executionHistory = JSON.parse(text);
+                            getExecutionContext();
+
+                        } catch (err) {
+
+                            /*
+
+                            It might happen that the file content is corrupt or it does not exist. The bot can not run without a Status Report,
+                            since it is risky to ignore its own history, so even for first time execution, a status report with the right format
+                            is needed.
+
+                            */
+
+                            const logText = "[ERROR] 'getExecutionHistory' - Bot cannot execute without the Execution History. ERROR : " + err.message;
+                            logger.write(logText);
+
+                        }
+                    }
+                }
+
+                function getExecutionContext() {
+
+                    let fileName = "Execution.Context.json"
+                    let dateForPath = statusReport.lastExecution.year + '/' + utilities.pad(statusReport.lastExecution.month, 2) + '/' + utilities.pad(statusReport.lastExecution.day, 2) + '/' + utilities.pad(statusReport.lastExecution.hours, 2) + '/' + utilities.pad(statusReport.lastExecution.minutes, 2));
+                    let filePath = EXCHANGE_NAME + "/" + bot.name + "/" + bot.dataSetVersion + "/Output/" + bot.process + "/" + dateForPath;
+
+                    mariamAzureFileStorage.getTextFile(filePath, fileName, onFileReceived, true);
+
+                    function onFileReceived(text) {
+
+                        let statusReport;
+
+                        try {
+
+                            executionContext = JSON.parse(text);
+                            onsistencyCheck();
+
+                        } catch (err) {
+
+                            /*
+
+                            It might happen that the file content is corrupt or it does not exist. The bot can not run without a Status Report,
+                            since it is risky to ignore its own history, so even for first time execution, a status report with the right format
+                            is needed.
+
+                            */
+
+                            const logText = "[ERROR] 'getStatusReport' - Bot cannot execute without a status report. ERROR : " + err.message;
+                            logger.write(logText);
+
+                        }
+                    }
+                }
+
             }
 
-            function consistencyCheck() {
+            function createCloudFiles() {
+
+                /*
+
+                When the bot is executed for the very first time, there are a few files that do not exist and need to be created, and that
+                is what we are going to do now.
+
+                */
+
+                executionHistory = [];
+
+                executionContext = {
+                    investment: {
+                        assetA: 0,
+                        assetB: 0
+                    },
+                    availableBalance: {
+                        assetA: 0,
+                        assetB: 0
+                    },
+                    positions: [],
+                    transactions: []
+                };
+
+                getPatterns();
+
+            }
+
+            function ordersExecutionCheck() {
 
                 /*
 
                 Here we check that all the positions we know we have are still at the exchange. If they are not, we will try to take appropiate
-                actions.
+                actions. Reasons why the positions might not be there are:
+
+                1. The user / account owner closed the positions manually.
+                2. The exchange for some eventuality closed the positions. In some exchanges positions have an expiratin time.
+                3. The orders were executed.
+
+                For as 1 and 2 are similar unexpected situations and we will stop the bot execution when we detect we are there. Number 3 is an
+                expected behaviour and we take appropiate action.
+
+                Also, the position might still be there but it might have been partially executed. To detect that we need to compare the
+                position amounts we have on file to the one we are receiving from the exchange.
 
                 */
 
+                let i = 0;
+
+                controlLoop();
+
+                function loopBody() {
+
+                    for (let j = 0; j < exchangePositions.length; j++) {
+
+                        if (executionContext.positions[i].id === exchangePositions[j].id) {
+
+                            /*
+
+                            We need to know if the order was partially executed. To know that, we compare the amounts on file with the ones
+                            received from the exchange. If they are the same, then the order is intact. Otherwise, to confirm a partial execution,
+                            we will request the associated trades from the exchange.
+
+                            */
+
+                            if (
+                                executionContext.positions[i].amountA === exchangePositions[j].amountA &&
+                                executionContext.positions[i].amountB === exchangePositions[j].amountB) {
+
+                                /* Position is still there, untouched. Nothing to do here. */
+
+                                next();
+                                return;
+
+                            } else {
+
+                                getOrderTradesAtExchange(executionContext.positions[i].id, confirmOrderWasPartiallyExecuted);
+                                return;
+
+                                function confirmOrderWasPartiallyExecuted(trades) {
+
+                                    /*
+
+                                    To confirm everything is ok, we will add all the amounts on trades plus the remaining amounts
+                                    at the order at the exchange and they must be equal to the one on file. Otherwise something very strange could
+                                    have happened, in which case we will halt the bot execution.
+
+                                    */
+
+                                    let sumAssetA = 0;
+                                    let sumAssetB = 0;
+
+                                    for (k = 0; k < trades.length; k++) {
+
+                                        sumAssetA = sumAssetA + trades[k].amountA;
+                                        sumAssetB = sumAssetB + trades[k].amountB;
+
+                                    }
+
+                                    /* To this we add the current position amounts. */
+
+                                    sumAssetA = sumAssetA + exchangePositions[j].amountA;
+                                    sumAssetB = sumAssetB + exchangePositions[j].amountB;
+
+                                    /* And finally we add the fees */
+
+                                    sumAssetA = sumAssetA + exchangePositions[j].fee;
+
+                                    if (
+                                        executionContext.positions[i].amountA !== sumAssetA ||
+                                        executionContext.positions[i].amountB !== sumAssetB
+                                        ) {
+
+                                        const logText = "[ERROR] 'confirmOrderWasPartiallyExecuted' - Cannot be confirmed that a partially execution was done well. Bot stopping execution. ";
+                                        logger.write(logText);
+                                        return;
+
+                                    }
+
+                                    /*
+
+                                    Confirmed that order was partially executed. Next thing to do is to remember the trades and the new position.
+
+                                    */
+
+                                    executionContext.positions[i].amountA = exchangePositions[j].amountA;
+                                    executionContext.positions[i].amountB = exchangePositions[j].amountB;
+                                    executionContext.positions[i].date = (new Date(exchangePositions[j].date)).valueOf();
+                                    
+                                    for (k = 0; k < trades.length; k++) {
+
+                                        executionContext.positions[i].trades.push(trades[k]);
+
+                                    }
+
+                                    /* All done. */
+
+                                    next();
+
+                                }
+
+                            }
+                        }
+                    }
+
+                    /* Position not found: we need to know if the order was executed. */
+
+                    getOrderTradesAtExchange(executionContext.positions[i].id, confirmOrderWasExecuted);
+
+                    function confirmOrderWasExecuted(trades) {
+
+                        /*
+
+                        To confirm everything is ok, we will add all the amounts on trades asociated to the order and
+                        they must be equal to the one on file. Otherwise something very strange could have happened,
+                        in which case we will halt the bot execution.
+
+                        */
+
+                        let sumAssetA = 0;
+                        let sumAssetB = 0;
+
+                        for (k = 0; k < trades.length; k++) {
+
+                            sumAssetA = sumAssetA + trades[k].amountA;
+                            sumAssetB = sumAssetB + trades[k].amountB;
+
+                        }
+
+                        /* We add the fees */
+
+                        sumAssetA = sumAssetA + exchangePositions[j].fee;
+
+                        if (
+                            executionContext.positions[i].amountA !== sumAssetA ||
+                            executionContext.positions[i].amountB !== sumAssetB
+                        ) {
+
+                            const logText = "[ERROR] 'confirmOrderWasExecuted' - Cannot be confirmed that the order was executed. It must be manually cancelled by the user or cancelled by the exchange itself. Bot stopping execution. ";
+                            logger.write(logText);
+                            return;
+
+                        }
+
+                        /*
+
+                        Confirmed that order was executed. Next thing to do is to remember the trades.
+
+                        */
+
+                        for (k = 0; k < trades.length; k++) {
+
+                            executionContext.positions[i].trades.push(trades[k]);
+
+                        }
+
+                        /* All done. */
+
+                        next();
+
+                    }
+
+                }
+
+                function next() {
+
+                    i++;
+
+                }
+
+                function controlLoop() {
+
+                    if (i < executionContext.positions.length) {
+
+                        loopBody();
+
+                    } else {
+
+                        final();
+
+                    }
+                }
+
+                function final() {
+
+                    getPatterns();
+
+                }
             }
 
+            function getOrderTradesAtExchange(orderId, callBackFunction) {
+
+                /*
+
+                Given one order, we request all the associated trades to it.
+
+                */
+
+                poloniexApiClient.returnOrderTrades(orderId, onExchangeCallReturned);
+
+                function onExchangeCallReturned(err, exchangeResponse) {
+
+                    try {
+
+                        if (err || exchangeResponse.error !== undefined) {
+                            try {
+
+                                if (err.message.indexOf("ETIMEDOUT") > 0) {
+
+                                    const logText = "[WARN] onExchangeCallReturned - Timeout reached while trying to access the Exchange API. Requesting new execution later. : ERROR = " + err.message;
+                                    logger.write(logText);
+
+                                    /* We abort the process and request a new execution at the configured amount of time. */
+
+                                    callBackFunction(true, nextIntervalLapse);
+                                    return;
+
+                                } else {
+
+                                    if (err.message.indexOf("ECONNRESET") > 0) {
+
+                                        const logText = "[WARN] onExchangeCallReturned - The exchange reseted the connection. Requesting new execution later. : ERROR = " + err.message;
+                                        logger.write(logText);
+
+                                        /* We abort the process and request a new execution at the configured amount of time. */
+
+                                        callBackFunction(true, nextIntervalLapse);
+                                        return;
+
+                                    } else {
+
+
+                                        const logText = "[ERROR] onExchangeCallReturned - Unexpected error trying to contact the Exchange. This will halt this bot process. : ERROR = " + err.message;
+                                        logger.write(logText);
+                                        closeMarket();
+                                        return;
+                                    }
+                                }
+
+                            } catch (err) {
+                                const logText = "[ERROR] onExchangeCallReturned : ERROR : exchangeResponse.error = " + exchangeResponse.error;
+                                logger.write(logText);
+                                return;
+                            }
+
+                            return;
+
+                        } else {
+
+                            /*
+
+                            This is what we receive from the exchange. We will convert this to our standard format for later use.
+
+                            [
+                            {
+                            "globalTradeID": 20825863,
+                            "tradeID": 147142,
+                            "currencyPair":
+                            "BTC_XVC",
+                            "type": "buy",
+                            "rate": "0.00018500",
+                            "amount": "455.34206390",
+                            "total": "0.08423828",
+                            "fee": "0.00200000",
+                            "date": "2016-03-14 01:04:36"
+                            },
+                            ...]
+
+                            */
+
+                            let trades = [];
+
+                            for (let i = 0; i < exchangeResponse.length; i++) {
+
+                                let trade = {
+                                    id: exchangeResponse[i].tradeID,
+                                    type: exchangeResponse[i].type,
+                                    rate: exchangeResponse[i].rate,
+                                    amountA: exchangeResponse[i].total,
+                                    amountB: exchangeResponse[i].amount,
+                                    fee: exchangeResponse[i].fee,
+                                    date: (new Date(exchangeResponse[i].date)).valueOf()
+                                }
+
+                                trades.push(trade);
+                            }
+
+                            callBackFunction(trades);
+                        }
+                    }
+                    catch (err) {
+                        const logText = "[ERROR] 'onExchangeCallReturned' - ERROR : " + err.message;
+                        logger.write(logText);
+
+                    }
+                }
+            }
+
+
             function getPatterns() {
 
                 /*