From edc3f86ad828858e5369a686bcbd9321b6b9ad46 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 22 Sep 2021 12:01:05 +0200 Subject: [PATCH 1/4] Governance: Auto processing of PRs --- Platform/Client/httpInterface.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Platform/Client/httpInterface.js b/Platform/Client/httpInterface.js index 0ff9a8061..cbe0d20aa 100644 --- a/Platform/Client/httpInterface.js +++ b/Platform/Client/httpInterface.js @@ -1067,6 +1067,15 @@ exports.newHttpInterface = function newHttpInterface() { ) SA.projects.foundations.utilities.httpResponses.respondWithContent(JSON.stringify(serverResponse), httpResponse) + + setInterval( + PL.servers.GITHUB_SERVER.mergePullRequests, + 60000, + params.commitMessage, + params.username, + params.token + ) + return } case 'payContributors': { From 325163e5facd356c70c0de8121c103a3f9e13411 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 22 Sep 2021 13:12:54 +0200 Subject: [PATCH 2/4] Desktop & Network working with Secrets --- Desktop/DesktopApp.js | 8 ++++---- DesktopRoot.js | 19 +++++++++---------- NetworkRoot.js | 18 +++++++++--------- .../Function-Libraries/ProfileConstructor.js | 2 ++ .../Network/NT/Modules/WebSocketsInterface.js | 6 +++--- .../Network/SA/Modules/WebSocketsClient.js | 15 ++++++--------- .../DK/modules/WebAppInterface.js | 7 ++----- .../Social-Trading/NT/Modules/Bootstrap.js | 15 ++++++++++++++- 8 files changed, 49 insertions(+), 41 deletions(-) diff --git a/Desktop/DesktopApp.js b/Desktop/DesktopApp.js index 66838c3fe..77cf0f03d 100644 --- a/Desktop/DesktopApp.js +++ b/Desktop/DesktopApp.js @@ -45,7 +45,7 @@ exports.newDesktopApp = function newDesktopApp() { */ queryMessage = { queryType: SA.projects.socialTrading.globals.queryTypes.USER_PROFILES, - emitterUserProfileId: DK.TEST_NETWORK_CLIENT_USER_PROFILE_ID, + emitterUserProfileId: SA.secrets.map.get('Social Trading Desktop').userProfileId, initialIndex: 'Last', amountRequested: 10, direction: 'Past' @@ -74,7 +74,7 @@ exports.newDesktopApp = function newDesktopApp() { let eventMessage = { eventId: SA.projects.foundations.utilities.miscellaneousFunctions.genereteUniqueId(), eventType: SA.projects.socialTrading.globals.eventTypes.FOLLOW_USER_PROFILE, - emitterUserProfileId: DK.TEST_NETWORK_CLIENT_USER_PROFILE_ID, + emitterUserProfileId: SA.secrets.map.get('Social Trading Desktop').userProfileId, targetUserProfileId: profile.userProfileId } @@ -99,8 +99,8 @@ exports.newDesktopApp = function newDesktopApp() { */ queryMessage = { queryType: SA.projects.socialTrading.globals.queryTypes.USER_PROFILE_STATS, - emitterUserProfileId: DK.TEST_NETWORK_CLIENT_USER_PROFILE_ID, - targetUserProfileId: DK.TEST_NETWORK_CLIENT_USER_PROFILE_ID + emitterUserProfileId: SA.secrets.map.get('Social Trading Desktop').userProfileId, + targetUserProfileId: SA.secrets.map.get('Social Trading Desktop').userProfileId } query = { diff --git a/DesktopRoot.js b/DesktopRoot.js index 87b9e57f9..ee357f8e2 100644 --- a/DesktopRoot.js +++ b/DesktopRoot.js @@ -53,17 +53,16 @@ exports.newDesktopRoot = function newDesktopRoot() { } /* - We will use this User Profile for Testing Purposes. - { - "githubUsername": "Test-Network-Client-Profile", - "address": "0xA0dCDA81b42C5EBDc45EC2875C72274B67560246", - "privateKey": "0x5fed4817a87431c8a328d0936561e63f45b6db289adac6125b9306ded644dfe9" - } + Setting up Secrets. */ - - DK.TEST_NETWORK_CLIENT_USER_PROFILE_ID = "baa2f02c-2920-4edb-a103-d452fc61d82f" - DK.TEST_NETWORK_CLIENT_USER_PROFILE_HANDLE = "Test-Network-Client-Profile" - DK.TEST_NETWORK_CLIENT_USER_PROFILE_PRIVATE_KEY = "0x5fed4817a87431c8a328d0936561e63f45b6db289adac6125b9306ded644dfe9" + SA.secrets = { + array: require('./My-Secrets/Secrets.json'), + map: new Map() + } + for (let i = 0; i < SA.secrets.array.length; i++) { + let secret = SA.secrets.array[i] + SA.secrets.map.set (secret.codeName, secret) + } run() diff --git a/NetworkRoot.js b/NetworkRoot.js index a0e5bbebf..d4bbf0c46 100644 --- a/NetworkRoot.js +++ b/NetworkRoot.js @@ -48,17 +48,17 @@ exports.newNetworkRoot = function newNetworkRoot() { ws: require('ws'), simpleGit: require('simple-git') } - /* - We will use this User Profile for testing purposes. - { - "githubUsername": "Test-Network-Node-Profile", - "address": "0xa153469c57A91F5a59Fc6c45A37aD8dbad85e417", - "privateKey": "0xac498ae59407e6b68429a814f64da2339550f93a767578e28c161ff119159271" - } + Setting up Secrets. */ - NT.NETWORK_NODE_USER_PROFILE_HANDLE = "Test-Network-Node-Profile" - NT.NETWORK_NODE_USER_PROFILE_PRIVATE_KEY = "0xac498ae59407e6b68429a814f64da2339550f93a767578e28c161ff119159271" + SA.secrets = { + array: require('./My-Secrets/Secrets.json'), + map: new Map() + } + for (let i = 0; i < SA.secrets.array.length; i++) { + let secret = SA.secrets.array[i] + SA.secrets.map.set(secret.codeName, secret) + } NT.app = require('./Network/NetwokNode.js').newNetworkNode() NT.app.run() diff --git a/Projects/Governance/UI/Function-Libraries/ProfileConstructor.js b/Projects/Governance/UI/Function-Libraries/ProfileConstructor.js index b772610c8..09562d135 100644 --- a/Projects/Governance/UI/Function-Libraries/ProfileConstructor.js +++ b/Projects/Governance/UI/Function-Libraries/ProfileConstructor.js @@ -306,11 +306,13 @@ function newGovernanceFunctionLibraryProfileConstructor() { node.config = "{}" UI.projects.foundations.utilities.nodeConfig.saveConfigProperty(node.payload, 'nodeName', signingAccount.name) + UI.projects.foundations.utilities.nodeConfig.saveConfigProperty(node.payload, 'codeName', signingAccount.name) UI.projects.foundations.utilities.nodeConfig.saveConfigProperty(node.payload, 'nodeType', signingAccount.type) UI.projects.foundations.utilities.nodeConfig.saveConfigProperty(node.payload, 'nodeId', signingAccount.id) UI.projects.foundations.utilities.nodeConfig.saveConfigProperty(node.payload, 'address', address) UI.projects.foundations.utilities.nodeConfig.saveConfigProperty(node.payload, 'privateKey', privateKey) UI.projects.foundations.utilities.nodeConfig.saveConfigProperty(node.payload, 'githubUsername', githubUsername) + UI.projects.foundations.utilities.nodeConfig.saveConfigProperty(node.payload, 'userProfileId', node.payload.referenceParent.id) /* Show nice message. */ diff --git a/Projects/Network/NT/Modules/WebSocketsInterface.js b/Projects/Network/NT/Modules/WebSocketsInterface.js index 07bb3e0c3..72d0bb8c3 100644 --- a/Projects/Network/NT/Modules/WebSocketsInterface.js +++ b/Projects/Network/NT/Modules/WebSocketsInterface.js @@ -242,11 +242,11 @@ exports.newNetworkModulesWebSocketsInterface = function newNetworkModulesWebSock */ let signedMessage = { callerProfileHandle: messageHeader.callerProfileHandle, - calledProfileHandle: NT.NETWORK_NODE_USER_PROFILE_HANDLE, + calledProfileHandle: SA.secrets.map.get('P2P Network Node').githubUsername, callerTimestamp: messageHeader.callerTimestamp, calledTimestamp: calledTimestamp } - let signature = web3.eth.accounts.sign(JSON.stringify(signedMessage), NT.NETWORK_NODE_USER_PROFILE_PRIVATE_KEY) + let signature = web3.eth.accounts.sign(JSON.stringify(signedMessage), SA.secrets.map.get('P2P Network Node').privateKey) let response = { result: 'Ok', @@ -322,7 +322,7 @@ exports.newNetworkModulesWebSocketsInterface = function newNetworkModulesWebSock We will check that the signature includes this Network Node handle, to avoid man in the middle attackts. */ - if (signedMessage.calledProfileHandle !== NT.NETWORK_NODE_USER_PROFILE_HANDLE) { + if (signedMessage.calledProfileHandle !== SA.secrets.map.get('P2P Network Node').githubUsername) { let response = { result: 'Error', message: 'calledProfileHandle Does Not Match This Network Node Handle.' diff --git a/Projects/Network/SA/Modules/WebSocketsClient.js b/Projects/Network/SA/Modules/WebSocketsClient.js index b78a9d58b..04807076d 100644 --- a/Projects/Network/SA/Modules/WebSocketsClient.js +++ b/Projects/Network/SA/Modules/WebSocketsClient.js @@ -30,16 +30,13 @@ exports.newNetworkModulesWebSocketsClient = function newNetworkModulesWebSockets /* Here we will pick a Network Node from all users profiles available that do have a Network Node running. // TODO In the meantime, we will assume that we have chosen the following Network Node to connect to. - { - "githubUsername": "Test-Network-Node-Profile", - "address": "0xa153469c57A91F5a59Fc6c45A37aD8dbad85e417" - } + */ web3 = new SA.nodeModules.web3() selectedNetworkNode = { - userProfileHandle: "Test-Network-Node-Profile", - blockchainAccount: "0xa153469c57A91F5a59Fc6c45A37aD8dbad85e417", + userProfileHandle: "Luis-Fernando-Molina", + blockchainAccount: "0xeBDCB7a73c4796ca9F025d005630eCe773dd9e54", ranking: 0, host: "localhost", port: global.env.NETWORK_WEB_SOCKETS_INTERFACE_PORT @@ -90,7 +87,7 @@ exports.newNetworkModulesWebSocketsClient = function newNetworkModulesWebSockets let message = { messageType: 'Handshake', callerRole: 'Network Client', - callerProfileHandle: DK.TEST_NETWORK_CLIENT_USER_PROFILE_HANDLE, + callerProfileHandle: SA.secrets.map.get('Social Trading Desktop').githubUsername, callerTimestamp: callerTimestamp, step: 'One' } @@ -141,7 +138,7 @@ exports.newNetworkModulesWebSocketsClient = function newNetworkModulesWebSockets We will check that the profile handle we sent to the Network Node, is returned at the signed message, to avoid man in the middle attackts. */ - if (signedMessage.callerProfileHandle !== DK.TEST_NETWORK_CLIENT_USER_PROFILE_HANDLE) { + if (signedMessage.callerProfileHandle !== SA.secrets.map.get('Social Trading Desktop').githubUsername) { console.log('[ERROR] Web Sockets Client -> stepOneResponse -> The Network Node callerProfileHandle does not match my own userProfileHandle.') reject() return @@ -171,7 +168,7 @@ exports.newNetworkModulesWebSocketsClient = function newNetworkModulesWebSockets */ socketClient.onmessage = socketMessage => { stepTwoResponse(socketMessage) } - let signature = web3.eth.accounts.sign(JSON.stringify(signedMessage), DK.TEST_NETWORK_CLIENT_USER_PROFILE_PRIVATE_KEY) + let signature = web3.eth.accounts.sign(JSON.stringify(signedMessage), SA.secrets.map.get('Social Trading Desktop').privateKey) let message = { messageType: 'Handshake', diff --git a/Projects/Social-Trading/DK/modules/WebAppInterface.js b/Projects/Social-Trading/DK/modules/WebAppInterface.js index 609cb33c5..eb232bc70 100644 --- a/Projects/Social-Trading/DK/modules/WebAppInterface.js +++ b/Projects/Social-Trading/DK/modules/WebAppInterface.js @@ -44,7 +44,7 @@ exports.newSocialTradingModulesWebAppInterface = function newSocialTradingModule } return JSON.stringify(response) } - queryMessage.emitterUserProfileId = DK.TEST_NETWORK_CLIENT_USER_PROFILE_ID + queryMessage.emitterUserProfileId = SA.secrets.map.get('Social Trading Desktop').userProfileId messageHeader.queryMessage = JSON.stringify(queryMessage) let response @@ -123,7 +123,7 @@ exports.newSocialTradingModulesWebAppInterface = function newSocialTradingModule eventMessage.postText = undefined } - eventMessage.emitterUserProfileId = DK.TEST_NETWORK_CLIENT_USER_PROFILE_ID + eventMessage.emitterUserProfileId = SA.secrets.map.get('Social Trading Desktop').userProfileId messageHeader.eventMessage = JSON.stringify(eventMessage) let response = { @@ -175,9 +175,6 @@ exports.newSocialTradingModulesWebAppInterface = function newSocialTradingModule async function getPostText(userProfileHandle, postHash, timestamp) { let promise = new Promise((resolve, reject) => { -//https://raw.githubusercontent.com/Luis-Fernando-Molina/My-Social-Trading-Data/main/User-Posts/2021/09/17/0608070d8d2aab12e63cf3aa871918a04e237edff4fc0f886d158311811cf7c9.json - - const userProfileHandle = "Luis-Fernando-Molina" /// TODO Delete this line const fileName = postHash + ".json" const filePath = 'My-Social-Trading-Data/main/User-Posts/' + SA.projects.foundations.utilities.filesAndDirectories.pathFromDate(timestamp) diff --git a/Projects/Social-Trading/NT/Modules/Bootstrap.js b/Projects/Social-Trading/NT/Modules/Bootstrap.js index 58ab6bbe1..f46a016b7 100644 --- a/Projects/Social-Trading/NT/Modules/Bootstrap.js +++ b/Projects/Social-Trading/NT/Modules/Bootstrap.js @@ -52,7 +52,20 @@ exports.newBootstrap = function newBootstrap() { NT.projects.socialTrading.globals.memory.maps.USER_PROFILES_BY_ID.set(userProfileId, userProfile) NT.projects.socialTrading.globals.memory.maps.USER_PROFILES_BY_HANDLE.set(userHandle, userProfile) NT.projects.socialTrading.globals.memory.maps.USER_PROFILES_BY_BLOCHAIN_ACCOUNT.set(blockchainAccount, userProfile) - + /* + Each User Profile might have Signing Accounts, meaning + accounts that can be siging on behalf of the User Profile. + */ + if (userProfilePlugin.signingAccounts !== undefined) { + for (let j = 0; j < userProfilePlugin.signingAccounts.signingAccounts.length; j++) { + let signingAccount = userProfilePlugin.signingAccounts.signingAccounts[j] + let config = JSON.parse(signingAccount.config) + let signatureObject = config.signature + let web3 = new SA.nodeModules.web3() + let blockchainAccount = web3.eth.accounts.recover(signatureObject) + NT.projects.socialTrading.globals.memory.maps.USER_PROFILES_BY_BLOCHAIN_ACCOUNT.set(blockchainAccount, userProfile) + } + } } } } \ No newline at end of file From ce336a3f8de1998760be0bfed1c506163dbffae9 Mon Sep 17 00:00:00 2001 From: Ferrox85 <64791738+Ferrox85@users.noreply.github.com> Date: Wed, 22 Sep 2021 13:38:39 +0200 Subject: [PATCH 3/4] Fixet empty rows and undefined strategy values Complete fix of the CSV snapshot generator. Implemented the trigger values and overwrite the exit values at exit state with keeping the trigger state values. Removed the empty lines in the file. --- .../Low-Frequency-Trading/Snapshots.js | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/Snapshots.js b/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/Snapshots.js index 1349d7993..065b6833c 100644 --- a/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/Snapshots.js +++ b/Projects/Foundations/TS/Bot-Modules/Trading-Bot/Low-Frequency-Trading/Snapshots.js @@ -62,6 +62,10 @@ exports.newFoundationsBotModulesSnapshots = function(processIndex) { } function finalize() { + //trigger exits added for extra save moment for datapreserve + strategyExit() + positionExit() + writeSnapshotFiles() tradingEngine = undefined tradingSystem = undefined @@ -220,8 +224,8 @@ exports.newFoundationsBotModulesSnapshots = function(processIndex) { instruction = instruction.replace(/!==/g, '') instruction = instruction.replace(/==/g, '') instruction = instruction.replace(/===/g, '') - instruction = instruction.replace(/{/g, '') //this wrecks something in JS markup? - instruction = instruction.replace(/}/g, '') //this wrecks something in JS markup? + instruction = instruction.replace(/{/g, '') + instruction = instruction.replace(/}/g, '') if (instruction.indexOf('chart') >= 0) { let parts = instruction.split('.') let timeFrame = parts[1] @@ -243,7 +247,6 @@ exports.newFoundationsBotModulesSnapshots = function(processIndex) { } let key = timeFrame + '-' + product + '-' + property let existingKey - if (addToStrategyValues === true) { addValues(strategyKeys, strategyValues) } @@ -373,15 +376,35 @@ exports.newFoundationsBotModulesSnapshots = function(processIndex) { function writeOutput(fileContent) { let separator = '\r\n'; + let contentArray = []; + + // if we have file content extract rows + if (fileContent != '') { + contentArray = fileContent.split(separator) + } parseRecord(snapshots.headers) - for (let i = 0; i < snapshotArray.length; i++) { let record = snapshotArray[i]; - parseRecord(record) + if (record.length > 0) { + parseRecord(record) + } } function parseRecord(record) { + // if we have undefined values, we need to extract the previous stored values for this trade and append them to the new record for completion + if (record[record.length - 1] === undefined) { + + let rowToMutateArray = contentArray[contentArray.length - 1].split(',') + // check if are for sure on the same tradenumber and merge the data and remove previous line incl separator + if (rowToMutateArray[0] == record[0]) { + record.pop() + rowToMutateArray.splice(0, record.length) + record = record.concat(rowToMutateArray) + fileContent = fileContent.replace(contentArray[contentArray.length - 1], '') + fileContent = fileContent.replace(/\r\n$/, '') + } + } for (let j = 0; j < record.length; j++) { let property = record[j] @@ -426,6 +449,9 @@ exports.newFoundationsBotModulesSnapshots = function(processIndex) { function onFileReceived(err, text) { if (err.result === TS.projects.foundations.globals.standardResponses.DEFAULT_OK_RESPONSE.result) { try { + // remove the last linebreak and seperator wich is introduced extra during save + text = text.replace(/\n$/, '') + text = text.replace(/\r\n$/, '') writeOutput(text) } catch (err) { TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, '[ERROR] start -> loadExistingFiles -> loopBody -> readExistingFile -> onFileReceived -> fileName = ' + filePath); From 2a373a243f97044ec79983506c56fdcc292e5885 Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Wed, 22 Sep 2021 14:34:37 +0200 Subject: [PATCH 4/4] Desktop & Network validating Signature Hash --- .../Network/NT/Modules/WebSocketsInterface.js | 17 +++++++++++++++-- Projects/Network/SA/Modules/WebSocketsClient.js | 12 +++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Projects/Network/NT/Modules/WebSocketsInterface.js b/Projects/Network/NT/Modules/WebSocketsInterface.js index 72d0bb8c3..1e4c548dc 100644 --- a/Projects/Network/NT/Modules/WebSocketsInterface.js +++ b/Projects/Network/NT/Modules/WebSocketsInterface.js @@ -304,12 +304,25 @@ exports.newNetworkModulesWebSocketsInterface = function newNetworkModulesWebSock caller.socket.send(JSON.stringify(response)) return } + let signedMessage = JSON.parse(signature.message) + /* + We will verify that the signature belongs to the signature.message. + To do this we will hash the signature.message and see if we get + the same hash of the signature. + */ + let hash = web3.eth.accounts.hashMessage(signature.message) + if (hash !== signature.messageHash) { + let response = { + result: 'Error', + message: 'signature.message Hashed Does Not Match signature.messageHash.' + } + caller.socket.send(JSON.stringify(response)) + return + } /* The user profile based on the blockchain account, based on the signature, it is our witness user profile, to validate the caller. */ - let signedMessage = JSON.parse(signature.message) - if (signedMessage.callerProfileHandle !== witnessUserProfile.userProfileHandle) { let response = { result: 'Error', diff --git a/Projects/Network/SA/Modules/WebSocketsClient.js b/Projects/Network/SA/Modules/WebSocketsClient.js index 04807076d..3518d86fd 100644 --- a/Projects/Network/SA/Modules/WebSocketsClient.js +++ b/Projects/Network/SA/Modules/WebSocketsClient.js @@ -124,7 +124,17 @@ exports.newNetworkModulesWebSocketsClient = function newNetworkModulesWebSockets } let signedMessage = JSON.parse(signature.message) - + /* + We will verify that the signature belongs to the signature.message. + To do this we will hash the signature.message and see if we get + the same hash of the signature. + */ + let hash = web3.eth.accounts.hashMessage(signature.message) + if (hash !== signature.messageHash) { + console.log('[ERROR] Web Sockets Client -> stepOneResponse -> signature.message Hashed Does Not Match signature.messageHash.') + reject() + return + } /* We will check that the Network Node that responded has the same User Profile Handle that we have on record, otherwise something is wrong and we should not proceed.