Merge branch 'develop' of https://github.com/Superalgos/Superalgos into develop

This commit is contained in:
gitforkgit committed 2021-10-15 16:45:42 +02:00
commit b1224f842c
10 files changed
+5436 -9521

No files matched your search

+1 -1
View File
@@ -5,7 +5,7 @@
"**/node_modules": true,
"**/My-Social-Trading-Data": true,
"**/My-Network-Nodes-Data": true,
"**/My-Workspaces": true,
"**/My-Workspaces": false,
"**/My-Secrets": true,
"**/My-Data-Storage": true,
"**/My-Log-Files": true,
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,619 @@
{
"type": "User Profile",
"name": "primoz990",
"config": "{\n \"signature\": {\n \"message\": \"primoz990\",\n \"messageHash\": \"0xaec850cb01d8c93bb6009d84114cee5661dd32e0f82a52248d53d95fca94bb37\",\n \"v\": \"0x1b\",\n \"r\": \"0x05de161bdde18a6fd7e6003481c16340ee32a2a905c0cbb7d6f77fca880c3cea\",\n \"s\": \"0x6713810cce4c3020daccf37ebcaeceb4a2a4a99ef81c1c618cf218f7d2bd7f2c\",\n \"signature\": \"0x05de161bdde18a6fd7e6003481c16340ee32a2a905c0cbb7d6f77fca880c3cea6713810cce4c3020daccf37ebcaeceb4a2a4a99ef81c1c618cf218f7d2bd7f2c1b\"\n }\n}",
"project": "Governance",
"tokenPowerSwitch": {
"type": "Token Power Switch",
"name": "New Token Power Switch",
"config": "{}",
"project": "Governance",
"tokenPowerSwitch": [],
"onboardingPrograms": {
"type": "Onboarding Programs",
"name": "New Onboarding Programs",
"config": "{\n\"percentage\": 0\n}",
"project": "Governance",
"airdropProgram": {
"type": "Airdrop Program",
"name": "New Airdrop Program",
"config": "{}",
"project": "Governance",
"tokensAwarded": {
"type": "Tokens Awarded",
"name": "New Tokens Awarded",
"project": "Governance",
"id": "e2dec054-20c2-426e-8303-4a6f475a02e0",
"savedPayload": {
"position": {
"x": 13249,
"y": 8866.077136594007
},
"targetPosition": {
"x": 13399,
"y": 9125.884757729338
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"id": "80ffa07e-7b69-4d76-a315-67ca38b1d340",
"savedPayload": {
"position": {
"x": 13399,
"y": 9125.884757729338
},
"targetPosition": {
"x": 13549,
"y": 9385.692378864669
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"githubProgram": {
"type": "Github Program",
"name": "New Github Program",
"config": "{}",
"project": "Governance",
"tokensAwarded": {
"type": "Tokens Awarded",
"name": "New Tokens Awarded",
"project": "Governance",
"id": "c4ed9b78-6a1f-44fc-b9dd-29b54a5b40a5",
"savedPayload": {
"position": {
"x": 14149,
"y": 9385.692378864669
},
"targetPosition": {
"x": 13849,
"y": 9385.692378864669
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"id": "22e35987-f640-40cf-b9e2-fe3af87b5bfd",
"savedPayload": {
"position": {
"x": 13849,
"y": 9385.692378864669
},
"targetPosition": {
"x": 13549,
"y": 9385.692378864669
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"id": "83f57cc3-8e3f-40a6-9f79-15ecd7732094",
"savedPayload": {
"position": {
"x": 13549,
"y": 9385.692378864669
},
"targetPosition": {
"x": 13399,
"y": 9645.5
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"communityBuildingPrograms": {
"type": "Community Building Programs",
"name": "New Community Building Programs",
"config": "{}",
"project": "Governance",
"referralProgram": {
"type": "Referral Program",
"name": "New Referral Program",
"config": "{}",
"project": "Governance",
"userReferrers": [
{
"type": "User Referrer",
"name": "Luis-Fernando-Molina",
"config": "{}",
"project": "Governance",
"id": "4cea4999-b3e8-46eb-9daa-c65f7a08bcfe",
"savedPayload": {
"position": {
"x": 13658.807621135331,
"y": 9495.5
},
"targetPosition": {
"x": 13808.807621135331,
"y": 9755.307621135331
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
},
"referenceParent": {
"type": "User Profile",
"name": "Luis-Fernando-Molina",
"id": "a8de78f0-c3e4-4a2a-b7e8-f659073969db"
},
"referenceParentCombinedNodePath": [
[
"Luis-Fernando-Molina",
"User Profile"
]
]
}
}
],
"tokensAwarded": {
"type": "Tokens Awarded",
"name": "New Tokens Awarded",
"project": "Governance",
"id": "6167c8e9-4a50-4e57-8b59-049c24b6dc38",
"savedPayload": {
"position": {
"x": 14068.615242270662,
"y": 9605.307621135331
},
"targetPosition": {
"x": 13808.807621135331,
"y": 9755.307621135331
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"tokensBonus": {
"type": "Tokens Bonus",
"name": "New Tokens Bonus",
"project": "Governance",
"id": "48882651-30fc-4e3b-a2ef-7525958f30e5",
"savedPayload": {
"position": {
"x": 13958.807621135331,
"y": 10015.115242270662
},
"targetPosition": {
"x": 13808.807621135331,
"y": 9755.307621135331
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"id": "0c2752bc-b508-44d4-90ef-827eab524906",
"savedPayload": {
"position": {
"x": 13808.807621135331,
"y": 9755.307621135331
},
"targetPosition": {
"x": 13549,
"y": 9905.307621135331
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"supportProgram": {
"type": "Support Program",
"name": "New Support Program",
"config": "{}",
"project": "Governance",
"userSupporters": [
{
"type": "User Supporter",
"name": "Luis-Fernando-Molina",
"config": "{}",
"project": "Governance",
"id": "bd0b08dd-e53b-42c3-9e9d-6b6ed27d2b40",
"savedPayload": {
"position": {
"x": 13958.807621135331,
"y": 10015.115242270662
},
"targetPosition": {
"x": 13699,
"y": 10165.115242270662
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
},
"referenceParent": {
"type": "User Profile",
"name": "Luis-Fernando-Molina",
"id": "a8de78f0-c3e4-4a2a-b7e8-f659073969db"
},
"referenceParentCombinedNodePath": [
[
"Luis-Fernando-Molina",
"User Profile"
]
]
}
}
],
"tokensAwarded": {
"type": "Tokens Awarded",
"name": "New Tokens Awarded",
"project": "Governance",
"id": "4c96a952-a2c5-48ee-bef1-ad04ceea2c66",
"savedPayload": {
"position": {
"x": 13849,
"y": 10424.922863405993
},
"targetPosition": {
"x": 13699,
"y": 10165.115242270662
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"tokensBonus": {
"type": "Tokens Bonus",
"name": "New Tokens Bonus",
"project": "Governance",
"id": "c625c034-39c7-40e8-bf25-dfac9a375898",
"savedPayload": {
"position": {
"x": 13439.192378864669,
"y": 10315.115242270662
},
"targetPosition": {
"x": 13699,
"y": 10165.115242270662
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"id": "5f3c617e-2d40-495c-bae7-766ca9094e85",
"savedPayload": {
"position": {
"x": 13699,
"y": 10165.115242270662
},
"targetPosition": {
"x": 13549,
"y": 9905.307621135331
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"mentorshipProgram": {
"type": "Mentorship Program",
"name": "New Mentorship Program",
"config": "{}",
"project": "Governance",
"userMentors": [
{
"type": "User Mentor",
"name": "Luis-Fernando-Molina",
"config": "{}",
"project": "Governance",
"id": "ff88d3bf-2afd-4765-8956-8c10059378cf",
"savedPayload": {
"position": {
"x": 13384.288568297003,
"y": 10520.019052838328
},
"targetPosition": {
"x": 13159.288568297003,
"y": 10130.307621135331
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
},
"referenceParent": {
"type": "User Profile",
"name": "Luis-Fernando-Molina",
"id": "a8de78f0-c3e4-4a2a-b7e8-f659073969db"
},
"referenceParentCombinedNodePath": [
[
"Luis-Fernando-Molina",
"User Profile"
]
]
}
}
],
"tokensAwarded": {
"type": "Tokens Awarded",
"name": "New Tokens Awarded",
"project": "Governance",
"id": "e36dd618-f49d-4e29-8d62-27ca7fce9d6d",
"savedPayload": {
"position": {
"x": 12769.577136594005,
"y": 10355.307621135331
},
"targetPosition": {
"x": 13159.288568297003,
"y": 10130.307621135331
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"tokensBonus": {
"type": "Tokens Bonus",
"name": "New Tokens Bonus",
"project": "Governance",
"id": "d5feda5e-5da7-41ad-bcda-a56dcab3d327",
"savedPayload": {
"position": {
"x": 12934.288568297003,
"y": 9740.596189432334
},
"targetPosition": {
"x": 13159.288568297003,
"y": 10130.307621135331
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"id": "0c5fa5e4-d857-4e82-86ec-303528ff37e1",
"savedPayload": {
"position": {
"x": 13159.288568297003,
"y": 10130.307621135331
},
"targetPosition": {
"x": 13549,
"y": 9905.307621135331
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 4,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"id": "cecc4fa4-4486-4785-8f98-27d5b4e6f6b8",
"savedPayload": {
"position": {
"x": 13549,
"y": 9905.307621135331
},
"targetPosition": {
"x": 13399,
"y": 9645.5
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"id": "2f247174-4c0a-4a8b-9b21-33504b7e6be7",
"savedPayload": {
"position": {
"x": 13399,
"y": 9645.5
},
"targetPosition": {
"x": 13099,
"y": 9645.5
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"tokensMined": {
"type": "Tokens Mined",
"name": "New Tokens Mined",
"project": "Governance",
"id": "037c23e3-5f09-4756-b0fb-ead75bff85fe",
"savedPayload": {
"position": {
"x": 12799,
"y": 9645.5
},
"targetPosition": {
"x": 13099,
"y": 9645.5
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
},
"id": "50fc2e5f-8120-4e82-83b0-a3d871c90f1d",
"savedPayload": {
"position": {
"x": 13099,
"y": 9645.5
},
"targetPosition": {
"x": 13099,
"y": 9645.5
},
"floatingObject": {
"isPinned": false,
"isFrozen": false,
"isCollapsed": false,
"angleToParent": 1,
"distanceToParent": 3,
"arrangementStyle": 0
},
"uiObject": {
"isRunning": false
}
}
}
@@ -223,6 +223,10 @@ function newGovernanceFunctionLibraryVotingProgram() {
node.type === 'Asset' ||
node.type === 'Feature' ||
node.type === 'Pool' ||
node.type === 'Position Class' ||
node.type === 'Asset Class' ||
node.type === 'Feature Class' ||
node.type === 'Pool Class' ||
node.type === 'Position Contribution Claim' ||
node.type === 'Asset Contribution Claim' ||
node.type === 'Feature Contribution Claim'
@@ -285,7 +289,7 @@ function newGovernanceFunctionLibraryVotingProgram() {
/*
Setup the sign of the votes.
*/
let sign = 1
let sign = 1
let negative = UI.projects.visualScripting.utilities.nodeConfig.loadConfigProperty(node.payload, 'negative')
if (negative === true) {
sign = - 1
@@ -418,76 +422,75 @@ function newGovernanceFunctionLibraryVotingProgram() {
function drawVotes(node, votes, percentage, userProfile) {
if (node.payload !== undefined) {
if (node.type === 'User Profile') {
return
}
if (node.type === 'Voting Program') {
drawProgram(node, userProfile)
if (percentage !== undefined) {
node.payload.uiObject.setPercentage(percentage.toFixed(2),
UI.projects.governance.globals.designer.SET_PERCENTAGE_COUNTER
)
}
return
}
if (node.type === 'User Profile Vote') {
drawUserNode(node, votes, percentage)
return
}
if (
node.type === 'Position' ||
node.type === 'Position Class' ||
node.type === 'Pool' ||
node.type === 'Tokens Switch' ||
node.type === 'Asset' ||
node.type === 'Asset Class' ||
node.type === 'Feature' ||
node.type === 'Feature Class' ||
node.type === 'Position Contribution Claim' ||
node.type === 'Asset Contribution Claim' ||
node.type === 'Feature Contribution Claim'
) {
node.payload.uiObject.valueAngleOffset = 0
node.payload.uiObject.valueAtAngle = false
node.payload.uiObject.percentageAngleOffset = 0
node.payload.uiObject.percentageAtAngle = false
} else {
node.payload.uiObject.valueAngleOffset = 180
node.payload.uiObject.valueAtAngle = true
node.payload.uiObject.percentageAngleOffset = 180
node.payload.uiObject.percentageAtAngle = true
}
let voteType = 'Voting Power'
if (
node.type.indexOf('Weight') >= 0 ||
node.type.indexOf('Pool') >= 0 ||
node.type.indexOf('Feature') >= 0 ||
node.type.indexOf('Asset') >= 0
) {
voteType = 'Weight Power'
}
if (node.type.indexOf('Claim') >= 0) {
voteType = 'Claim Support Power'
}
const votesText = parseFloat(votes.toFixed(0)).toLocaleString('en') + ' ' + voteType
node.payload.uiObject.valueAngleOffset = 0
node.payload.uiObject.valueAtAngle = true
node.payload.uiObject.setValue(votesText, UI.projects.governance.globals.designer.SET_VALUE_COUNTER)
if (votes === undefined) { return }
if (isNaN(votes) === true) { return }
if (node.payload === undefined) { return }
if (node.type === 'User Profile') {
return
}
if (node.type === 'Voting Program') {
drawProgram(node, userProfile)
if (percentage !== undefined) {
node.payload.uiObject.setPercentage(percentage.toFixed(2),
UI.projects.governance.globals.designer.SET_PERCENTAGE_COUNTER
)
}
return
}
if (node.type === 'User Profile Vote') {
drawUserNode(node, votes, percentage)
return
}
if (
node.type === 'Position' ||
node.type === 'Position Class' ||
node.type === 'Pool' ||
node.type === 'Pool Class' ||
node.type === 'Tokens Switch' ||
node.type === 'Asset' ||
node.type === 'Asset Class' ||
node.type === 'Feature' ||
node.type === 'Feature Class' ||
node.type === 'Position Contribution Claim' ||
node.type === 'Asset Contribution Claim' ||
node.type === 'Feature Contribution Claim'
) {
return
}
node.payload.uiObject.valueAngleOffset = 180
node.payload.uiObject.valueAtAngle = true
node.payload.uiObject.percentageAngleOffset = 180
node.payload.uiObject.percentageAtAngle = true
let voteType = 'Voting Power'
if (
node.type.indexOf('Weight') >= 0 ||
node.type.indexOf('Pool') >= 0 ||
node.type.indexOf('Feature') >= 0 ||
node.type.indexOf('Asset') >= 0
) {
voteType = 'Weight Power'
}
if (node.type.indexOf('Claim') >= 0) {
voteType = 'Claim Support Power'
}
const votesText = parseFloat(votes.toFixed(0)).toLocaleString('en') + ' ' + voteType
node.payload.uiObject.setValue(votesText, UI.projects.governance.globals.designer.SET_VALUE_COUNTER)
if (percentage !== undefined) {
node.payload.uiObject.percentageAngleOffset = 180
node.payload.uiObject.percentageAtAngle = true
node.payload.uiObject.setPercentage(percentage.toFixed(2),
UI.projects.governance.globals.designer.SET_PERCENTAGE_COUNTER
)
}
function drawUserNode(node, votes, percentage) {
if (node.payload !== undefined) {
+5
View File
@@ -0,0 +1,5 @@
/* Load Environment Variables */
let ENVIRONMENT = require('./Environment.js');
let ENVIRONMENT_MODULE = ENVIRONMENT.newEnvironment()
global.env = ENVIRONMENT_MODULE
+61
View File
@@ -0,0 +1,61 @@
/**
* @name get list of links
*
* @desc Scrapes docs div for links
*/
const fs = require('fs');
const puppeteer = require('puppeteer');
// URL of the page we want to scrape
const url = 'http://localhost:34248';
const path = "docs/" + `page-${Date.now()}.html`;
(async () => {
const browser = await puppeteer.launch({ headless: false })
const page = await browser.newPage()
await page.setViewport({ width: 1280, height: 720 })
// Wait that network traffic stops. This is localhost, maybe not relevant?
await page.goto(url, {waitUntil: 'networkidle2'})
// Wait for page load and get the selector needed
await page.waitForSelector('.credits', {hidden: false});
//await page.waitForNavigation(5); // Another solution??
// Gather asset page urls for all the document pages
const assetUrls = await page.$$eval(
'a.onClick', onClick => { return anchors.map(anchor => anchor.textContent)});
const results = [];
// Visit each assets page one by one
for (let assetsUrl of assetUrls) {
await page.goto(assetsUrl);
// Now collect all the ICO urls.
const icoUrls = await page.$$eval(
'#page-wrapper > main > div.container > div > table > tbody > tr > td:nth-child(2) a',
links => links.map(link => link.href)
);
// Visit each ICO one by one and collect the data.
for (let icoUrl of icoUrls) {
await page.goto(icoUrl);
const icoImgUrl = await page.$eval('#asset-logo-wrapper img', img => img.src);
const icoName = await page.$eval('h1', h1 => h1.innerText.trim());
// TODO: Gather all the needed info like description etc here.
results.push([{
icoName,
icoUrl,
icoImgUrl
}]);
}
}
// Results are ready
console.log(results);
fs.writeFileSync(path, await page.content());
await page.tracing.stop()
await browser.close()
})()
+190
View File
@@ -0,0 +1,190 @@
/**
* Copyright 2018 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* @author ebidel@ (Eric Bidelman)
*/
/**
* Discovers all the pages in site or single page app (SPA) and creates
* a tree of the result in ./output/<site slug/crawl.json. Optionally
* takes screenshots of each page as it is visited.
*
* Usage:
* node crawlsite.js
* URL=https://yourspa.com node crawlsite.js
* URL=https://yourspa.com node crawlsite.js --screenshots
*
* Then open the visualizer in a browser:
* http://localhost:8080/html/d3tree.html
* http://localhost:8080/html/d3tree.html?url=../output/https___yourspa.com/crawl.json
*
*Start Server:
* node server.js
*
*/
/**
* Altered for Superalgos documentation export
*
* @author NAmorim@ (Nuno Amorim)
*
*/
const fs = require('fs');
const del = require('del');
const util = require('util');
const puppeteer = require('puppeteer');
const sharp = require('sharp');
const URL = process.env.URL || 'http://localhost:34248/';
const SCREENSHOTS = process.argv.includes('--screenshots');
const DEPTH = parseInt(process.env.DEPTH) || 2;
const VIEWPORT = SCREENSHOTS ? {width: 1028, height: 800, deviceScaleFactor: 2} : null;
const OUT_DIR = process.env.OUTDIR || `doc_output/${slugify(URL)}`;
const crawledPages = new Map();
const maxDepth = DEPTH; // Subpage depth to crawl site.
function slugify(str) {
return str.replace(/[\/:]/g, '_');
}
function mkdirSync(dirPath) {
try {
dirPath.split('/').reduce((parentPath, dirName) => {
const currentPath = parentPath + dirName;
if (!fs.existsSync(currentPath)) {
fs.mkdirSync(currentPath);
}
return currentPath + '/';
}, '');
} catch (err) {
if (err.code !== 'EEXIST') {
throw err;
}
}
}
/**
* Finds all anchors on the page, inclusive of those within shadow roots.
* Note: Intended to be run in the context of the page.
* @param {boolean=} sameOrigin When true, only considers links from the same origin as the app.
* @return {!Array<string>} List of anchor hrefs.
*/
function collectAllSameOriginAnchorsDeep(sameOrigin = true) {
const allElements = [];
const findAllElements = function(nodes) {
for (let i = 0, el; el = nodes[i]; ++i) {
allElements.push(el);
// If the element has a shadow root, dig deeper.
if (el.shadowRoot) {
findAllElements(el.shadowRoot.querySelectorAll('*'));
}
}
};
findAllElements(document.querySelectorAll('doc*'));
const filtered = allElements
.filter(el => el.localName === 'a' && el.href) // element is an anchor with an href.
.filter(el => el.href !== location.href) // link doesn't point to page's own URL.
.filter(el => {
if (sameOrigin) {
return new URL(location).origin === new URL(el.href).origin;
}
return true;
})
.map(a => a.href);
return Array.from(new Set(filtered));
}
/**
* Crawls a URL by visiting an url, then recursively visiting any child subpages.
* @param {!Browser} browser
* @param {{url: string, title: string, img?: string, children: !Array<!Object>}} page Current page.
* @param {number=} depth Current subtree depth of crawl.
*/
async function crawl(browser, page, depth = 0) {
if (depth > maxDepth) {
return;
}
// If we've already crawled the URL, we know its children.
if (crawledPages.has(page.url)) {
console.log(`Reusing route: ${page.url}`);
const item = crawledPages.get(page.url);
page.title = item.title;
page.img = item.img;
page.children = item.children;
// Fill in the children with details (if they already exist).
page.children.forEach(c => {
const item = crawledPages.get(c.url);
c.title = item ? item.title : '';
c.img = item ? item.img : null;
});
return;
} else {
console.log(`Loading: ${page.url}`);
const newPage = await browser.newPage();
await newPage.goto(page.url, {waitUntil: 'networkidle2'});
let anchors = await newPage.evaluate(collectAllSameOriginAnchorsDeep);
anchors = anchors.filter(a => a !== URL) // link doesn't point to start url of crawl.
page.title = await newPage.evaluate('document.title');
page.children = anchors.map(url => ({url}));
if (SCREENSHOTS) {
const path = `./${OUT_DIR}/${slugify(page.url)}.png`;
let imgBuff = await newPage.screenshot({fullPage: false});
imgBuff = await sharp(imgBuff).resize(null, 150).toBuffer(); // resize image to 150 x auto.
util.promisify(fs.writeFile)(path, imgBuff); // async
page.img = `data:img/png;base64,${imgBuff.toString('base64')}`;
}
crawledPages.set(page.url, page); // cache it.
await newPage.close();
}
// Crawl subpages.
for (const childPage of page.children) {
await crawl(browser, childPage, depth + 1);
}
}
(async() => {
mkdirSync(OUT_DIR); // create output dir if it doesn't exist.
await del([`${OUT_DIR}/*`]); // cleanup after last run.
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
if (VIEWPORT) {
await page.setViewport(VIEWPORT);
}
const root = {url: URL};
await crawl(browser, root);
await util.promisify(fs.writeFile)(`./${OUT_DIR}/crawl.json`, JSON.stringify(root, null, ' '));
await browser.close();
})();
+276 -7938
View File
File diff suppressed because it is too large. Load diff
+4 -2
View File
@@ -12,7 +12,8 @@
"start": "node platform minMemo noBrowser",
"setup": "node setup noShortcuts",
"release": "electron-builder -w",
"postinstall": "electron-builder install-app-deps"
"postinstall": "electron-builder install-app-deps",
"genDoc": "node gen_docs"
},
"dependencies": {
"@octokit/rest": "^18.11.4",
@@ -38,7 +39,8 @@
},
"devDependencies": {
"electron": "^15.1.0",
"electron-builder": "^22.11.7"
"electron-builder": "^22.11.7",
"puppeteer": "^10.4.0"
},
"build": {
"appId": "org.superalgos.platform",