diff --git a/.gitignore b/.gitignore index 96281caaf..e53c5f489 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +/build/* +/src/drivers/**/categories.json +/src/drivers/**/technologies/* +/src/drivers/**/wappalyzer.js + /src/images/icons/converted/* node_modules Thumbs.db @@ -5,3 +10,4 @@ Desktop.ini *.DS_Store *.log .idea + diff --git a/README.md b/README.md index 04a317e2c..ebab1fac4 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,12 @@ npm install ## Usage +### Command line + +```sh +node cli.js https://example.com +``` + ### Chrome extension - Go to `chrome:extensions` @@ -507,4 +513,4 @@ Application version information can be obtained from a pattern using a capture g - + \ No newline at end of file diff --git a/cli.js b/cli.js new file mode 100644 index 000000000..49d87dd8e --- /dev/null +++ b/cli.js @@ -0,0 +1,189 @@ +#!/usr/bin/env node + +const Wappalyzer = require('./src/drivers/driver'); + +const args = process.argv.slice(2); + +const options = {}; + +let url; +let arg; + +const aliases = { + a: 'userAgent', + b: 'batchSize', + d: 'debug', + f: 'fast', + t: 'delay', + h: 'help', + H: 'header', + D: 'maxDepth', + m: 'maxUrls', + p: 'probe', + P: 'pretty', + r: 'recursive', + w: 'maxWait', + n: 'noScripts', + N: 'noRedirect', + e: 'extended' +}; + +while (true) { + arg = args.shift(); + + if (!arg) { + break; + } + + const matches = /^-?-([^=]+)(?:=(.+)?)?/.exec(arg); + + if (matches) { + const key = + aliases[matches[1]] || + matches[1].replace(/-\w/g, (_matches) => _matches[1].toUpperCase()); + + const value = matches[2] + ? matches[2] + : args[0] && !args[0].startsWith('-') + ? args.shift() + : true; + + if (options[key]) { + if (!Array.isArray(options[key])) { + options[key] = [options[key]]; + } + + options[key].push(value); + } else { + options[key] = value; + } + } else { + url = arg; + } +} + +if (!url || options.help) { + process.stdout.write(`Usage: + wappalyzer [options] + +Examples: + wappalyzer https://www.example.com + node cli.js https://www.example.com -r -D 3 -m 50 -H "Cookie: username=admin" + docker wappalyzer/cli https://www.example.com --pretty + +Options: + -b, --batch-size=... Process links in batches + -d, --debug Output debug messages + -f, --fast Prioritise speed over accuracy + -t, --delay=ms Wait for ms milliseconds between requests + -h, --help This text + -H, --header Extra header to send with requests + --html-max-cols=... Limit the number of HTML characters per line processed + --html-max-rows=... Limit the number of HTML lines processed + -D, --max-depth=... Don't analyse pages more than num levels deep + -m, --max-urls=... Exit when num URLs have been analysed + -w, --max-wait=... Wait no more than ms milliseconds for page resources to load + -p, --probe=[basic|full] Perform a deeper scan by performing additional requests and inspecting DNS records + -P, --pretty Pretty-print JSON output + --proxy=... Proxy URL, e.g. 'http://user:pass@proxy:8080' + -r, --recursive Follow links on pages (crawler) + -a, --user-agent=... Set the user agent string + -n, --no-scripts Disabled JavaScript on web pages + -N, --no-redirect Disable cross-domain redirects + -e, --extended Output additional information + --local-storage=... JSON object to use as local storage + --session-storage=... JSON object to use as session storage + --defer=ms Defer scan for ms milliseconds after page load +`); + process.exit(options.help ? 0 : 1); +} + +try { + const { hostname } = new URL(url); + + if (!hostname) { + throw new Error('Invalid URL'); + } +} catch (error) { + console.log(error.message || error.toString()); + + process.exit(1); +} + +const headers = {}; + +if (options.header) { + (Array.isArray(options.header) ? options.header : [options.header]).forEach( + (header) => { + const [key, value] = header.split(':'); + + headers[key.trim()] = (value || '').trim(); + } + ); +} + +const storage = { + local: {}, + session: {} +}; + +for (const type of Object.keys(storage)) { + if (options[`${type}Storage`]) { + try { + storage[type] = JSON.parse(options[`${type}Storage`]); + + if ( + !options[`${type}Storage`] || + !Object.keys(options[`${type}Storage`]).length + ) { + throw new Error('Object has no properties'); + } + } catch (error) { + console.log(`${type}Storage error: ${error.message || error}`); + + process.exit(1); + } + } +} + +(async function () { + const wappalyzer = new Wappalyzer(options); + + try { + await wappalyzer.init(); + + const site = await wappalyzer.open(url, headers, storage); + + await new Promise((resolve) => + setTimeout(resolve, parseInt(options.defer || 0, 10)) + ); + + const results = await site.analyze(); + + process.stdout.write( + `${JSON.stringify(results, null, options.pretty ? 2 : null)}\n` + ); + + await wappalyzer.destroy(); + + process.exit(0); + } catch (error) { + try { + await Promise.race([ + wappalyzer.destroy(), + new Promise((resolve, reject) => + setTimeout( + () => reject(new Error('Attempt to close the browser timed out')), + 3000 + ) + ) + ]); + } catch (error) { + console.error(error.message || String(error)); + } + + console.error(error.message || String(error)); + + process.exit(1); + } +})(); diff --git a/package-lock.json b/package-lock.json index 7dc9bcea5..f46d1896e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,12 @@ "requires": true, "packages": { "": { + "dependencies": { + "puppeteer": "~19.7.0" + }, + "bin": { + "wappalyzer": "cli.js" + }, "devDependencies": { "@google-cloud/bigquery": "8.3.1", "@google-cloud/storage": "7.21.0", @@ -24,7 +30,6 @@ "version": "7.29.0", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", - "dev": true, "license": "MIT", "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", @@ -51,6 +56,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -176,7 +182,6 @@ "version": "7.28.5", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -1755,7 +1760,7 @@ "version": "25.2.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.2.2.tgz", "integrity": "sha512-BkmoP5/FhRYek5izySdkOneRyXYN35I860MFAGupTdebyE66uZaR+bXLHq8k4DirE5DwQi3NuhvRU1jqTVwUrQ==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "undici-types": "~7.16.0" @@ -1805,6 +1810,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.1.tgz", @@ -2100,6 +2115,7 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2133,6 +2149,7 @@ "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -2276,7 +2293,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, "license": "Python-2.0" }, "node_modules/arrify": { @@ -2419,7 +2435,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -2486,6 +2501,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/brace-expansion": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", @@ -2536,6 +2562,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", @@ -2560,6 +2587,39 @@ "node-int64": "^0.4.0" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -2592,7 +2652,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -2694,6 +2753,24 @@ "node": ">= 6" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/chromium-bidi": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.4.5.tgz", + "integrity": "sha512-rkav9YzRfAshSTG3wNXF7P7yNiI29QAo1xBXElPoCoSQR5n20q3cOyVhDv6S7+GlF/CJ/emUxlQiR0xOPurkGg==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "3.0.0" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/ci-info": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.4.0.tgz", @@ -2824,6 +2901,35 @@ } } }, + "node_modules/cross-fetch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "license": "MIT", + "dependencies": { + "node-fetch": "2.6.7" + } + }, + "node_modules/cross-fetch/node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -2880,7 +2986,6 @@ "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -2959,6 +3064,13 @@ "node": ">=8" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1094867", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1094867.tgz", + "integrity": "sha512-pmMDBKiRVjh0uKK6CT1WqZmM3hBVSgD+N2MrgyV1uNizAZMw4tx6i/RTc+/uCsKSCmg0xXx7arCP/OFcIwTsiQ==", + "license": "BSD-3-Clause", + "peer": true + }, "node_modules/diff": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.3.tgz", @@ -3055,7 +3167,6 @@ "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "dev": true, "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -3088,7 +3199,6 @@ "version": "1.3.4", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", - "dev": true, "license": "MIT", "dependencies": { "is-arrayish": "^0.2.1" @@ -3172,6 +3282,7 @@ "integrity": "sha512-GVTD7s1vdIl6UYvAfriOPeY1Df8LIZjfofLvHwde+erDHGGuHyuM6xoxRxmHiebhYuD2p1vN4wWh0XzPARSGDQ==", "dev": true, "license": "MIT", + "peer": true, "workspaces": [ "packages/*" ], @@ -3508,6 +3619,41 @@ "dev": true, "license": "MIT" }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/extract-zip/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -3602,6 +3748,15 @@ "bser": "2.1.1" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/fetch-blob": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", @@ -3731,6 +3886,12 @@ "node": ">=12.20.0" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -3933,7 +4094,6 @@ "version": "13.0.1", "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.1.tgz", "integrity": "sha512-B7U/vJpE3DkJ5WXTgTpTRN63uV42DseiXXKMwG14LQBXmsdeIoHAPbU/MEo6II0k5ED74uc2ZGTC6MwHFQhF6w==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "minimatch": "^10.1.2", @@ -4250,6 +4410,26 @@ "node": ">=10.17.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -4264,7 +4444,6 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", - "dev": true, "license": "MIT", "dependencies": { "parent-module": "^1.0.0", @@ -4311,14 +4490,12 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, "license": "ISC" }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", - "dev": true, "license": "MIT" }, "node_modules/is-binary-path": { @@ -5150,14 +5327,12 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", - "dev": true, "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -5200,7 +5375,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true, "license": "MIT" }, "node_modules/json-schema-traverse": { @@ -5236,6 +5410,7 @@ "integrity": "sha512-75EA7EWZExL/j+MDKQrRbdzcRI2HOkRlmUw8fZJc1ioqFEOvBsq7Rt+A6yCxOt9w/TYNpkt52gC6nm/g5tFIng==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "acorn": "^8.5.0", "eslint-visitor-keys": "^5.0.0", @@ -5322,7 +5497,6 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, "license": "MIT" }, "node_modules/locate-path": { @@ -5474,7 +5648,6 @@ "version": "10.2.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "brace-expansion": "^5.0.5" @@ -5490,7 +5663,6 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, "license": "MIT", "engines": { "node": "18 || 20 || >=22" @@ -5500,7 +5672,6 @@ "version": "5.0.5", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -5513,12 +5684,23 @@ "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, "license": "ISC", "engines": { "node": ">=16 || 14 >=14.17" } }, + "node_modules/mitt": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.0.tgz", + "integrity": "sha512-7dX2/10ITVyqh4aOSVI9gdape+t9l2/8QxHrFmUXu4EEUpdlxl6RudZUPZoc+zuY2hk1j7XxVroIVIan/pD/SQ==", + "license": "MIT" + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, "node_modules/mocha": { "version": "10.8.2", "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.8.2.tgz", @@ -5636,7 +5818,6 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/napi-postinstall": { @@ -5768,7 +5949,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -5854,14 +6034,12 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "dev": true, "license": "BlueOak-1.0.0" }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, "license": "MIT", "dependencies": { "callsites": "^3.0.0" @@ -5874,7 +6052,6 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", - "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.0.0", @@ -5929,7 +6106,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "lru-cache": "^11.0.0", @@ -5946,17 +6122,30 @@ "version": "11.2.5", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.5.tgz", "integrity": "sha512-vFrFJkWtJvJnD5hg+hJvVE8Lh/TcMzKnTgCWmtBipwI5yLX/iX+5UB2tfuyODF5E7k9xEzMdYgGqaSb1c0c5Yw==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": "20 || >=22" } }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, "license": "ISC" }, "node_modules/picomatch": { @@ -6067,6 +6256,7 @@ "integrity": "sha512-/FVl766LpUfB5vXgCYOYa0MeV/441Ia99AeICQIQFTY/Nw0roZwULcXpku5i1/m5kt/baz+s4Zogspd839HSMg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -6119,6 +6309,31 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -6129,6 +6344,142 @@ "node": ">=6" } }, + "node_modules/puppeteer": { + "version": "19.7.5", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-19.7.5.tgz", + "integrity": "sha512-UqD8K+yaZa6/hwzP54AATCiHrEYGGxzQcse9cZzrtsVGd8wT0llCdYhsBp8n+zvnb1ofY0YFgI3TYZ/MiX5uXQ==", + "deprecated": "< 24.15.0 is no longer supported", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "cosmiconfig": "8.1.0", + "https-proxy-agent": "5.0.1", + "progress": "2.0.3", + "proxy-from-env": "1.1.0", + "puppeteer-core": "19.7.5" + } + }, + "node_modules/puppeteer-core": { + "version": "19.7.5", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-19.7.5.tgz", + "integrity": "sha512-EJuNha+SxPfaYFbkoWU80H3Wb1SiQH5fFyb2xdbWda0ziax5mhV63UMlqNfPeTDIWarwtR4OIcq/9VqY8HPOsg==", + "license": "Apache-2.0", + "dependencies": { + "chromium-bidi": "0.4.5", + "cross-fetch": "3.1.5", + "debug": "4.3.4", + "devtools-protocol": "0.0.1094867", + "extract-zip": "2.0.1", + "https-proxy-agent": "5.0.1", + "proxy-from-env": "1.1.0", + "rimraf": "4.4.0", + "tar-fs": "2.1.1", + "unbzip2-stream": "1.4.3", + "ws": "8.12.1" + }, + "engines": { + "node": ">=14.14.0" + }, + "peerDependencies": { + "typescript": ">= 4.7.4" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/puppeteer-core/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/puppeteer-core/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "license": "MIT", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/puppeteer-core/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/puppeteer-core/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "license": "MIT" + }, + "node_modules/puppeteer/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/puppeteer/node_modules/cosmiconfig": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.1.0.tgz", + "integrity": "sha512-0tLZ9URlPGU7JsKq0DQOQ3FoRsYX8xDZ7xMiATQfaiGMz7EHowNkbU9u1coAOmnh9p/1ySpm0RB3JNWRXM5GCg==", + "license": "MIT", + "dependencies": { + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "parse-json": "^5.0.0", + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + } + }, + "node_modules/puppeteer/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/pure-rand": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-7.0.1.tgz", @@ -6197,7 +6548,6 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -6268,7 +6618,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -6312,7 +6661,6 @@ "version": "6.1.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-6.1.2.tgz", "integrity": "sha512-cFCkPslJv7BAXJsYlK1dZsbP8/ZNLkCAQ0bi1hf5EKX2QHegmDFEFA6QhuYJlk7UDdc+02JjO80YSOrWPpw06g==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "glob": "^13.0.0", @@ -6332,7 +6680,6 @@ "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, "funding": [ { "type": "github", @@ -6521,7 +6868,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" @@ -6651,6 +6997,34 @@ "url": "https://opencollective.com/synckit" } }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/teeny-request": { "version": "10.1.2", "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.2.tgz", @@ -6711,6 +7085,12 @@ "node": ">=18" } }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" + }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -6735,7 +7115,6 @@ "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "dev": true, "license": "MIT" }, "node_modules/tslib": { @@ -6782,11 +7161,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "license": "MIT", + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/unrs-resolver": { @@ -6869,7 +7258,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, "license": "MIT" }, "node_modules/v8-to-istanbul": { @@ -6911,7 +7299,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "dev": true, "license": "BSD-2-Clause" }, "node_modules/webpagetest": { @@ -6947,7 +7334,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dev": true, "license": "MIT", "dependencies": { "tr46": "~0.0.3", @@ -7009,7 +7395,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, "license": "ISC" }, "node_modules/write-file-atomic": { @@ -7039,6 +7424,27 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/ws": { + "version": "8.12.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.1.tgz", + "integrity": "sha512-1qo+M9Ba+xNhPB+YTWUlK6M17brTut5EXbcBaMRN5pH5dFrXz7lzz1ChFSUq3bOUl8yEvSenhHmYUNJxFzdJew==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/xml-naming": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", @@ -7164,6 +7570,16 @@ "node": ">=12" } }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index e6da12fc5..4da3c9a30 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,11 @@ { + "main": "src/drivers/driver.js", + "bin": { + "wappalyzer": "./cli.js" + }, + "dependencies": { + "puppeteer": "~19.7.0" + }, "devDependencies": { "@google-cloud/bigquery": "8.3.1", "@google-cloud/storage": "7.21.0", diff --git a/src/drivers/README.md b/src/drivers/README.md new file mode 100644 index 000000000..4940abdda --- /dev/null +++ b/src/drivers/README.md @@ -0,0 +1,154 @@ +# Wappalyzer + +[Wappalyzer](https://www.wappalyzer.com/) indentifies technologies on websites. + +*Note:* The [wappalyzer-core](https://www.npmjs.com/package/wappalyzer-core) package provides a low-level API without dependencies. + +## Command line + +To run the CLI command `wappalyzer` globally, link it from the repository root: + +```shell +$ npm link +``` + +### Usage + +``` +wappalyzer [options] +``` + +#### Options + +``` +-b, --batch-size=... Process links in batches +-d, --debug Output debug messages +-t, --delay=ms Wait for ms milliseconds between requests +-h, --help This text +-H, --header Extra header to send with requests +--html-max-cols=... Limit the number of HTML characters per line processed +--html-max-rows=... Limit the number of HTML lines processed +-D, --max-depth=... Don't analyse pages more than num levels deep +-m, --max-urls=... Exit when num URLs have been analysed +-w, --max-wait=... Wait no more than ms milliseconds for page resources to load +-p, --probe=[basic|full] Perform a deeper scan by performing additional requests and inspecting DNS records +-P, --pretty Pretty-print JSON output +--proxy=... Proxy URL, e.g. 'http://user:pass@proxy:8080' +-r, --recursive Follow links on pages (crawler) +-a, --user-agent=... Set the user agent string +-n, --no-scripts Disabled JavaScript on web pages +-N, --no-redirect Disable cross-domain redirects +-e, --extended Output additional information +--local-storage=... JSON object to use as local storage +--session-storage=... JSON object to use as session storage +--defer=ms Defer scan for ms milliseconds after page load + +``` + +## Dependency + +To install this package in another local project, reference it by its local path: + +```shell +$ npm install /path/to/wappalyzer +``` + +### Usage + +```javascript +const Wappalyzer = require('./driver') + +const url = 'https://www.wappalyzer.com' + +const options = { + debug: false, + delay: 500, + headers: {}, + maxDepth: 3, + maxUrls: 10, + maxWait: 5000, + recursive: true, + probe: true, + proxy: false, + userAgent: 'Wappalyzer', + htmlMaxCols: 2000, + htmlMaxRows: 2000, + noScripts: false, + noRedirect: false, +}; + +const wappalyzer = new Wappalyzer(options) + +;(async function() { + try { + await wappalyzer.init() + + // Optionally set additional request headers + const headers = {} + + // Optionally set local and/or session storage + const storage = { + local: {} + session: {} + } + + const site = await wappalyzer.open(url, headers, storage) + + // Optionally capture and output errors + site.on('error', console.error) + + const results = await site.analyze() + + console.log(JSON.stringify(results, null, 2)) + } catch (error) { + console.error(error) + } + + await wappalyzer.destroy() +})() +``` + +Multiple URLs can be processed in parallel: + +```javascript +const Wappalyzer = require('./driver'); + +const urls = ['https://www.wappalyzer.com', 'https://www.example.com'] + +const wappalyzer = new Wappalyzer() + +;(async function() { + try { + await wappalyzer.init() + + const results = await Promise.all( + urls.map(async (url) => { + const site = await wappalyzer.open(url) + + const results = await site.analyze() + + return { url, results } + }) + ) + + console.log(JSON.stringify(results, null, 2)) + } catch (error) { + console.error(error) + } + + await wappalyzer.destroy() +})() +``` + +### Events + +Listen to events with `site.on(eventName, callback)`. Use the `page` parameter to access the Puppeteer page instance ([reference](https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#class-page)). + +| Event | Parameters | Description | +|-------------|--------------------------------|------------------------------------------| +| `log` | `message`, `source` | Debug messages | +| `error` | `message`, `source` | Error messages | +| `request` | `page`, `request` | Emitted at the start of a request | +| `response` | `page`, `request` | Emitted upon receiving a server response | +| `goto` | `page`, `url`, `html`, `cookies`, `scriptsSrc`, `scripts`, `meta`, `js`, `language` `links` | Emitted after a page has been analysed | +| `analyze` | `urls`, `technologies`, `meta` | Emitted when the site has been analysed | diff --git a/src/drivers/driver.js b/src/drivers/driver.js new file mode 100644 index 000000000..7346d78ae --- /dev/null +++ b/src/drivers/driver.js @@ -0,0 +1,1104 @@ +const fs = require('fs'); +const dns = require('dns').promises; +const path = require('path'); +const http = require('http'); +const https = require('https'); +const puppeteer = require('puppeteer'); +const Wappalyzer = require('../js/wappalyzer'); + +const { setTechnologies, setCategories, analyze, analyzeManyToMany, resolve } = + Wappalyzer; + +const { CHROMIUM_BIN, CHROMIUM_DATA_DIR, CHROMIUM_WEBSOCKET, CHROMIUM_ARGS } = + process.env; + +const chromiumArgs = CHROMIUM_ARGS + ? CHROMIUM_ARGS.split(' ') + : [ + '--headless', + '--single-process', + '--no-sandbox', + '--no-zygote', + '--disable-gpu', + '--ignore-certificate-errors', + '--allow-running-insecure-content', + '--disable-web-security', + `--user-data-dir=${CHROMIUM_DATA_DIR || '/tmp/chromium'}` + ]; + +const extensions = /^([^.]+$|\.(asp|aspx|cgi|htm|html|jsp|php)$)/; + +const categoriesPath = path.resolve(`${__dirname}/../categories.json`); + +const categories = JSON.parse(fs.readFileSync(categoriesPath)); + +let technologies = {}; + +const technologiesDir = path.resolve(`${__dirname}/../technologies`); + +for (const index of Array(27).keys()) { + const character = index ? String.fromCharCode(index + 96) : '_'; + + technologies = { + ...technologies, + ...JSON.parse( + fs.readFileSync(path.resolve(`${technologiesDir}/${character}.json`)) + ) + }; +} + +setTechnologies(technologies); +setCategories(categories); + +const xhrDebounce = []; + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function getJs(page, technologies = Wappalyzer.technologies) { + return page.evaluate((technologies) => { + return technologies + .filter(({ js }) => Object.keys(js).length) + .map(({ name, js }) => ({ name, chains: Object.keys(js) })) + .reduce((technologies, { name, chains }) => { + chains.forEach((chain) => { + chain = chain.replace(/\[([^\]]+)\]/g, '.$1'); + + const parts = chain.split('.'); + + const root = /^[a-z_$][a-z0-9_$]*$/i.test(parts[0]) + ? new Function( + `return typeof ${ + parts[0] + } === 'undefined' ? undefined : ${parts.shift()}` + )() + : window; + + const value = parts.reduce( + (value, method) => + value && + value instanceof Object && + Object.prototype.hasOwnProperty.call(value, method) + ? value[method] + : '__UNDEFINED__', + root || '__UNDEFINED__' + ); + + if (value !== '__UNDEFINED__') { + technologies.push({ + name, + chain, + value: + typeof value === 'string' || typeof value === 'number' + ? value + : !!value + }); + } + }); + + return technologies; + }, []); + }, technologies); +} + +function analyzeJs(js, technologies = Wappalyzer.technologies) { + return js + .map(({ name, chain, value }) => { + return analyzeManyToMany( + technologies.find(({ name: _name }) => name === _name), + 'js', + { [chain]: [value] } + ); + }) + .flat(); +} + +function getDom(page, technologies = Wappalyzer.technologies) { + return page.evaluate((technologies) => { + return technologies + .filter(({ dom }) => dom && dom.constructor === Object) + .reduce((technologies, { name, dom }) => { + const toScalar = (value) => + typeof value === 'string' || typeof value === 'number' + ? value + : !!value; + + Object.keys(dom).forEach((selector) => { + let nodes = []; + + try { + nodes = document.querySelectorAll(selector); + } catch (error) { + // Continue + } + + if (!nodes.length) { + return; + } + + dom[selector].forEach(({ exists, text, properties, attributes }) => { + nodes.forEach((node) => { + if ( + technologies.filter(({ name: _name }) => _name === name) + .length >= 50 + ) { + return; + } + + if ( + exists && + technologies.findIndex( + ({ name: _name, selector: _selector, exists }) => + name === _name && selector === _selector && exists === '' + ) === -1 + ) { + technologies.push({ + name, + selector, + exists: '' + }); + } + + if (text) { + const value = ( + node.textContent ? node.textContent.trim() : '' + ).slice(0, 1000000); + + if ( + value && + technologies.findIndex( + ({ name: _name, selector: _selector, text }) => + name === _name && selector === _selector && text === value + ) === -1 + ) { + technologies.push({ + name, + selector, + text: value + }); + } + } + + if (properties) { + Object.keys(properties).forEach((property) => { + if ( + Object.prototype.hasOwnProperty.call(node, property) && + technologies.findIndex( + ({ + name: _name, + selector: _selector, + property: _property, + value + }) => + name === _name && + selector === _selector && + property === _property && + value === toScalar(value) + ) === -1 + ) { + const value = node[property]; + + if (typeof value !== 'undefined') { + technologies.push({ + name, + selector, + property, + value: toScalar(value) + }); + } + } + }); + } + + if (attributes) { + Object.keys(attributes).forEach((attribute) => { + if ( + node.hasAttribute(attribute) && + technologies.findIndex( + ({ + name: _name, + selector: _selector, + attribute: _atrribute, + value + }) => + name === _name && + selector === _selector && + attribute === _atrribute && + value === toScalar(value) + ) === -1 + ) { + const value = node.getAttribute(attribute); + + technologies.push({ + name, + selector, + attribute, + value: toScalar(value) + }); + } + }); + } + }); + }); + }); + + return technologies; + }, []); + }, technologies); +} + +function analyzeDom(dom, technologies = Wappalyzer.technologies) { + return dom + .map(({ name, selector, exists, text, property, attribute, value }) => { + const technology = technologies.find(({ name: _name }) => name === _name); + + if (typeof exists !== 'undefined') { + return analyzeManyToMany(technology, 'dom.exists', { + [selector]: [''] + }); + } + + if (typeof text !== 'undefined') { + return analyzeManyToMany(technology, 'dom.text', { + [selector]: [text] + }); + } + + if (typeof property !== 'undefined') { + return analyzeManyToMany(technology, `dom.properties.${property}`, { + [selector]: [value] + }); + } + + if (typeof attribute !== 'undefined') { + return analyzeManyToMany(technology, `dom.attributes.${attribute}`, { + [selector]: [value] + }); + } + }) + .flat(); +} + +function get(url, options = {}) { + const timeout = + options.timeout || + (this.options.fast + ? this.Math.min(this.options.maxWait, 3000) + : this.options.maxWait); + + if (['http:', 'https:'].includes(url.protocol)) { + const { get } = url.protocol === 'http:' ? http : https; + + return new Promise((resolve, reject) => + get( + url, + { + rejectUnauthorized: false, + headers: { + 'User-Agent': options.userAgent + } + }, + (response) => { + if (response.statusCode >= 300) { + return reject( + new Error(`${response.statusCode} ${response.statusMessage}`) + ); + } + + response.setEncoding('utf8'); + + let body = ''; + + response.on('data', (data) => (body += data)); + response.on('error', (error) => reject(new Error(error.message))); + response.on('end', () => resolve(body)); + } + ) + .setTimeout(timeout, () => + reject(new Error(`Timeout (${url}, ${timeout}ms)`)) + ) + .on('error', (error) => reject(new Error(error.message))) + ); + } else { + throw new Error(`Invalid protocol: ${url.protocol}`); + } +} + +class Driver { + constructor(options = {}) { + this.options = { + batchSize: 5, + debug: false, + delay: 500, + htmlMaxCols: 2000, + htmlMaxRows: 3000, + maxDepth: 3, + maxUrls: 10, + maxWait: 30000, + recursive: false, + probe: false, + proxy: false, + noScripts: false, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36', + extended: false, + ...options + }; + + this.options.debug = Boolean(+this.options.debug); + this.options.fast = Boolean(+this.options.fast); + this.options.recursive = Boolean(+this.options.recursive); + this.options.probe = + String(this.options.probe || '').toLowerCase() === 'basic' + ? 'basic' + : String(this.options.probe || '').toLowerCase() === 'full' + ? 'full' + : Boolean(+this.options.probe) && 'full'; + this.options.delay = parseInt(this.options.delay, 10); + this.options.maxDepth = parseInt(this.options.maxDepth, 10); + this.options.maxUrls = parseInt(this.options.maxUrls, 10); + this.options.maxWait = parseInt(this.options.maxWait, 10); + this.options.htmlMaxCols = parseInt(this.options.htmlMaxCols, 10); + this.options.htmlMaxRows = parseInt(this.options.htmlMaxRows, 10); + this.options.noScripts = Boolean(+this.options.noScripts); + this.options.extended = Boolean(+this.options.extended); + + if (this.options.proxy) { + chromiumArgs.push(`--proxy-server=${this.options.proxy}`); + } + + this.destroyed = false; + } + + async init() { + for (let attempt = 1; attempt <= 2; attempt++) { + this.log(`Launching browser (attempt ${attempt})...`); + + try { + if (CHROMIUM_WEBSOCKET) { + this.browser = await puppeteer.connect({ + ignoreHTTPSErrors: true, + acceptInsecureCerts: true, + browserWSEndpoint: CHROMIUM_WEBSOCKET + }); + } else { + this.browser = await puppeteer.launch({ + ignoreHTTPSErrors: true, + acceptInsecureCerts: true, + args: chromiumArgs, + executablePath: CHROMIUM_BIN, + timeout: this.options.fast + ? Math.min(this.options.maxWait, 10000) + : this.options.maxWait + }); + } + + break; + } catch (error) { + this.log(error); + + if (attempt >= 2) { + throw new Error(error.message || error.toString()); + } + } + } + + this.browser.on('disconnected', () => { + this.browser = undefined; + + this.log('Browser disconnected'); + }); + } + + async destroy() { + if (this.browser) { + try { + await sleep(1); + + await this.browser.close(); + + this.log('Browser closed'); + } catch (error) { + throw new Error(error.toString()); + } + } + } + + async open(url, headers = {}, storage = {}) { + const site = new Site(url.split('#')[0], headers, this); + + if (storage.local || storage.session) { + this.log('Setting storage...'); + + const page = await site.newPage(site.originalUrl); + + await page.setRequestInterception(true); + + page.on('request', (request) => + request.respond({ + status: 200, + contentType: 'text/plain', + body: 'ok' + }) + ); + + await page.goto(url); + + await page.evaluate((storage) => { + ['local', 'session'].forEach((type) => { + Object.keys(storage[type] || {}).forEach((key) => { + window[`${type}Storage`].setItem(key, storage[type][key]); + }); + }); + }, storage); + + try { + await page.close(); + } catch { + // Continue + } + } + + return site; + } + + log(message, source = 'driver') { + if (this.options.debug) { + console.log(`log | ${source} |`, message); + } + } +} + +class Site { + constructor(url, headers = {}, driver) { + ({ + options: this.options, + browser: this.browser, + init: this.initDriver + } = driver); + + this.options.headers = { + ...this.options.headers, + ...headers + }; + + this.driver = driver; + + try { + this.originalUrl = new URL(url); + } catch (error) { + throw new Error(error.toString()); + } + + this.analyzedUrls = {}; + this.analyzedXhr = {}; + this.analyzedRequires = {}; + this.detections = []; + + this.listeners = {}; + + this.pages = []; + + this.cache = {}; + + this.probed = false; + } + + log(message, source = 'driver', type = 'log') { + if (this.options.debug) { + console[type](`${type} | ${source} |`, message); + } + + this.emit(type, { message, source }); + } + + error(error, source = 'driver') { + this.log(error, source, 'error'); + } + + on(event, callback) { + if (!this.listeners[event]) { + this.listeners[event] = []; + } + + this.listeners[event].push(callback); + } + + emit(event, params) { + if (this.listeners[event]) { + return Promise.allSettled( + this.listeners[event].map((listener) => listener(params)) + ); + } + } + + promiseTimeout( + promise, + fallback, + errorMessage = 'Operation took too long to complete', + maxWait = this.options.fast + ? Math.min(this.options.maxWait, 2000) + : this.options.maxWait + ) { + let timeout = null; + + if (!(promise instanceof Promise)) { + return Promise.resolve(promise); + } + + return Promise.race([ + new Promise((resolve, reject) => { + timeout = setTimeout(() => { + clearTimeout(timeout); + + const error = new Error(errorMessage); + + error.code = 'PROMISE_TIMEOUT_ERROR'; + + if (fallback !== undefined) { + this.error(error); + + resolve(fallback); + } else { + reject(error); + } + }, maxWait); + }), + promise.then((value) => { + clearTimeout(timeout); + + return value; + }) + ]); + } + + async goto(url) { + // Return when the URL is a duplicate or maxUrls has been reached + if (this.analyzedUrls[url.href]) { + return []; + } + + this.log(`Navigate to ${url}`); + + this.analyzedUrls[url.href] = { + status: 0 + }; + + const page = await this.newPage(url); + + await page.setRequestInterception(true); + + let responseReceived = false; + + page.on('request', async (request) => { + try { + if (request.resourceType() === 'xhr') { + let hostname; + + try { + ({ hostname } = new URL(request.url())); + } catch (error) { + request.abort('blockedbyclient'); + + return; + } + + if (!xhrDebounce.includes(hostname)) { + xhrDebounce.push(hostname); + + setTimeout(async () => { + xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1); + + this.analyzedXhr[url.hostname] = + this.analyzedXhr[url.hostname] || []; + + if (!this.analyzedXhr[url.hostname].includes(hostname)) { + this.analyzedXhr[url.hostname].push(hostname); + + await this.onDetect(url, analyze({ xhr: hostname })); + } + }, 1000); + } + } + + if ( + (responseReceived && request.isNavigationRequest()) || + request.frame() !== page.mainFrame() || + !['document', ...(this.options.noScripts ? [] : ['script'])].includes( + request.resourceType() + ) + ) { + request.abort('blockedbyclient'); + } else { + await this.emit('request', { page, request }); + + if (Object.keys(this.options.headers).length) { + const headers = { + ...request.headers(), + ...this.options.headers + }; + + request.continue({ headers }); + } else { + request.continue(); + } + } + } catch (error) { + error.message += ` (${url})`; + + this.error(error); + } + }); + + page.on('response', async (response) => { + if (!page || page.__closed || page.isClosed()) { + return; + } + + try { + if ( + response.status() < 300 && + response.frame().url() === url.href && + response.request().resourceType() === 'script' + ) { + const scripts = await response.text(); + + await this.onDetect(response.url(), analyze({ scripts })); + } + } catch (error) { + if (error.constructor.name !== 'ProtocolError') { + error.message += ` (${url})`; + + this.error(error); + } + } + + try { + if (response.url() === url.href) { + this.analyzedUrls[url.href] = { + status: response.status() + }; + + const rawHeaders = response.headers(); + const headers = {}; + + Object.keys(rawHeaders).forEach((key) => { + headers[key] = [ + ...(headers[key] || []), + ...(Array.isArray(rawHeaders[key]) + ? rawHeaders[key] + : [rawHeaders[key]]) + ]; + }); + + // Prevent cross-domain redirects + if (response.status() >= 300 && response.status() < 400) { + if (headers.location) { + const _url = new URL(headers.location.slice(-1), url); + + const redirects = Object.keys(this.analyzedUrls).length - 1; + + if ( + _url.hostname.replace(/^www\./, '') === + this.originalUrl.hostname.replace(/^www\./, '') || + (redirects < 3 && !this.options.noRedirect) + ) { + url = _url; + + return; + } + } + } + + responseReceived = true; + + const certIssuer = response.securityDetails() + ? response.securityDetails().issuer() + : ''; + + await this.onDetect(url, analyze({ headers, certIssuer })); + + await this.emit('response', { page, response, headers, certIssuer }); + } + } catch (error) { + error.message += ` (${url})`; + + this.error(error); + } + }); + + try { + await page.goto(url.href); + + if (page.url() === 'about:blank') { + const error = new Error(`The page failed to load (${url})`); + + error.code = 'WAPPALYZER_PAGE_EMPTY'; + + throw error; + } + + if (!this.options.noScripts) { + await sleep(this.options.fast ? 1000 : 3000); + } + + // page.on('console', (message) => this.log(message.text())) + + // Cookies + let cookies = []; + + try { + cookies = (await page.cookies()).reduce( + (cookies, { name, value }) => ({ + ...cookies, + [name.toLowerCase()]: [value] + }), + {} + ); + + // Change Google Analytics 4 cookie from _ga_XXXXXXXXXX to _ga_* + Object.keys(cookies).forEach((name) => { + if (/_ga_[A-Z0-9]+/.test(name)) { + cookies['_ga_*'] = cookies[name]; + + delete cookies[name]; + } + }); + } catch (error) { + error.message += ` (${url})`; + + this.error(error); + } + + // HTML + let html = await this.promiseTimeout( + page.content(), + '', + 'Timeout (html)' + ); + + if (this.options.htmlMaxCols && this.options.htmlMaxRows) { + const batches = []; + const rows = html.length / this.options.htmlMaxCols; + + for (let i = 0; i < rows; i += 1) { + if (batches.length >= this.options.htmlMaxRows) { + break; + } + + batches.push( + html.slice( + i * this.options.htmlMaxCols, + (i + 1) * this.options.htmlMaxCols + ) + ); + } + + html = batches.join('\n'); + } + + // CSS + let css = []; + + try { + const styles = await page.evaluate(() => + Array.from(document.styleSheets) + .map((styleSheet) => { + try { + return Array.from(styleSheet.cssRules) + .map(({ cssText }) => cssText) + .join(' '); + } catch (error) { + // Continue + } + }) + .filter(Boolean) + ); + + css = styles.map((style) => style.slice(0, 1000000)); + } catch (error) { + error.message += ` (${url})`; + + this.error(error); + } + + // Script tags + let scriptSrc = []; + + try { + scriptSrc = await page.evaluate(() => + Array.from(document.scripts) + .map(({ src }) => src) + .filter(Boolean) + ); + } catch (error) { + error.message += ` (${url})`; + + this.error(error); + } + + // Meta tags + let meta = {}; + + try { + meta = await page.evaluate(() => + Array.from(document.querySelectorAll('meta')).reduce((meta, node) => { + const name = + node.getAttribute('name') || + node.getAttribute('property') || + node.getAttribute('itemprop'); + const content = node.getAttribute('content'); + + if (name && content) { + meta[name.toLowerCase()] = meta[name.toLowerCase()] || []; + + meta[name.toLowerCase()].push(content.slice(0, 1000000)); + } + + return meta; + }, {}) + ); + } catch (error) { + error.message += ` (${url})`; + + this.error(error); + } + + const js = await this.promiseTimeout( + getJs(page), + [], + 'Timeout (js evaluation)' + ); + + const dom = await this.promiseTimeout( + getDom(page), + [], + 'Timeout (dom evaluation)' + ); + + await this.onDetect(url, analyze({ html, css, scriptSrc, meta })); + + await this.onDetect(url, analyzeJs(js)); + + await this.onDetect(url, analyzeDom(dom)); + + await this.emit('goto', { page, url, html, cookies, js, dom }); + + // Crawler + if (this.options.recursive) { + const links = await page.evaluate(() => + Array.from(document.querySelectorAll('a[href]')).map( + ({ href }) => href + ) + ); + + await Promise.all( + links.map(async (link) => { + let _url; + + try { + _url = new URL(link); + } catch (error) { + return; + } + + // Must be on the same domain + if ( + _url.hostname.replace(/^www\./, '') === + url.hostname.replace(/^www\./, '') && + _url.protocol.startsWith('http') && + extensions.test(_url.pathname) + ) { + const redirects = Object.keys(this.analyzedUrls).length - 1; + + if (redirects < this.options.maxUrls) { + const depth = url.pathname.split('/').length - 1; + + if (depth < this.options.maxDepth) { + await this.goto(_url); + } + } + } + }) + ); + } + } catch (error) { + if (error.code !== 'WAPPALYZER_PAGE_EMPTY') { + error.message += ` (${url})`; + + this.error(error); + } + } finally { + try { + await page.close(); + } catch { + // Continue + } + } + } + + async analyze() { + await this.goto(this.originalUrl); + + if (this.options.probe) { + await this.probe(); + } + + const resolved = resolve(this.detections); + + return { + urls: this.analyzedUrls, + technologies: resolved.map((resolved) => { + const technology = Wappalyzer.technologies.find( + ({ name }) => name === resolved.name + ); + + const { name, slug, description, confidence, icon, website } = + technology; + + const categories = technology.categories.map((id) => + Wappalyzer.categories.find((category) => category.id === id) + ); + + return { + slug, + name, + description: description || null, + confidence, + version: resolved.version || null, + icon, + website, + cpe: technology.cpe || null, + categories: categories.map(({ id, slug, name }) => ({ + id, + slug, + name + })), + rootPath: resolved.rootPath || undefined + }; + }) + }; + } + + async probe() { + this.log(`Probe ${this.originalUrl}`); + + this.probed = true; + + // DNS + const dnsResolvers = { + txt: dns.resolveTxt(this.originalUrl.hostname), + mx: dns.resolveMx(this.originalUrl.hostname) + }; + + const dnsResults = await Promise.allSettled( + Object.values(dnsResolvers) + ).then((results) => + results.reduce( + (results, result, index) => ({ + ...results, + [Object.keys(dnsResolvers)[index]]: + result.status === 'fulfilled' ? result.value : [] + }), + {} + ) + ); + + const dnsRecords = { + txt: dnsResults.txt.flat(), + mx: dnsResults.mx.map(({ exchange }) => exchange) + }; + + await this.onDetect(this.originalUrl, analyze({ dns: dnsRecords })); + + await this.emit('probe', { dns: dnsRecords }); + + // Deeper paths + const paths = Wappalyzer.technologies + .filter(({ probe }) => probe) + .reduce((paths, { probe }) => { + probe.paths.forEach((path) => { + path = path.replace(/^\//, ''); + + if (!paths.includes(path)) { + paths.push(path); + } + }); + + return paths; + }, []); + + const chunks = []; + + for (let i = 0; i < paths.length; i += this.options.batchSize) { + chunks.push(paths.slice(i, i + this.options.batchSize)); + } + + for (const chunk of chunks) { + await Promise.all( + chunk.map(async (path) => { + const url = new URL(path, this.originalUrl); + + try { + const body = await get.call(this, url, { + userAgent: this.options.userAgent + }); + + await this.onDetect(url, analyze({ probe: { [path]: body } })); + } catch (error) { + // Continue + } + }) + ); + } + } + + async onDetect(url, detections) { + if (detections.length) { + detections.forEach((detection) => { + detection.rootPath = + url.pathname === '/' || + url.pathname === + (this.analyzedUrls[this.originalUrl.href] || {}).pathname; + + this.log( + `Detected ${detection.name}${ + detection.version ? ` ${detection.version}` : '' + } (confidence ${detection.confidence}%)`, + 'driver' + ); + + this.emit('detected', { detection }); + }); + + this.detections = this.detections.concat(detections); + } + } + + async newPage(url) { + if (!this.browser) { + await this.initDriver(); + } + + const page = await this.browser.newPage(); + + this.pages.push(page); + + page.on('close', () => { + page.__closed = true; + + this.pages = this.pages.filter((_page) => _page !== page); + }); + + await page.setUserAgent(this.options.userAgent); + + if (this.options.noScripts) { + await page.setJavaScriptEnabled(false); + } + + return page; + } +} + +module.exports = Driver; diff --git a/src/technologies/l.json b/src/technologies/l.json index 156c1e7a4..a392ef87b 100644 --- a/src/technologies/l.json +++ b/src/technologies/l.json @@ -2120,4 +2120,4 @@ "oss": true, "website": "https://github.com/paulirish/lite-youtube-embed" } -} +} \ No newline at end of file diff --git a/src/technologies/o.json b/src/technologies/o.json index 581badf8c..420d23268 100644 --- a/src/technologies/o.json +++ b/src/technologies/o.json @@ -2023,4 +2023,4 @@ }, "website": "https://owncloud.org" } -} +} \ No newline at end of file diff --git a/src/technologies/p.json b/src/technologies/p.json index 59d4c6372..fca864144 100644 --- a/src/technologies/p.json +++ b/src/technologies/p.json @@ -3991,4 +3991,4 @@ }, "website": "https://punbb.informer.com" } -} +} \ No newline at end of file diff --git a/src/technologies/s.json b/src/technologies/s.json index 881b3059e..a00b8307c 100644 --- a/src/technologies/s.json +++ b/src/technologies/s.json @@ -7533,4 +7533,4 @@ }, "website": "https://styled-components.com" } -} +} \ No newline at end of file