diff --git a/lib/install.js b/lib/install.js index 82705428..4e6f897d 100644 --- a/lib/install.js +++ b/lib/install.js @@ -12,8 +12,8 @@ const versioning = require('./util/versioning.js'); const napi = require('./util/napi.js'); const s3_setup = require('./util/s3_setup.js'); const url = require('url'); -// for fetching binaries -const fetch = require('node-fetch'); +const https = require('https'); +const http = require('http'); const tar = require('tar'); let npgVersion = 'unknown'; @@ -84,6 +84,51 @@ function place_binary_authenticated(opts, targetDir, callback) { } } +/** + * Downloads a URL following redirects, using Node's built-in http/https modules. + * Calls back with (err, IncomingMessage) — the response is a Readable stream + * that can be .pipe()'d directly. + * + * @param {string} uri + * @param {{ headers?: object, agent?: object, ca?: string|Buffer }} opts + * @param {number} maxRedirects + * @param {(err: Error|null, res?: import('http').IncomingMessage) => void} callback + */ +function fetch_with_redirects(uri, opts, maxRedirects, callback) { + let parsed; + try { + parsed = new URL(uri); + } catch (e) { + return callback(new Error(`Invalid URL: ${uri}`)); + } + + const mod = parsed.protocol === 'https:' ? https : http; + const reqOpts = { + hostname: parsed.hostname, + port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80), + path: parsed.pathname + parsed.search, + headers: opts.headers || {}, + agent: opts.agent || undefined, + ca: opts.ca || undefined + }; + + const req = mod.get(reqOpts, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + res.resume(); // release socket + if (maxRedirects === 0) { + return callback(new Error('Too many redirects')); + } + const next = new URL(res.headers.location, uri).href; + return fetch_with_redirects(next, opts, maxRedirects - 1, callback); + } + callback(null, res); + }); + + req.on('error', callback); +} + +exports.fetch_with_redirects = fetch_with_redirects; + function place_binary(uri, targetDir, opts, callback) { log.log('GET', uri); @@ -92,22 +137,16 @@ function place_binary(uri, targetDir, opts, callback) { 'node ' + process.version; const sanitized = uri.replace('+', '%2B'); - const requestOpts = { - uri: sanitized, - headers: { - 'User-Agent': 'node-pre-gyp (v' + npgVersion + ', ' + envVersionInfo + ')' - }, - follow_max: 10 - }; + let ca; if (opts.cafile) { try { - requestOpts.ca = fs.readFileSync(opts.cafile); + ca = fs.readFileSync(opts.cafile); } catch (e) { return callback(e); } } else if (opts.ca) { - requestOpts.ca = opts.ca; + ca = opts.ca; } const proxyUrl = opts.proxy || @@ -121,51 +160,49 @@ function place_binary(uri, targetDir, opts, callback) { log.log('download', `proxy agent configured using: "${proxyUrl}"`); } - fetch(sanitized, { agent }) - .then((res) => { - if (!res.ok) { - // If we get 403 Forbidden, the binary might be private - try authenticated download - if (res.status === 403) { - log.info('install', 'Received 403 Forbidden - attempting authenticated download'); - // Call place_binary_authenticated and return a special marker - // to prevent the promise chain from calling callback again - place_binary_authenticated(opts, targetDir, callback); - return { authenticated: true }; - } - throw new Error(`response status ${res.status} ${res.statusText} on ${sanitized}`); + fetch_with_redirects(sanitized, { + headers: { 'User-Agent': 'node-pre-gyp (v' + npgVersion + ', ' + envVersionInfo + ')' }, + agent, + ca + }, 10, (err, res) => { + if (err) { + log.error(`install ${err.message}`); + return callback(err); + } + + const ok = res.statusCode >= 200 && res.statusCode < 300; + if (!ok) { + res.resume(); // release socket + if (res.statusCode === 403) { + log.info('install', 'Received 403 Forbidden - attempting authenticated download'); + return place_binary_authenticated(opts, targetDir, callback); } - const dataStream = res.body; - - return new Promise((resolve, reject) => { - let extractions = 0; - const countExtractions = (entry) => { - extractions += 1; - log.info('install', `unpacking ${entry.path}`); - }; - - dataStream.pipe(extract(targetDir, countExtractions)) - .on('error', (e) => { - reject(e); - }); - dataStream.on('end', () => { - resolve(`extracted file count: ${extractions}`); - }); - dataStream.on('error', (e) => { - reject(e); - }); + return callback(new Error(`response status ${res.statusCode} ${res.statusMessage} on ${sanitized}`)); + } + + new Promise((resolve, reject) => { + let extractions = 0; + const countExtractions = (entry) => { + extractions += 1; + log.info('install', `unpacking ${entry.path}`); + }; + + res.pipe(extract(targetDir, countExtractions)) + .on('error', reject); + res.on('end', () => { + resolve(`extracted file count: ${extractions}`); }); + res.on('error', reject); }) - .then((text) => { - if (text && text.authenticated) { - return; // Don't call callback - place_binary_authenticated will handle it - } - log.info(text); - callback(); - }) - .catch((e) => { - log.error(`install ${e.message}`); - callback(e); - }); + .then((text) => { + log.info(text); + callback(); + }) + .catch((e) => { + log.error(`install ${e.message}`); + callback(e); + }); + }); } function extract(to, onentry) { diff --git a/lib/util/abi_crosswalk.json b/lib/util/abi_crosswalk.json index 2b376e16..cbc007e7 100644 --- a/lib/util/abi_crosswalk.json +++ b/lib/util/abi_crosswalk.json @@ -3119,6 +3119,38 @@ "node_abi": 115, "v8": "11.3" }, + "20.19.2": { + "node_abi": 115, + "v8": "11.3" + }, + "20.19.3": { + "node_abi": 115, + "v8": "11.3" + }, + "20.19.4": { + "node_abi": 115, + "v8": "11.3" + }, + "20.19.5": { + "node_abi": 115, + "v8": "11.3" + }, + "20.19.6": { + "node_abi": 115, + "v8": "11.3" + }, + "20.20.0": { + "node_abi": 115, + "v8": "11.3" + }, + "20.20.1": { + "node_abi": 115, + "v8": "11.3" + }, + "20.20.2": { + "node_abi": 115, + "v8": "11.3" + }, "21.0.0": { "node_abi": 120, "v8": "11.8" @@ -3247,6 +3279,58 @@ "node_abi": 127, "v8": "12.4" }, + "22.15.1": { + "node_abi": 127, + "v8": "12.4" + }, + "22.16.0": { + "node_abi": 127, + "v8": "12.4" + }, + "22.17.0": { + "node_abi": 127, + "v8": "12.4" + }, + "22.17.1": { + "node_abi": 127, + "v8": "12.4" + }, + "22.18.0": { + "node_abi": 127, + "v8": "12.4" + }, + "22.19.0": { + "node_abi": 127, + "v8": "12.4" + }, + "22.20.0": { + "node_abi": 127, + "v8": "12.4" + }, + "22.21.0": { + "node_abi": 127, + "v8": "12.4" + }, + "22.21.1": { + "node_abi": 127, + "v8": "12.4" + }, + "22.22.0": { + "node_abi": 127, + "v8": "12.4" + }, + "22.22.1": { + "node_abi": 127, + "v8": "12.4" + }, + "22.22.2": { + "node_abi": 127, + "v8": "12.4" + }, + "22.22.3": { + "node_abi": 127, + "v8": "12.4" + }, "23.0.0": { "node_abi": 131, "v8": "12.9" @@ -3299,8 +3383,168 @@ "node_abi": 131, "v8": "12.9" }, + "23.11.1": { + "node_abi": 131, + "v8": "12.9" + }, "24.0.0": { "node_abi": 137, "v8": "13.6" + }, + "24.0.1": { + "node_abi": 137, + "v8": "13.6" + }, + "24.0.2": { + "node_abi": 137, + "v8": "13.6" + }, + "24.1.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.2.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.3.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.4.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.4.1": { + "node_abi": 137, + "v8": "13.6" + }, + "24.5.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.6.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.7.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.8.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.9.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.10.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.11.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.11.1": { + "node_abi": 137, + "v8": "13.6" + }, + "24.12.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.13.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.13.1": { + "node_abi": 137, + "v8": "13.6" + }, + "24.14.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.14.1": { + "node_abi": 137, + "v8": "13.6" + }, + "24.15.0": { + "node_abi": 137, + "v8": "13.6" + }, + "24.16.0": { + "node_abi": 137, + "v8": "13.6" + }, + "25.0.0": { + "node_abi": 141, + "v8": "14.1" + }, + "25.1.0": { + "node_abi": 141, + "v8": "14.1" + }, + "25.2.0": { + "node_abi": 141, + "v8": "14.1" + }, + "25.2.1": { + "node_abi": 141, + "v8": "14.1" + }, + "25.3.0": { + "node_abi": 141, + "v8": "14.1" + }, + "25.4.0": { + "node_abi": 141, + "v8": "14.1" + }, + "25.5.0": { + "node_abi": 141, + "v8": "14.1" + }, + "25.6.0": { + "node_abi": 141, + "v8": "14.1" + }, + "25.6.1": { + "node_abi": 141, + "v8": "14.1" + }, + "25.7.0": { + "node_abi": 141, + "v8": "14.1" + }, + "25.8.0": { + "node_abi": 141, + "v8": "14.1" + }, + "25.8.1": { + "node_abi": 141, + "v8": "14.1" + }, + "25.8.2": { + "node_abi": 141, + "v8": "14.1" + }, + "25.9.0": { + "node_abi": 141, + "v8": "14.1" + }, + "26.0.0": { + "node_abi": 147, + "v8": "14.6" + }, + "26.1.0": { + "node_abi": 147, + "v8": "14.6" + }, + "26.2.0": { + "node_abi": 147, + "v8": "14.6" } -} +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 49fe71cb..80a2f7bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,6 @@ "consola": "^3.2.3", "detect-libc": "^2.0.0", "https-proxy-agent": "^7.0.5", - "node-fetch": "^2.6.7", "nopt": "^8.0.0", "semver": "^7.5.3", "tar": "^7.5.7" @@ -5911,6 +5910,7 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, "license": "MIT", "dependencies": { "whatwg-url": "^5.0.0" @@ -7378,6 +7378,7 @@ "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, "license": "MIT" }, "node_modules/ts-api-utils": { @@ -7663,12 +7664,14 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, "license": "BSD-2-Clause" }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, "license": "MIT", "dependencies": { "tr46": "~0.0.3", diff --git a/package.json b/package.json index 8fa9d3f4..c77cae99 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "consola": "^3.2.3", "detect-libc": "^2.0.0", "https-proxy-agent": "^7.0.5", - "node-fetch": "^2.6.7", "nopt": "^8.0.0", "semver": "^7.5.3", "tar": "^7.5.7" diff --git a/test/proxy-bcrypt.test.js b/test/proxy-bcrypt.test.js index 72c43c4a..6e08ff57 100644 --- a/test/proxy-bcrypt.test.js +++ b/test/proxy-bcrypt.test.js @@ -7,18 +7,15 @@ const os = require('os'); const tar = require('tar-fs'); const { HttpsProxyAgent } = require('https-proxy-agent'); -const fetch = require('node-fetch'); const { rimraf } = require('rimraf'); const test = require('tape'); const proxy = require('./proxy.util'); +const { fetch_with_redirects } = require('../lib/install.js'); const proxyPort = 8124; const proxyServer = `http://localhost:${proxyPort}`; -// options for fetch -const options = {}; - // the temporary download directory and file const downloadDir = `${os.tmpdir()}/npg-download`; @@ -44,8 +41,6 @@ test('setup proxy server', (t) => { proxy.startServer({ port: proxyPort }); process.env.https_proxy = process.env.http_proxy = proxyServer; - options.agent = new HttpsProxyAgent(proxyServer); - // make sure the download directory deleted then create an empty one rimraf(downloadDir).then(() => { fs.mkdir('download', (e) => { @@ -56,21 +51,13 @@ test('setup proxy server', (t) => { t.end(); }); }); - - }); -test('verify node fetch with a proxy successfully downloads bcrypt pre-built', (t) => { +test('verify download with a proxy successfully downloads bcrypt pre-built', (t) => { // "{module_name}-v{version}-napi-v{napi_build_version}-{platform}-{arch}-{libc}.tar.gz" const url = 'https://github.com/kelektiv/node.bcrypt.js/releases/download/v5.0.1/bcrypt_lib-v5.0.1-napi-v3-linux-x64-glibc.tar.gz'; - async function getBcrypt() { - const res = await fetch(url, options); - if (res.status !== 200) { - throw new Error(`fetch got error ${res.status}`); - } - return res.body; - } + const agent = new HttpsProxyAgent(proxyServer); const withDir = path.join(downloadDir, 'napi-v3', 'bcrypt_lib.node'); const rawPath = 'napi-v3/bcrypt_lib.node'; @@ -85,21 +72,19 @@ test('verify node fetch with a proxy successfully downloads bcrypt pre-built', ( } }; - getBcrypt() + new Promise((resolve, reject) => { + fetch_with_redirects(url, { agent }, 10, (err, res) => { + if (err) return reject(err); + if (res.statusCode !== 200) return reject(new Error(`fetch got error ${res.statusCode}`)); + resolve(res); + }); + }) .then((stream) => { const unzip = createUnzip(); - stream - .pipe(unzip); - - unzip - .pipe(tar.extract(`${downloadDir}`, tarOptions)); - - return unzip; - }) - .then((stream) => { + stream.pipe(unzip).pipe(tar.extract(`${downloadDir}`, tarOptions)); return new Promise((resolve, reject) => { - stream.on('end', resolve); - stream.on('error', reject); + unzip.on('end', resolve); + unzip.on('error', reject); }); }) // if no errors on download and the file is there that's good enough. napi version